feat: derived stream (#4160)

`DerivedStream`, as part of `Pipeline`, enables the use case of querying
an existing stream, `source`, with optional conditions and aggregations.
The resulted data will then be ingested into a new stream,
`destination`.

This PR provides the feature's initial offering. Please follow below
issue to track & suggest future improvements
#4170 

Co-authored-by: omkark06 <omkar@zinclabs.io>
Co-authored-by: hengfei yang <hengfei.yang@gmail.com>

src/cli/basic/cli.rs

@@ -161,7 +161,7 @@ pub async fn cli() -> Result<bool, anyhow::Error> {
                     db::user::reset().await?;
                 }
                 "alert" => {
-                    db::alerts::reset().await?;
+                    db::alerts::alert::reset().await?;
                 }
                 "dashboard" => {
                     db::dashboards::reset().await?;

src/common/infra/config.rs

@@ -24,7 +24,7 @@ use vector_enrichment::TableRegistry;
 
 use crate::{
     common::meta::{
-        alerts,
+        alerts::{alert::Alert, destinations::Destination, templates::Template},
         dashboards::reports,
         functions::{StreamFunctionsList, Transform},
         maxmind::MaxmindClient,
@@ -60,14 +60,11 @@ pub static METRIC_CLUSTER_MAP: Lazy<Arc<RwAHashMap<String, Vec<String>>>> =
     Lazy::new(|| Arc::new(tokio::sync::RwLock::new(HashMap::new())));
 pub static METRIC_CLUSTER_LEADER: Lazy<Arc<RwAHashMap<String, ClusterLeader>>> =
     Lazy::new(|| Arc::new(tokio::sync::RwLock::new(HashMap::new())));
-pub static STREAM_ALERTS: Lazy<RwAHashMap<String, Vec<alerts::Alert>>> =
-    Lazy::new(Default::default);
+pub static STREAM_ALERTS: Lazy<RwAHashMap<String, Vec<Alert>>> = Lazy::new(Default::default);
 pub static REALTIME_ALERT_TRIGGERS: Lazy<RwAHashMap<String, db_scheduler::Trigger>> =
     Lazy::new(Default::default);
-pub static ALERTS_TEMPLATES: Lazy<RwHashMap<String, alerts::templates::Template>> =
-    Lazy::new(Default::default);
-pub static ALERTS_DESTINATIONS: Lazy<RwHashMap<String, alerts::destinations::Destination>> =
-    Lazy::new(Default::default);
+pub static ALERTS_TEMPLATES: Lazy<RwHashMap<String, Template>> = Lazy::new(Default::default);
+pub static ALERTS_DESTINATIONS: Lazy<RwHashMap<String, Destination>> = Lazy::new(Default::default);
 pub static DASHBOARD_REPORTS: Lazy<RwHashMap<String, reports::Report>> =
     Lazy::new(Default::default);
 pub static SYSLOG_ROUTES: Lazy<RwHashMap<String, SyslogRoute>> = Lazy::new(Default::default);

src/common/meta/alerts/alert.rs

@@ -0,0 +1,80 @@
+// Copyright 2024 Zinc Labs Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use config::meta::stream::StreamType;
+use hashbrown::HashMap;
+use serde::{Deserialize, Serialize};
+use utoipa::ToSchema;
+
+use super::{QueryCondition, TriggerCondition};
+
+#[derive(Clone, Debug, Serialize, Deserialize, ToSchema)]
+pub struct Alert {
+    #[serde(default)]
+    pub name: String,
+    #[serde(default)]
+    pub org_id: String,
+    #[serde(default)]
+    pub stream_type: StreamType,
+    #[serde(default)]
+    pub stream_name: String,
+    #[serde(default)]
+    pub is_real_time: bool,
+    #[serde(default)]
+    pub query_condition: QueryCondition,
+    #[serde(default)]
+    pub trigger_condition: TriggerCondition,
+    pub destinations: Vec<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub context_attributes: Option<HashMap<String, String>>,
+    #[serde(default)]
+    pub row_template: String,
+    #[serde(default)]
+    pub description: String,
+    #[serde(default)]
+    pub enabled: bool,
+    #[serde(default)]
+    /// Timezone offset in minutes.
+    /// The negative secs means the Western Hemisphere
+    pub tz_offset: i32,
+}
+
+impl PartialEq for Alert {
+    fn eq(&self, other: &Self) -> bool {
+        self.name == other.name
+            && self.stream_type == other.stream_type
+            && self.stream_name == other.stream_name
+    }
+}
+
+impl Default for Alert {
+    fn default() -> Self {
+        Self {
+            name: "".to_string(),
+            org_id: "".to_string(),
+            stream_type: StreamType::default(),
+            stream_name: "".to_string(),
+            is_real_time: false,
+            query_condition: QueryCondition::default(),
+            trigger_condition: TriggerCondition::default(),
+            destinations: vec![],
+            context_attributes: None,
+            row_template: "".to_string(),
+            description: "".to_string(),
+            enabled: false,
+            tz_offset: 0, // UTC
+        }
+    }
+}

src/common/meta/alerts/derived_streams.rs

@@ -0,0 +1,66 @@
+// Copyright 2024 Zinc Labs Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use hashbrown::HashMap;
+use serde::{Deserialize, Serialize};
+use utoipa::ToSchema;
+
+use crate::common::meta::{
+    alerts::{QueryCondition, TriggerCondition},
+    stream::StreamParams,
+};
+
+#[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)]
+pub struct DerivedStreamMeta {
+    #[serde(default)]
+    pub name: String,
+    #[serde(default)]
+    pub source: StreamParams,
+    #[serde(default)]
+    pub destination: StreamParams,
+    #[serde(default)]
+    pub is_real_time: bool,
+    #[serde(default)]
+    pub query_condition: QueryCondition,
+    #[serde(default)]
+    pub trigger_condition: TriggerCondition, // Frequency type only supports minutes
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub context_attributes: Option<HashMap<String, String>>,
+    #[serde(default)]
+    pub description: String,
+    #[serde(default)]
+    pub enabled: bool,
+    #[serde(default)]
+    /// Timezone offset in minutes.
+    /// The negative secs means the Western Hemisphere
+    pub tz_offset: i32,
+}
+
+impl Default for DerivedStreamMeta {
+    fn default() -> Self {
+        Self {
+            name: "".to_string(),
+            source: StreamParams::default(),
+            destination: StreamParams::default(),
+            is_real_time: false,
+            query_condition: QueryCondition::default(),
+            trigger_condition: TriggerCondition::default(),
+            context_attributes: None,
+            description: "".to_string(),
+            enabled: true,
+            tz_offset: 0, // UTC
+        }
+    }
+}

src/common/meta/alerts/mod.rs

@@ -13,74 +13,16 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use config::{meta::stream::StreamType, utils::json::Value};
-use hashbrown::HashMap;
+use config::{meta::search::SearchEventType, utils::json::Value};
 use serde::{Deserialize, Serialize};
 use utoipa::ToSchema;
 
+pub mod alert;
+pub mod derived_streams;
 pub mod destinations;
 pub mod templates;
 
-#[derive(Clone, Debug, Serialize, Deserialize, ToSchema)]
-pub struct Alert {
-    #[serde(default)]
-    pub name: String,
-    #[serde(default)]
-    pub org_id: String,
-    #[serde(default)]
-    pub stream_type: StreamType,
-    #[serde(default)]
-    pub stream_name: String,
-    #[serde(default)]
-    pub is_real_time: bool,
-    #[serde(default)]
-    pub query_condition: QueryCondition,
-    #[serde(default)]
-    pub trigger_condition: TriggerCondition,
-    pub destinations: Vec<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub context_attributes: Option<HashMap<String, String>>,
-    #[serde(default)]
-    pub row_template: String,
-    #[serde(default)]
-    pub description: String,
-    #[serde(default)]
-    pub enabled: bool,
-    #[serde(default)]
-    /// Timezone offset in minutes.
-    /// The negative secs means the Western Hemisphere
-    pub tz_offset: i32,
-}
-
-impl PartialEq for Alert {
-    fn eq(&self, other: &Self) -> bool {
-        self.name == other.name
-            && self.stream_type == other.stream_type
-            && self.stream_name == other.stream_name
-    }
-}
-
-impl Default for Alert {
-    fn default() -> Self {
-        Self {
-            name: "".to_string(),
-            org_id: "".to_string(),
-            stream_type: StreamType::default(),
-            stream_name: "".to_string(),
-            is_real_time: false,
-            query_condition: QueryCondition::default(),
-            trigger_condition: TriggerCondition::default(),
-            destinations: vec![],
-            context_attributes: None,
-            row_template: "".to_string(),
-            description: "".to_string(),
-            enabled: false,
-            tz_offset: 0, // UTC
-        }
-    }
-}
-
-#[derive(Clone, Debug, Default, Serialize, Deserialize, ToSchema)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize, ToSchema, PartialEq)]
 pub struct TriggerCondition {
     pub period: i64, // 10 minutes
     #[serde(default)]
@@ -92,13 +34,13 @@ pub struct TriggerCondition {
     #[serde(default)]
     pub cron: String, // Cron Expression
     #[serde(default)]
-    pub frequency_type: AlertFrequencyType,
+    pub frequency_type: FrequencyType,
     #[serde(default)]
     pub silence: i64, // silence for 10 minutes after fire an alert
 }
 
 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, ToSchema)]
-pub enum AlertFrequencyType {
+pub enum FrequencyType {
     #[serde(rename = "cron")]
     Cron,
     #[serde(rename = "minutes")]
@@ -106,7 +48,7 @@ pub enum AlertFrequencyType {
     Minutes,
 }
 
-#[derive(Clone, Debug, Default, Serialize, Deserialize, ToSchema)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize, ToSchema, PartialEq)]
 pub struct QueryCondition {
     #[serde(default)]
     #[serde(rename = "type")]
@@ -118,9 +60,11 @@ pub struct QueryCondition {
     pub aggregation: Option<Aggregation>,
     #[serde(default)]
     pub vrl_function: Option<String>,
+    #[serde(default)]
+    pub search_event_type: Option<SearchEventType>,
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize, ToSchema)]
+#[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)]
 pub struct Aggregation {
     pub group_by: Option<Vec<String>>,
     pub function: AggFunction,

src/common/meta/dashboards/v4/mod.rs

@@ -214,7 +214,6 @@ pub struct DrillDown {
     data: Option<DrillDownData>,
 }
 
-
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct MarkLine {

src/common/meta/organization.rs

@@ -16,7 +16,7 @@
 use serde::{Deserialize, Serialize};
 use utoipa::ToSchema;
 
-use super::{alerts::Alert, functions::Transform};
+use super::{alerts::alert::Alert, functions::Transform};
 
 pub const DEFAULT_ORG: &str = "default";
 pub const CUSTOM: &str = "custom";

src/common/meta/pipelines.rs

@@ -22,7 +22,9 @@ use config::{
 use serde::{Deserialize, Serialize};
 use utoipa::ToSchema;
 
-use crate::common::meta::functions::StreamFunctionsList;
+use crate::common::meta::{
+    alerts::derived_streams::DerivedStreamMeta, functions::StreamFunctionsList,
+};
 
 #[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)]
 pub struct PipeLine {
@@ -36,6 +38,8 @@ pub struct PipeLine {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub routing: Option<HashMap<String, Vec<RoutingCondition>>>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    pub derived_streams: Option<Vec<DerivedStreamMeta>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub meta: Option<HashMap<String, Value>>,
 }
 
@@ -47,6 +51,7 @@ impl PipeLine {
             stream_name: self.stream_name,
             stream_type: self.stream_type,
             routing: self.routing,
+            derived_streams: self.derived_streams,
             functions,
             meta: self.meta,
         }
@@ -67,6 +72,8 @@ pub struct PipeLineResponse {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub functions: Option<StreamFunctionsList>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    pub derived_streams: Option<Vec<DerivedStreamMeta>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub meta: Option<HashMap<String, Value>>,
 }
 

src/common/meta/stream.rs

@@ -64,13 +64,31 @@ pub struct ListStream {
     pub list: Vec<Stream>,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct StreamParams {
     pub org_id: faststr::FastStr,
     pub stream_name: faststr::FastStr,
     pub stream_type: StreamType,
 }
 
+impl PartialEq for StreamParams {
+    fn eq(&self, other: &Self) -> bool {
+        self.org_id == other.org_id
+            && self.stream_name == other.stream_name
+            && self.stream_type == other.stream_type
+    }
+}
+
+impl Default for StreamParams {
+    fn default() -> Self {
+        Self {
+            org_id: String::default().into(),
+            stream_name: String::default().into(),
+            stream_type: StreamType::default(),
+        }
+    }
+}
+
 impl StreamParams {
     pub fn new(org_id: &str, stream_name: &str, stream_type: StreamType) -> Self {
         Self {
@@ -79,6 +97,10 @@ impl StreamParams {
             stream_type,
         }
     }
+
+    pub fn is_valid(&self) -> bool {
+        !(self.org_id.is_empty() || self.stream_name.is_empty())
+    }
 }
 
 pub struct SchemaEvolution {

src/common/utils/http.rs

@@ -62,6 +62,7 @@ pub(crate) fn get_search_type_from_request(
             "alerts" => Some(SearchEventType::Alerts),
             "values" => Some(SearchEventType::Values),
             "rum" => Some(SearchEventType::RUM),
+            "derived_stream" => Some(SearchEventType::DerivedStream),
             _ => {
                 return Err(Error::new(
                     ErrorKind::Other,

src/config/src/config.rs

@@ -872,6 +872,8 @@ pub struct Limit {
     pub alert_schedule_timeout: i64,
     #[env_config(name = "ZO_REPORT_SCHEDULE_TIMEOUT", default = 300)] // seconds
     pub report_schedule_timeout: i64,
+    #[env_config(name = "ZO_DERIVED_STREAM_SCHEDULE_INTERVAL", default = 300)] // seconds
+    pub derived_stream_schedule_interval: i64,
     #[env_config(name = "ZO_SCHEDULER_MAX_RETRIES", default = 3)]
     pub scheduler_max_retries: i32,
     #[env_config(name = "ZO_SCHEDULER_CLEAN_INTERVAL", default = 30)] // seconds

src/config/src/meta/cluster.rs

@@ -164,7 +164,9 @@ impl From<&str> for RoleGroup {
 impl From<SearchEventType> for RoleGroup {
     fn from(value: SearchEventType) -> Self {
         match value {
-            SearchEventType::Reports | SearchEventType::Alerts => RoleGroup::Background,
+            SearchEventType::Reports | SearchEventType::Alerts | SearchEventType::DerivedStream => {
+                RoleGroup::Background
+            }
             _ => RoleGroup::Interactive,
         }
     }

src/config/src/meta/search.rs

@@ -506,18 +506,20 @@ pub enum SearchEventType {
     Values,
     Other,
     RUM,
+    DerivedStream,
 }
 
 impl std::fmt::Display for SearchEventType {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         match self {
-            SearchEventType::UI => write!(f, "UI"),
-            SearchEventType::Dashboards => write!(f, "Dashboards"),
-            SearchEventType::Reports => write!(f, "Reports"),
-            SearchEventType::Alerts => write!(f, "Alerts"),
-            SearchEventType::Other => write!(f, "Other"),
+            SearchEventType::UI => write!(f, "ui"),
+            SearchEventType::Dashboards => write!(f, "dashboards"),
+            SearchEventType::Reports => write!(f, "reports"),
+            SearchEventType::Alerts => write!(f, "alerts"),
             SearchEventType::Values => write!(f, "_values"),
-            SearchEventType::RUM => write!(f, "RUM"),
+            SearchEventType::Other => write!(f, "other"),
+            SearchEventType::RUM => write!(f, "rum"),
+            SearchEventType::DerivedStream => write!(f, "derived_stream"),
         }
     }
 }
@@ -531,9 +533,10 @@ impl FromStr for SearchEventType {
             "dashboards" => Ok(SearchEventType::Dashboards),
             "reports" => Ok(SearchEventType::Reports),
             "alerts" => Ok(SearchEventType::Alerts),
-            "values" => Ok(SearchEventType::Values),
+            "values" | "_values" => Ok(SearchEventType::Values),
             "other" => Ok(SearchEventType::Other),
             "rum" => Ok(SearchEventType::RUM),
+            "derived_stream" | "derivedstream" => Ok(SearchEventType::DerivedStream),
             _ => Err(format!("Invalid search event type: {s}")),
         }
     }

src/config/src/meta/stream.rs

@@ -80,6 +80,20 @@ impl From<&str> for StreamType {
     }
 }
 
+impl From<StreamType> for cluster_rpc::StreamType {
+    fn from(value: StreamType) -> Self {
+        match value {
+            StreamType::Logs => cluster_rpc::StreamType::Logs,
+            StreamType::Metrics => cluster_rpc::StreamType::Metrics,
+            StreamType::Traces => cluster_rpc::StreamType::Traces,
+            StreamType::EnrichmentTables => cluster_rpc::StreamType::EnrichmentTables,
+            StreamType::Filelist => cluster_rpc::StreamType::Filelist,
+            StreamType::Metadata => cluster_rpc::StreamType::Metadata,
+            StreamType::Index => cluster_rpc::StreamType::Index,
+        }
+    }
+}
+
 impl std::fmt::Display for StreamType {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         match self {

src/config/src/meta/usage.rs

@@ -41,6 +41,8 @@ pub enum TriggerDataType {
     Report,
     #[serde(rename = "alert")]
     Alert,
+    #[serde(rename = "derived_stream")]
+    DerivedStream,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]

src/handler/grpc/request/ingest.rs

@@ -0,0 +1,101 @@
+// Copyright 2024 Zinc Labs Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use async_trait::async_trait;
+use config::utils::json;
+use proto::cluster_rpc::{ingest_server::Ingest, IngestionRequest, IngestionResponse, StreamType};
+use tonic::{Request, Response, Status};
+
+use crate::service::ingestion::create_log_ingestion_req;
+
+#[derive(Debug, Default)]
+pub struct Ingester;
+
+#[async_trait]
+impl Ingest for Ingester {
+    async fn ingest(
+        &self,
+        request: Request<IngestionRequest>,
+    ) -> Result<Response<IngestionResponse>, Status> {
+        // let metadata = request.metadata().clone();
+        let req = request.into_inner();
+        let org_id = req.org_id;
+        let stream_name = req.stream_name;
+        let in_data = req.data.unwrap_or_default();
+
+        let resp = match StreamType::try_from(req.stream_type) {
+            Ok(StreamType::Logs) => {
+                let log_ingestion_type = req.ingestion_type.unwrap_or_default();
+                let data = bytes::Bytes::from(in_data.data);
+                match create_log_ingestion_req(log_ingestion_type, &data) {
+                    Err(e) => Err(e),
+                    Ok(ingestion_req) => crate::service::logs::ingest::ingest(
+                        &org_id,
+                        &stream_name,
+                        ingestion_req,
+                        "",
+                        None,
+                    )
+                    .await
+                    .map_or_else(Err, |_| Ok(())),
+                }
+            }
+            Ok(StreamType::EnrichmentTables) => {
+                let json_records: Vec<json::Map<String, json::Value>> =
+                    json::from_slice(&in_data.data).unwrap_or({
+                        let vec_value: Vec<json::Value> = json::from_slice(&in_data.data).unwrap();
+                        vec_value
+                            .into_iter()
+                            .filter_map(|v| match v {
+                                json::Value::Object(map) => Some(map),
+                                _ => None,
+                            })
+                            .collect()
+                    });
+                crate::service::enrichment_table::save_enrichment_data(
+                    &org_id,
+                    &stream_name,
+                    json_records,
+                    false,
+                )
+                .await
+                .map_or_else(
+                    |e| {
+                        Err(anyhow::anyhow!(
+                            "Internal gPRC ingestion service errors saving enrichment data: {}",
+                            e.to_string()
+                        ))
+                    },
+                    |_| Ok(()),
+                )
+            }
+            _ => Err(anyhow::anyhow!(
+                "Internal gPRC ingestion service currently only supports Logs and EnrichmentTables",
+            )),
+        };
+
+        let reply = match resp {
+            Ok(_) => IngestionResponse {
+                status_code: 200,
+                message: "OK".to_string(),
+            },
+            Err(err) => IngestionResponse {
+                status_code: 500,
+                message: err.to_string(),
+            },
+        };
+        Ok(Response::new(reply))
+    }
+}

src/handler/grpc/request/metrics/ingester.rs

@@ -21,10 +21,10 @@ use opentelemetry_proto::tonic::collector::metrics::v1::{
 use tonic::{Response, Status};
 
 #[derive(Default)]
-pub struct Ingester;
+pub struct MetricsIngester;
 
 #[async_trait]
-impl MetricsService for Ingester {
+impl MetricsService for MetricsIngester {
     async fn export(
         &self,
         request: tonic::Request<ExportMetricsServiceRequest>,

src/handler/grpc/request/metrics/querier.rs

@@ -41,10 +41,10 @@ use crate::{
     service::{promql::search as SearchService, search::match_source},
 };
 
-pub struct Querier;
+pub struct MetricsQuerier;
 
 #[tonic::async_trait]
-impl Metrics for Querier {
+impl Metrics for MetricsQuerier {
     #[tracing::instrument(name = "grpc:metrics:query", skip_all, fields(org_id = req.get_ref().org_id))]
     async fn query(
         &self,

src/handler/grpc/request/mod.rs

@@ -17,6 +17,7 @@ use opentelemetry::propagation::Extractor;
 
 pub mod event;
 pub mod file_list;
+pub mod ingest;
 pub mod logs;
 pub mod metrics;
 pub mod query_cache;

src/handler/http/request/alerts/alert.rs

@@ -0,0 +1,373 @@
+// Copyright 2024 Zinc Labs Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{collections::HashMap, io::Error};
+
+use actix_web::{delete, get, http, post, put, web, HttpRequest, HttpResponse};
+
+use crate::{
+    common::{
+        meta::{alerts::alert::Alert, http::HttpResponse as MetaHttpResponse},
+        utils::http::get_stream_type_from_request,
+    },
+    service::alerts::alert,
+};
+
+/// CreateAlert
+#[utoipa::path(
+    context_path = "/api",
+    tag = "Alerts",
+    operation_id = "SaveAlert",
+    security(
+        ("Authorization"= [])
+    ),
+    params(
+        ("org_id" = String, Path, description = "Organization name"),
+        ("stream_name" = String, Path, description = "Stream name"),
+      ),
+    request_body(content = Alert, description = "Alert data", content_type = "application/json"),    
+    responses(
+        (status = 200, description = "Success", content_type = "application/json", body = HttpResponse),
+        (status = 400, description = "Error",   content_type = "application/json", body = HttpResponse),
+    )
+)]
+#[post("/{org_id}/{stream_name}/alerts")]
+pub async fn save_alert(
+    path: web::Path<(String, String)>,
+    alert: web::Json<Alert>,
+) -> Result<HttpResponse, Error> {
+    let (org_id, stream_name) = path.into_inner();
+
+    // Hack for frequency: convert minutes to seconds
+    let mut alert = alert.into_inner();
+    alert.trigger_condition.frequency *= 60;
+
+    match alert::save(&org_id, &stream_name, "", alert, true).await {
+        Ok(_) => Ok(MetaHttpResponse::ok("Alert saved")),
+        Err(e) => Ok(MetaHttpResponse::bad_request(e)),
+    }
+}
+
+/// UpdateAlert
+#[utoipa::path(
+    context_path = "/api",
+    tag = "Alerts",
+    operation_id = "UpdateAlert",
+    security(
+        ("Authorization"= [])
+    ),
+    params(
+        ("org_id" = String, Path, description = "Organization name"),
+        ("stream_name" = String, Path, description = "Stream name"),
+        ("alert_name" = String, Path, description = "Alert name"),
+      ),
+    request_body(content = Alert, description = "Alert data", content_type = "application/json"),    
+    responses(
+        (status = 200, description = "Success", content_type = "application/json", body = HttpResponse),
+        (status = 400, description = "Error",   content_type = "application/json", body = HttpResponse),
+    )
+)]
+#[put("/{org_id}/{stream_name}/alerts/{alert_name}")]
+pub async fn update_alert(
+    path: web::Path<(String, String, String)>,
+    alert: web::Json<Alert>,
+) -> Result<HttpResponse, Error> {
+    let (org_id, stream_name, name) = path.into_inner();
+
+    // Hack for frequency: convert minutes to seconds
+    let mut alert = alert.into_inner();
+    alert.trigger_condition.frequency *= 60;
+    match alert::save(&org_id, &stream_name, &name, alert, false).await {
+        Ok(_) => Ok(MetaHttpResponse::ok("Alert Updated")),
+        Err(e) => Ok(MetaHttpResponse::bad_request(e)),
+    }
+}
+
+/// ListStreamAlerts
+#[utoipa::path(
+    context_path = "/api",
+    tag = "Alerts",
+    operation_id = "ListStreamAlerts",
+    security(
+        ("Authorization"= [])
+    ),
+    params(
+        ("org_id" = String, Path, description = "Organization name"),
+        ("stream_name" = String, Path, description = "Stream name"),
+      ),
+    responses(
+        (status = 200, description = "Success", content_type = "application/json", body = HttpResponse),
+        (status = 400, description = "Error",   content_type = "application/json", body = HttpResponse),
+    )
+)]
+#[get("/{org_id}/{stream_name}/alerts")]
+async fn list_stream_alerts(
+    path: web::Path<(String, String)>,
+    req: HttpRequest,
+) -> Result<HttpResponse, Error> {
+    let (org_id, stream_name) = path.into_inner();
+    let query = web::Query::<HashMap<String, String>>::from_query(req.query_string()).unwrap();
+    let stream_type = match get_stream_type_from_request(&query) {
+        Ok(v) => v,
+        Err(e) => {
+            return Ok(MetaHttpResponse::bad_request(e));
+        }
+    };
+    match alert::list(&org_id, stream_type, Some(stream_name.as_str()), None).await {
+        Ok(mut data) => {
+            // Hack for frequency: convert seconds to minutes
+            for alert in data.iter_mut() {
+                alert.trigger_condition.frequency /= 60;
+            }
+
+            let mut mapdata = HashMap::new();
+            mapdata.insert("list", data);
+            Ok(MetaHttpResponse::json(mapdata))
+        }
+        Err(e) => Ok(MetaHttpResponse::bad_request(e)),
+    }
+}
+
+/// ListAlerts
+#[utoipa::path(
+    context_path = "/api",
+    tag = "Alerts",
+    operation_id = "ListAlerts",
+    security(
+        ("Authorization"= [])
+    ),
+    params(
+        ("org_id" = String, Path, description = "Organization name"),
+      ),
+    responses(
+        (status = 200, description = "Success", content_type = "application/json", body = HttpResponse),
+    )
+)]
+#[get("/{org_id}/alerts")]
+async fn list_alerts(path: web::Path<String>, _req: HttpRequest) -> Result<HttpResponse, Error> {
+    let org_id = path.into_inner();
+
+    let mut _alert_list_from_rbac = None;
+    // Get List of allowed objects
+    #[cfg(feature = "enterprise")]
+    {
+        let user_id = _req.headers().get("user_id").unwrap();
+        match crate::handler::http::auth::validator::list_objects_for_user(
+            &org_id,
+            user_id.to_str().unwrap(),
+            "GET",
+            "alert",
+        )
+        .await
+        {
+            Ok(stream_list) => {
+                _alert_list_from_rbac = stream_list;
+            }
+            Err(e) => {
+                return Ok(crate::common::meta::http::HttpResponse::forbidden(
+                    e.to_string(),
+                ));
+            }
+        }
+        // Get List of allowed objects ends
+    }
+
+    match alert::list(&org_id, None, None, _alert_list_from_rbac).await {
+        Ok(mut data) => {
+            // Hack for frequency: convert seconds to minutes
+            for alert in data.iter_mut() {
+                alert.trigger_condition.frequency /= 60;
+            }
+
+            let mut mapdata = HashMap::new();
+            mapdata.insert("list", data);
+            Ok(MetaHttpResponse::json(mapdata))
+        }
+        Err(e) => Ok(MetaHttpResponse::bad_request(e)),
+    }
+}
+
+/// GetAlertByName
+#[utoipa::path(
+    context_path = "/api",
+    tag = "Alerts",
+    operation_id = "GetAlert",
+    security(
+        ("Authorization"= [])
+    ),
+    params(
+        ("org_id" = String, Path, description = "Organization name"),
+        ("stream_name" = String, Path, description = "Stream name"),
+        ("alert_name" = String, Path, description = "Alert name"),
+      ),
+    responses(
+        (status = 200, description = "Success",  content_type = "application/json", body = Alert),
+        (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse),
+    )
+)]
+#[get("/{org_id}/{stream_name}/alerts/{alert_name}")]
+async fn get_alert(
+    path: web::Path<(String, String, String)>,
+    req: HttpRequest,
+) -> Result<HttpResponse, Error> {
+    let (org_id, stream_name, name) = path.into_inner();
+    let query = web::Query::<HashMap<String, String>>::from_query(req.query_string()).unwrap();
+    let stream_type = match get_stream_type_from_request(&query) {
+        Ok(v) => v.unwrap_or_default(),
+        Err(e) => {
+            return Ok(MetaHttpResponse::bad_request(e));
+        }
+    };
+    match alert::get(&org_id, stream_type, &stream_name, &name).await {
+        Ok(mut data) => {
+            // Hack for frequency: convert seconds to minutes
+            if let Some(ref mut data) = data {
+                data.trigger_condition.frequency /= 60;
+            }
+            Ok(MetaHttpResponse::json(data))
+        }
+        Err(e) => Ok(MetaHttpResponse::not_found(e)),
+    }
+}
+
+/// DeleteAlert
+#[utoipa::path(
+    context_path = "/api",
+    tag = "Alerts",
+    operation_id = "DeleteAlert",
+    security(
+        ("Authorization"= [])
+    ),
+    params(
+        ("org_id" = String, Path, description = "Organization name"),
+        ("stream_name" = String, Path, description = "Stream name"),
+        ("alert_name" = String, Path, description = "Alert name"),
+    ),
+    responses(
+        (status = 200, description = "Success",  content_type = "application/json", body = HttpResponse),
+        (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse),
+        (status = 500, description = "Failure",  content_type = "application/json", body = HttpResponse),
+    )
+)]
+#[delete("/{org_id}/{stream_name}/alerts/{alert_name}")]
+async fn delete_alert(
+    path: web::Path<(String, String, String)>,
+    req: HttpRequest,
+) -> Result<HttpResponse, Error> {
+    let (org_id, stream_name, name) = path.into_inner();
+    let query = web::Query::<HashMap<String, String>>::from_query(req.query_string()).unwrap();
+    let stream_type = match get_stream_type_from_request(&query) {
+        Ok(v) => v.unwrap_or_default(),
+        Err(e) => {
+            return Ok(MetaHttpResponse::bad_request(e));
+        }
+    };
+    match alert::delete(&org_id, stream_type, &stream_name, &name).await {
+        Ok(_) => Ok(MetaHttpResponse::ok("Alert deleted")),
+        Err(e) => match e {
+            (http::StatusCode::NOT_FOUND, e) => Ok(MetaHttpResponse::not_found(e)),
+            (_, e) => Ok(MetaHttpResponse::internal_error(e)),
+        },
+    }
+}
+
+/// EnableAlert
+#[utoipa::path(
+    context_path = "/api",
+    tag = "Alerts",
+    operation_id = "EnableAlert",
+    security(
+        ("Authorization"= [])
+    ),
+    params(
+        ("org_id" = String, Path, description = "Organization name"),
+        ("stream_name" = String, Path, description = "Stream name"),
+        ("alert_name" = String, Path, description = "Alert name"),
+        ("value" = bool, Query, description = "Enable or disable alert"),
+    ),
+    responses(
+        (status = 200, description = "Success",  content_type = "application/json", body = HttpResponse),
+        (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse),
+        (status = 500, description = "Failure",  content_type = "application/json", body = HttpResponse),
+    )
+)]
+#[put("/{org_id}/{stream_name}/alerts/{alert_name}/enable")]
+async fn enable_alert(
+    path: web::Path<(String, String, String)>,
+    req: HttpRequest,
+) -> Result<HttpResponse, Error> {
+    let (org_id, stream_name, name) = path.into_inner();
+    let query = web::Query::<HashMap<String, String>>::from_query(req.query_string()).unwrap();
+    let stream_type = match get_stream_type_from_request(&query) {
+        Ok(v) => v.unwrap_or_default(),
+        Err(e) => {
+            return Ok(MetaHttpResponse::bad_request(e));
+        }
+    };
+    let enable = match query.get("value") {
+        Some(v) => v.parse::<bool>().unwrap_or_default(),
+        None => false,
+    };
+    let mut resp = HashMap::new();
+    resp.insert("enabled".to_string(), enable);
+    match alert::enable(&org_id, stream_type, &stream_name, &name, enable).await {
+        Ok(_) => Ok(MetaHttpResponse::json(resp)),
+        Err(e) => match e {
+            (http::StatusCode::NOT_FOUND, e) => Ok(MetaHttpResponse::not_found(e)),
+            (_, e) => Ok(MetaHttpResponse::internal_error(e)),
+        },
+    }
+}
+
+/// TriggerAlert
+#[utoipa::path(
+    context_path = "/api",
+    tag = "Alerts",
+    operation_id = "TriggerAlert",
+    security(
+        ("Authorization"= [])
+    ),
+    params(
+        ("org_id" = String, Path, description = "Organization name"),
+        ("stream_name" = String, Path, description = "Stream name"),
+        ("alert_name" = String, Path, description = "Alert name"),
+    ),
+    responses(
+        (status = 200, description = "Success",  content_type = "application/json", body = HttpResponse),
+        (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse),
+        (status = 500, description = "Failure",  content_type = "application/json", body = HttpResponse),
+    )
+)]
+#[put("/{org_id}/{stream_name}/alerts/{alert_name}/trigger")]
+async fn trigger_alert(
+    path: web::Path<(String, String, String)>,
+    req: HttpRequest,
+) -> Result<HttpResponse, Error> {
+    let (org_id, stream_name, name) = path.into_inner();
+    let query = web::Query::<HashMap<String, String>>::from_query(req.query_string()).unwrap();
+    let stream_type = match get_stream_type_from_request(&query) {
+        Ok(v) => v.unwrap_or_default(),
+        Err(e) => {
+            return Ok(MetaHttpResponse::bad_request(e));
+        }
+    };
+    match alert::trigger(&org_id, stream_type, &stream_name, &name).await {
+        Ok(_) => Ok(MetaHttpResponse::ok("Alert triggered")),
+        Err(e) => match e {
+            (http::StatusCode::NOT_FOUND, e) => Ok(MetaHttpResponse::not_found(e)),
+            (_, e) => Ok(MetaHttpResponse::internal_error(e)),
+        },
+    }
+}

src/handler/http/request/alerts/mod.rs

@@ -13,364 +13,6 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::{collections::HashMap, io::Error};
-
-use actix_web::{delete, get, http, post, put, web, HttpRequest, HttpResponse};
-
-use crate::{
-    common::{
-        meta::{alerts::Alert, http::HttpResponse as MetaHttpResponse},
-        utils::http::get_stream_type_from_request,
-    },
-    service::alerts,
-};
-
+pub mod alert;
 pub mod destinations;
 pub mod templates;
-
-/// CreateAlert
-#[utoipa::path(
-    context_path = "/api",
-    tag = "Alerts",
-    operation_id = "SaveAlert",
-    security(
-        ("Authorization"= [])
-    ),
-    params(
-        ("org_id" = String, Path, description = "Organization name"),
-        ("stream_name" = String, Path, description = "Stream name"),
-      ),
-    request_body(content = Alert, description = "Alert data", content_type = "application/json"),    
-    responses(
-        (status = 200, description = "Success", content_type = "application/json", body = HttpResponse),
-        (status = 400, description = "Error",   content_type = "application/json", body = HttpResponse),
-    )
-)]
-#[post("/{org_id}/{stream_name}/alerts")]
-pub async fn save_alert(
-    path: web::Path<(String, String)>,
-    alert: web::Json<Alert>,
-) -> Result<HttpResponse, Error> {
-    let (org_id, stream_name) = path.into_inner();
-
-    // Hack for frequency: convert minutes to seconds
-    let mut alert = alert.into_inner();
-    alert.trigger_condition.frequency *= 60;
-
-    match alerts::save(&org_id, &stream_name, "", alert, true).await {
-        Ok(_) => Ok(MetaHttpResponse::ok("Alert saved")),
-        Err(e) => Ok(MetaHttpResponse::bad_request(e)),
-    }
-}
-
-/// UpdateAlert
-#[utoipa::path(
-    context_path = "/api",
-    tag = "Alerts",
-    operation_id = "UpdateAlert",
-    security(
-        ("Authorization"= [])
-    ),
-    params(
-        ("org_id" = String, Path, description = "Organization name"),
-        ("stream_name" = String, Path, description = "Stream name"),
-        ("alert_name" = String, Path, description = "Alert name"),
-      ),
-    request_body(content = Alert, description = "Alert data", content_type = "application/json"),    
-    responses(
-        (status = 200, description = "Success", content_type = "application/json", body = HttpResponse),
-        (status = 400, description = "Error",   content_type = "application/json", body = HttpResponse),
-    )
-)]
-#[put("/{org_id}/{stream_name}/alerts/{alert_name}")]
-pub async fn update_alert(
-    path: web::Path<(String, String, String)>,
-    alert: web::Json<Alert>,
-) -> Result<HttpResponse, Error> {
-    let (org_id, stream_name, name) = path.into_inner();
-
-    // Hack for frequency: convert minutes to seconds
-    let mut alert = alert.into_inner();
-    alert.trigger_condition.frequency *= 60;
-    match alerts::save(&org_id, &stream_name, &name, alert, false).await {
-        Ok(_) => Ok(MetaHttpResponse::ok("Alert Updated")),
-        Err(e) => Ok(MetaHttpResponse::bad_request(e)),
-    }
-}
-
-/// ListStreamAlerts
-#[utoipa::path(
-    context_path = "/api",
-    tag = "Alerts",
-    operation_id = "ListStreamAlerts",
-    security(
-        ("Authorization"= [])
-    ),
-    params(
-        ("org_id" = String, Path, description = "Organization name"),
-        ("stream_name" = String, Path, description = "Stream name"),
-      ),
-    responses(
-        (status = 200, description = "Success", content_type = "application/json", body = HttpResponse),
-        (status = 400, description = "Error",   content_type = "application/json", body = HttpResponse),
-    )
-)]
-#[get("/{org_id}/{stream_name}/alerts")]
-async fn list_stream_alerts(
-    path: web::Path<(String, String)>,
-    req: HttpRequest,
-) -> Result<HttpResponse, Error> {
-    let (org_id, stream_name) = path.into_inner();
-    let query = web::Query::<HashMap<String, String>>::from_query(req.query_string()).unwrap();
-    let stream_type = match get_stream_type_from_request(&query) {
-        Ok(v) => v,
-        Err(e) => {
-            return Ok(MetaHttpResponse::bad_request(e));
-        }
-    };
-    match alerts::list(&org_id, stream_type, Some(stream_name.as_str()), None).await {
-        Ok(mut data) => {
-            // Hack for frequency: convert seconds to minutes
-            for alert in data.iter_mut() {
-                alert.trigger_condition.frequency /= 60;
-            }
-
-            let mut mapdata = HashMap::new();
-            mapdata.insert("list", data);
-            Ok(MetaHttpResponse::json(mapdata))
-        }
-        Err(e) => Ok(MetaHttpResponse::bad_request(e)),
-    }
-}
-
-/// ListAlerts
-#[utoipa::path(
-    context_path = "/api",
-    tag = "Alerts",
-    operation_id = "ListAlerts",
-    security(
-        ("Authorization"= [])
-    ),
-    params(
-        ("org_id" = String, Path, description = "Organization name"),
-      ),
-    responses(
-        (status = 200, description = "Success", content_type = "application/json", body = HttpResponse),
-    )
-)]
-#[get("/{org_id}/alerts")]
-async fn list_alerts(path: web::Path<String>, _req: HttpRequest) -> Result<HttpResponse, Error> {
-    let org_id = path.into_inner();
-
-    let mut _alert_list_from_rbac = None;
-    // Get List of allowed objects
-    #[cfg(feature = "enterprise")]
-    {
-        let user_id = _req.headers().get("user_id").unwrap();
-        match crate::handler::http::auth::validator::list_objects_for_user(
-            &org_id,
-            user_id.to_str().unwrap(),
-            "GET",
-            "alert",
-        )
-        .await
-        {
-            Ok(stream_list) => {
-                _alert_list_from_rbac = stream_list;
-            }
-            Err(e) => {
-                return Ok(crate::common::meta::http::HttpResponse::forbidden(
-                    e.to_string(),
-                ));
-            }
-        }
-        // Get List of allowed objects ends
-    }
-
-    match alerts::list(&org_id, None, None, _alert_list_from_rbac).await {
-        Ok(mut data) => {
-            // Hack for frequency: convert seconds to minutes
-            for alert in data.iter_mut() {
-                alert.trigger_condition.frequency /= 60;
-            }
-
-            let mut mapdata = HashMap::new();
-            mapdata.insert("list", data);
-            Ok(MetaHttpResponse::json(mapdata))
-        }
-        Err(e) => Ok(MetaHttpResponse::bad_request(e)),
-    }
-}
-
-/// GetAlertByName
-#[utoipa::path(
-    context_path = "/api",
-    tag = "Alerts",
-    operation_id = "GetAlert",
-    security(
-        ("Authorization"= [])
-    ),
-    params(
-        ("org_id" = String, Path, description = "Organization name"),
-        ("stream_name" = String, Path, description = "Stream name"),
-        ("alert_name" = String, Path, description = "Alert name"),
-      ),
-    responses(
-        (status = 200, description = "Success",  content_type = "application/json", body = Alert),
-        (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse),
-    )
-)]
-#[get("/{org_id}/{stream_name}/alerts/{alert_name}")]
-async fn get_alert(
-    path: web::Path<(String, String, String)>,
-    req: HttpRequest,
-) -> Result<HttpResponse, Error> {
-    let (org_id, stream_name, name) = path.into_inner();
-    let query = web::Query::<HashMap<String, String>>::from_query(req.query_string()).unwrap();
-    let stream_type = match get_stream_type_from_request(&query) {
-        Ok(v) => v.unwrap_or_default(),
-        Err(e) => {
-            return Ok(MetaHttpResponse::bad_request(e));
-        }
-    };
-    match alerts::get(&org_id, stream_type, &stream_name, &name).await {
-        Ok(mut data) => {
-            // Hack for frequency: convert seconds to minutes
-            if let Some(ref mut data) = data {
-                data.trigger_condition.frequency /= 60;
-            }
-            Ok(MetaHttpResponse::json(data))
-        }
-        Err(e) => Ok(MetaHttpResponse::not_found(e)),
-    }
-}
-
-/// DeleteAlert
-#[utoipa::path(
-    context_path = "/api",
-    tag = "Alerts",
-    operation_id = "DeleteAlert",
-    security(
-        ("Authorization"= [])
-    ),
-    params(
-        ("org_id" = String, Path, description = "Organization name"),
-        ("stream_name" = String, Path, description = "Stream name"),
-        ("alert_name" = String, Path, description = "Alert name"),
-    ),
-    responses(
-        (status = 200, description = "Success",  content_type = "application/json", body = HttpResponse),
-        (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse),
-        (status = 500, description = "Failure",  content_type = "application/json", body = HttpResponse),
-    )
-)]
-#[delete("/{org_id}/{stream_name}/alerts/{alert_name}")]
-async fn delete_alert(
-    path: web::Path<(String, String, String)>,
-    req: HttpRequest,
-) -> Result<HttpResponse, Error> {
-    let (org_id, stream_name, name) = path.into_inner();
-    let query = web::Query::<HashMap<String, String>>::from_query(req.query_string()).unwrap();
-    let stream_type = match get_stream_type_from_request(&query) {
-        Ok(v) => v.unwrap_or_default(),
-        Err(e) => {
-            return Ok(MetaHttpResponse::bad_request(e));
-        }
-    };
-    match alerts::delete(&org_id, stream_type, &stream_name, &name).await {
-        Ok(_) => Ok(MetaHttpResponse::ok("Alert deleted")),
-        Err(e) => match e {
-            (http::StatusCode::NOT_FOUND, e) => Ok(MetaHttpResponse::not_found(e)),
-            (_, e) => Ok(MetaHttpResponse::internal_error(e)),
-        },
-    }
-}
-
-/// EnableAlert
-#[utoipa::path(
-    context_path = "/api",
-    tag = "Alerts",
-    operation_id = "EnableAlert",
-    security(
-        ("Authorization"= [])
-    ),
-    params(
-        ("org_id" = String, Path, description = "Organization name"),
-        ("stream_name" = String, Path, description = "Stream name"),
-        ("alert_name" = String, Path, description = "Alert name"),
-        ("value" = bool, Query, description = "Enable or disable alert"),
-    ),
-    responses(
-        (status = 200, description = "Success",  content_type = "application/json", body = HttpResponse),
-        (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse),
-        (status = 500, description = "Failure",  content_type = "application/json", body = HttpResponse),
-    )
-)]
-#[put("/{org_id}/{stream_name}/alerts/{alert_name}/enable")]
-async fn enable_alert(
-    path: web::Path<(String, String, String)>,
-    req: HttpRequest,
-) -> Result<HttpResponse, Error> {
-    let (org_id, stream_name, name) = path.into_inner();
-    let query = web::Query::<HashMap<String, String>>::from_query(req.query_string()).unwrap();
-    let stream_type = match get_stream_type_from_request(&query) {
-        Ok(v) => v.unwrap_or_default(),
-        Err(e) => {
-            return Ok(MetaHttpResponse::bad_request(e));
-        }
-    };
-    let enable = match query.get("value") {
-        Some(v) => v.parse::<bool>().unwrap_or_default(),
-        None => false,
-    };
-    let mut resp = HashMap::new();
-    resp.insert("enabled".to_string(), enable);
-    match alerts::enable(&org_id, stream_type, &stream_name, &name, enable).await {
-        Ok(_) => Ok(MetaHttpResponse::json(resp)),
-        Err(e) => match e {
-            (http::StatusCode::NOT_FOUND, e) => Ok(MetaHttpResponse::not_found(e)),
-            (_, e) => Ok(MetaHttpResponse::internal_error(e)),
-        },
-    }
-}
-
-/// TriggerAlert
-#[utoipa::path(
-    context_path = "/api",
-    tag = "Alerts",
-    operation_id = "TriggerAlert",
-    security(
-        ("Authorization"= [])
-    ),
-    params(
-        ("org_id" = String, Path, description = "Organization name"),
-        ("stream_name" = String, Path, description = "Stream name"),
-        ("alert_name" = String, Path, description = "Alert name"),
-    ),
-    responses(
-        (status = 200, description = "Success",  content_type = "application/json", body = HttpResponse),
-        (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse),
-        (status = 500, description = "Failure",  content_type = "application/json", body = HttpResponse),
-    )
-)]
-#[put("/{org_id}/{stream_name}/alerts/{alert_name}/trigger")]
-async fn trigger_alert(
-    path: web::Path<(String, String, String)>,
-    req: HttpRequest,
-) -> Result<HttpResponse, Error> {
-    let (org_id, stream_name, name) = path.into_inner();
-    let query = web::Query::<HashMap<String, String>>::from_query(req.query_string()).unwrap();
-    let stream_type = match get_stream_type_from_request(&query) {
-        Ok(v) => v.unwrap_or_default(),
-        Err(e) => {
-            return Ok(MetaHttpResponse::bad_request(e));
-        }
-    };
-    match alerts::trigger(&org_id, stream_type, &stream_name, &name).await {
-        Ok(_) => Ok(MetaHttpResponse::ok("Alert triggered")),
-        Err(e) => match e {
-            (http::StatusCode::NOT_FOUND, e) => Ok(MetaHttpResponse::not_found(e)),
-            (_, e) => Ok(MetaHttpResponse::internal_error(e)),
-        },
-    }
-}

src/handler/http/request/enrichment_table/mod.rs

@@ -22,7 +22,7 @@ use hashbrown::HashMap;
 
 use crate::{
     common::meta::http::HttpResponse as MetaHttpResponse,
-    service::enrichment_table::save_enrichment_data,
+    service::enrichment_table::{extract_multipart, save_enrichment_data},
 };
 
 /// CreateEnrichmentTable
@@ -81,7 +81,8 @@ pub async fn save_enrichment_table(
                     Some(append_data) => append_data.parse::<bool>().unwrap_or(false),
                     None => false,
                 };
-                save_enrichment_data(&org_id, &table_name, payload, append_data).await
+                let json_record = extract_multipart(payload).await?;
+                save_enrichment_data(&org_id, &table_name, json_record, append_data).await
             } else {
                 Ok(MetaHttpResponse::bad_request(
                     "Bad Request, content-type must be multipart/form-data",

src/handler/http/request/pipelines.rs

@@ -49,11 +49,11 @@ pub async fn save_pipeline(
     req: HttpRequest,
 ) -> Result<HttpResponse, Error> {
     let (org_id, stream_name) = path.into_inner();
+    let query = web::Query::<HashMap<String, String>>::from_query(req.query_string()).unwrap();
     let mut pipeline = pipeline.into_inner();
     pipeline.name = pipeline.name.trim().to_string();
     pipeline.stream_name = stream_name;
-    let query = web::Query::<HashMap<String, String>>::from_query(req.query_string()).unwrap();
-    let stream_type = match get_stream_type_from_request(&query) {
+    pipeline.stream_type = match get_stream_type_from_request(&query) {
         Ok(v) => v.unwrap_or_default(),
         Err(e) => {
             return Ok(
@@ -80,7 +80,6 @@ pub async fn save_pipeline(
             routing.insert(formatted_key, value);
         }
     }
-    pipeline.stream_type = stream_type;
     crate::service::pipelines::save_pipeline(org_id, pipeline).await
 }
 
@@ -162,7 +161,8 @@ async fn delete_pipeline(
             return Ok(crate::common::meta::http::HttpResponse::bad_request(e));
         }
     };
-    crate::service::pipelines::delete_pipeline(&org_id, stream_type, &stream_name, &name).await
+    let pipeline_source = meta::stream::StreamParams::new(&org_id, &stream_name, stream_type);
+    crate::service::pipelines::delete_pipeline(&name, pipeline_source).await
 }
 
 /// UpdatePipeline
@@ -200,8 +200,8 @@ pub async fn update_pipeline(
     let name = name.trim();
     let mut pipeline = pipeline.into_inner();
     pipeline.name = name.to_string();
-    pipeline.stream_name = stream_name;
     pipeline.stream_type = stream_type;
+    pipeline.stream_name = stream_name;
 
     if let Some(ref mut routing) = &mut pipeline.routing {
         let keys_to_update: Vec<_> = routing.keys().cloned().collect();
@@ -219,5 +219,5 @@ pub async fn update_pipeline(
             routing.insert(formatted_key, value);
         }
     }
-    crate::service::pipelines::update_pipeline(&org_id, name, pipeline).await
+    crate::service::pipelines::update_pipeline(&org_id, pipeline).await
 }

src/handler/http/router/mod.rs

@@ -420,14 +420,14 @@ pub fn get_service_routes(cfg: &mut web::ServiceConfig) {
             .service(dashboards::reports::delete_report)
             .service(dashboards::reports::enable_report)
             .service(dashboards::reports::trigger_report)
-            .service(alerts::save_alert)
-            .service(alerts::update_alert)
-            .service(alerts::get_alert)
-            .service(alerts::list_alerts)
-            .service(alerts::list_stream_alerts)
-            .service(alerts::delete_alert)
-            .service(alerts::enable_alert)
-            .service(alerts::trigger_alert)
+            .service(alerts::alert::save_alert)
+            .service(alerts::alert::update_alert)
+            .service(alerts::alert::get_alert)
+            .service(alerts::alert::list_alerts)
+            .service(alerts::alert::list_stream_alerts)
+            .service(alerts::alert::delete_alert)
+            .service(alerts::alert::enable_alert)
+            .service(alerts::alert::trigger_alert)
             .service(alerts::templates::save_template)
             .service(alerts::templates::update_template)
             .service(alerts::templates::get_template)

src/handler/http/router/openapi.rs

@@ -86,14 +86,14 @@ use crate::{common::meta, handler::http::request};
         request::dashboards::folders::get_folder,
         request::dashboards::folders::update_folder,
         request::dashboards::move_dashboard,
-        request::alerts::save_alert,
-        request::alerts::update_alert,
-        request::alerts::list_stream_alerts,
-        request::alerts::list_alerts,
-        request::alerts::get_alert,
-        request::alerts::delete_alert,
-        request::alerts::enable_alert,
-        request::alerts::trigger_alert,
+        request::alerts::alert::save_alert,
+        request::alerts::alert::update_alert,
+        request::alerts::alert::list_stream_alerts,
+        request::alerts::alert::list_alerts,
+        request::alerts::alert::get_alert,
+        request::alerts::alert::delete_alert,
+        request::alerts::alert::enable_alert,
+        request::alerts::alert::trigger_alert,
         request::alerts::templates::list_templates,
         request::alerts::templates::get_template,
         request::alerts::templates::save_template,
@@ -168,14 +168,14 @@ use crate::{common::meta, handler::http::request};
             meta::saved_view::DeleteViewResponse,
             meta::saved_view::CreateViewResponse,
             meta::saved_view::UpdateViewRequest,
-            meta::alerts::Alert,
+            meta::alerts::alert::Alert,
             meta::alerts::Condition,
             meta::alerts::Operator,
             meta::alerts::Aggregation,
             meta::alerts::AggFunction,
             meta::alerts::QueryType,
             meta::alerts::TriggerCondition,
-            meta::alerts::AlertFrequencyType,
+            meta::alerts::FrequencyType,
             meta::alerts::QueryCondition,
             meta::alerts::destinations::Destination,
             meta::alerts::destinations::DestinationWithTemplate,

src/infra/src/scheduler/mod.rs

@@ -81,6 +81,7 @@ pub enum TriggerModule {
     Report,
     #[default]
     Alert,
+    DerivedStream,
 }
 
 impl std::fmt::Display for TriggerModule {
@@ -88,6 +89,7 @@ impl std::fmt::Display for TriggerModule {
         match self {
             TriggerModule::Alert => write!(f, "alert"),
             TriggerModule::Report => write!(f, "report"),
+            TriggerModule::DerivedStream => write!(f, "derived_stream"),
         }
     }
 }

src/job/alert_manager.rs

@@ -70,7 +70,7 @@ async fn run_schedule_jobs() -> Result<(), anyhow::Error> {
     interval.tick().await; // trigger the first run
     loop {
         interval.tick().await;
-        if let Err(e) = service::alerts::alert_manager::run().await {
+        if let Err(e) = service::alerts::scheduler::run().await {
             log::error!("[ALERT MANAGER] run schedule jobs error: {}", e);
         }
     }

src/job/mod.rs

@@ -109,7 +109,7 @@ pub async fn init() -> Result<(), anyhow::Error> {
     tokio::task::spawn(async move { db::alerts::templates::watch().await });
     tokio::task::spawn(async move { db::alerts::destinations::watch().await });
     tokio::task::spawn(async move { db::alerts::realtime_triggers::watch().await });
-    tokio::task::spawn(async move { db::alerts::watch().await });
+    tokio::task::spawn(async move { db::alerts::alert::watch().await });
     tokio::task::spawn(async move { db::dashboards::reports::watch().await });
     tokio::task::spawn(async move { db::organization::watch().await });
     #[cfg(feature = "enterprise")]
@@ -147,7 +147,9 @@ pub async fn init() -> Result<(), anyhow::Error> {
     db::alerts::realtime_triggers::cache()
         .await
         .expect("alerts realtime triggers cache failed");
-    db::alerts::cache().await.expect("alerts cache failed");
+    db::alerts::alert::cache()
+        .await
+        .expect("alerts cache failed");
     db::dashboards::reports::cache()
         .await
         .expect("reports cache failed");

src/main.rs

@@ -32,8 +32,9 @@ use openobserve::{
             request::{
                 event::Eventer,
                 file_list::Filelister,
+                ingest::Ingester,
                 logs::LogsServer,
-                metrics::{ingester::Ingester, querier::Querier},
+                metrics::{ingester::MetricsIngester, querier::MetricsQuerier},
                 query_cache::QueryCacheServerImpl,
                 traces::TraceServer,
                 usage::UsageServerImpl,
@@ -53,8 +54,9 @@ use opentelemetry_proto::tonic::collector::{
 };
 use opentelemetry_sdk::{propagation::TraceContextPropagator, trace as sdktrace, Resource};
 use proto::cluster_rpc::{
-    event_server::EventServer, filelist_server::FilelistServer, metrics_server::MetricsServer,
-    query_cache_server::QueryCacheServer, search_server::SearchServer, usage_server::UsageServer,
+    event_server::EventServer, filelist_server::FilelistServer, ingest_server::IngestServer,
+    metrics_server::MetricsServer, query_cache_server::QueryCacheServer,
+    search_server::SearchServer, usage_server::UsageServer,
 };
 #[cfg(feature = "profiling")]
 use pyroscope::PyroscopeAgent;
@@ -380,10 +382,10 @@ async fn init_common_grpc_server(
     let filelist_svc = FilelistServer::new(Filelister)
         .send_compressed(CompressionEncoding::Gzip)
         .accept_compressed(CompressionEncoding::Gzip);
-    let metrics_svc = MetricsServer::new(Querier)
+    let metrics_svc = MetricsServer::new(MetricsQuerier)
         .send_compressed(CompressionEncoding::Gzip)
         .accept_compressed(CompressionEncoding::Gzip);
-    let metrics_ingest_svc = MetricsServiceServer::new(Ingester)
+    let metrics_ingest_svc = MetricsServiceServer::new(MetricsIngester)
         .send_compressed(CompressionEncoding::Gzip)
         .accept_compressed(CompressionEncoding::Gzip);
     let usage_svc = UsageServer::new(UsageServerImpl)
@@ -399,6 +401,9 @@ async fn init_common_grpc_server(
     let query_cache_svc = QueryCacheServer::new(QueryCacheServerImpl)
         .send_compressed(CompressionEncoding::Gzip)
         .accept_compressed(CompressionEncoding::Gzip);
+    let ingest_svc = IngestServer::new(Ingester)
+        .send_compressed(CompressionEncoding::Gzip)
+        .accept_compressed(CompressionEncoding::Gzip);
 
     log::info!("starting gRPC server at {}", gaddr);
     init_tx.send(()).ok();
@@ -413,6 +418,7 @@ async fn init_common_grpc_server(
         .add_service(usage_svc)
         .add_service(logs_svc)
         .add_service(query_cache_svc)
+        .add_service(ingest_svc)
         .serve_with_shutdown(gaddr, async {
             shutdown_rx.await.ok();
             log::info!("gRPC server starts shutting down");

src/proto/build.rs

@@ -58,6 +58,7 @@ fn main() -> Result<()> {
                 "proto/cluster/filelist.proto",
                 "proto/cluster/metrics.proto",
                 "proto/cluster/search.proto",
+                "proto/cluster/ingest.proto",
                 "proto/cluster/usage.proto",
                 "proto/cluster/querycache.proto",
                 "proto/cluster/plan.proto",

src/proto/proto/cluster/common.proto

@@ -23,12 +23,14 @@ message FileMeta {
 }
 
 enum StreamType {
-    LOGS     = 0;
-    METRICS  = 1;
-    TRACES   = 2;
-    METADATA = 3;
-    FILELIST = 4;
-    INDEX    = 5;
+    LOGS             = 0;
+    METRICS          = 1;
+    TRACES           = 2;
+    METADATA         = 3;
+    EnrichmentTables = 4;
+    FILELIST         = 5;
+    INDEX            = 6;
+
 }
 
 // Search request type

src/proto/proto/cluster/ingest.proto

@@ -0,0 +1,39 @@
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "org.openobserve.cluster";
+option java_outer_classname = "ingestProto";
+
+package cluster;
+
+import "cluster/common.proto";
+
+service Ingest {
+    rpc Ingest (IngestionRequest) returns (IngestionResponse) {}
+}
+
+message IngestionData {
+    bytes data = 1;
+}
+
+message IngestionRequest {
+    string                         org_id = 1;
+    string                    stream_name = 2;
+    IngestionData                    data = 3;
+    StreamType                stream_type = 4;
+    optional IngestionType ingestion_type = 5;
+}
+
+enum IngestionType {
+    JSON      = 0;
+    MULTI     = 1;
+    GCP       = 2;
+    KINESISFH = 3;
+    RUM       = 4;
+    USAGE     = 5;
+}
+
+message IngestionResponse {
+    int32 status_code = 1;
+    string    message = 2;    
+}

src/proto/src/generated/cluster.rs

@@ -96,8 +96,9 @@ pub enum StreamType {
     Metrics = 1,
     Traces = 2,
     Metadata = 3,
-    Filelist = 4,
-    Index = 5,
+    EnrichmentTables = 4,
+    Filelist = 5,
+    Index = 6,
 }
 impl StreamType {
     /// String value of the enum field names used in the ProtoBuf definition.
@@ -110,6 +111,7 @@ impl StreamType {
             StreamType::Metrics => "METRICS",
             StreamType::Traces => "TRACES",
             StreamType::Metadata => "METADATA",
+            StreamType::EnrichmentTables => "EnrichmentTables",
             StreamType::Filelist => "FILELIST",
             StreamType::Index => "INDEX",
         }
@@ -121,6 +123,7 @@ impl StreamType {
             "METRICS" => Some(Self::Metrics),
             "TRACES" => Some(Self::Traces),
             "METADATA" => Some(Self::Metadata),
+            "EnrichmentTables" => Some(Self::EnrichmentTables),
             "FILELIST" => Some(Self::Filelist),
             "INDEX" => Some(Self::Index),
             _ => None,
@@ -1926,6 +1929,338 @@ pub mod search_server {
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
+pub struct IngestionData {
+    #[prost(bytes = "vec", tag = "1")]
+    pub data: ::prost::alloc::vec::Vec<u8>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct IngestionRequest {
+    #[prost(string, tag = "1")]
+    pub org_id: ::prost::alloc::string::String,
+    #[prost(string, tag = "2")]
+    pub stream_name: ::prost::alloc::string::String,
+    #[prost(message, optional, tag = "3")]
+    pub data: ::core::option::Option<IngestionData>,
+    #[prost(enumeration = "StreamType", tag = "4")]
+    pub stream_type: i32,
+    #[prost(enumeration = "IngestionType", optional, tag = "5")]
+    pub ingestion_type: ::core::option::Option<i32>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct IngestionResponse {
+    #[prost(int32, tag = "1")]
+    pub status_code: i32,
+    #[prost(string, tag = "2")]
+    pub message: ::prost::alloc::string::String,
+}
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum IngestionType {
+    Json = 0,
+    Multi = 1,
+    Gcp = 2,
+    Kinesisfh = 3,
+    Rum = 4,
+    Usage = 5,
+}
+impl IngestionType {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            IngestionType::Json => "JSON",
+            IngestionType::Multi => "MULTI",
+            IngestionType::Gcp => "GCP",
+            IngestionType::Kinesisfh => "KINESISFH",
+            IngestionType::Rum => "RUM",
+            IngestionType::Usage => "USAGE",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "JSON" => Some(Self::Json),
+            "MULTI" => Some(Self::Multi),
+            "GCP" => Some(Self::Gcp),
+            "KINESISFH" => Some(Self::Kinesisfh),
+            "RUM" => Some(Self::Rum),
+            "USAGE" => Some(Self::Usage),
+            _ => None,
+        }
+    }
+}
+/// Generated client implementations.
+pub mod ingest_client {
+    #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)]
+    use tonic::codegen::{http::Uri, *};
+    #[derive(Debug, Clone)]
+    pub struct IngestClient<T> {
+        inner: tonic::client::Grpc<T>,
+    }
+    impl IngestClient<tonic::transport::Channel> {
+        /// Attempt to create a new client by connecting to a given endpoint.
+        pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
+        where
+            D: TryInto<tonic::transport::Endpoint>,
+            D::Error: Into<StdError>,
+        {
+            let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
+            Ok(Self::new(conn))
+        }
+    }
+    impl<T> IngestClient<T>
+    where
+        T: tonic::client::GrpcService<tonic::body::BoxBody>,
+        T::Error: Into<StdError>,
+        T::ResponseBody: Body<Data = Bytes> + Send + 'static,
+        <T::ResponseBody as Body>::Error: Into<StdError> + Send,
+    {
+        pub fn new(inner: T) -> Self {
+            let inner = tonic::client::Grpc::new(inner);
+            Self { inner }
+        }
+        pub fn with_origin(inner: T, origin: Uri) -> Self {
+            let inner = tonic::client::Grpc::with_origin(inner, origin);
+            Self { inner }
+        }
+        pub fn with_interceptor<F>(
+            inner: T,
+            interceptor: F,
+        ) -> IngestClient<InterceptedService<T, F>>
+        where
+            F: tonic::service::Interceptor,
+            T::ResponseBody: Default,
+            T: tonic::codegen::Service<
+                    http::Request<tonic::body::BoxBody>,
+                    Response = http::Response<
+                        <T as tonic::client::GrpcService<tonic::body::BoxBody>>::ResponseBody,
+                    >,
+                >,
+            <T as tonic::codegen::Service<http::Request<tonic::body::BoxBody>>>::Error:
+                Into<StdError> + Send + Sync,
+        {
+            IngestClient::new(InterceptedService::new(inner, interceptor))
+        }
+        /// Compress requests with the given encoding.
+        ///
+        /// This requires the server to support it otherwise it might respond with an
+        /// error.
+        #[must_use]
+        pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
+            self.inner = self.inner.send_compressed(encoding);
+            self
+        }
+        /// Enable decompressing responses.
+        #[must_use]
+        pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
+            self.inner = self.inner.accept_compressed(encoding);
+            self
+        }
+        /// Limits the maximum size of a decoded message.
+        ///
+        /// Default: `4MB`
+        #[must_use]
+        pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
+            self.inner = self.inner.max_decoding_message_size(limit);
+            self
+        }
+        /// Limits the maximum size of an encoded message.
+        ///
+        /// Default: `usize::MAX`
+        #[must_use]
+        pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
+            self.inner = self.inner.max_encoding_message_size(limit);
+            self
+        }
+        pub async fn ingest(
+            &mut self,
+            request: impl tonic::IntoRequest<super::IngestionRequest>,
+        ) -> std::result::Result<tonic::Response<super::IngestionResponse>, tonic::Status> {
+            self.inner.ready().await.map_err(|e| {
+                tonic::Status::new(
+                    tonic::Code::Unknown,
+                    format!("Service was not ready: {}", e.into()),
+                )
+            })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/cluster.Ingest/Ingest");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("cluster.Ingest", "Ingest"));
+            self.inner.unary(req, path, codec).await
+        }
+    }
+}
+/// Generated server implementations.
+pub mod ingest_server {
+    #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)]
+    use tonic::codegen::*;
+    /// Generated trait containing gRPC methods that should be implemented for use with
+    /// IngestServer.
+    #[async_trait]
+    pub trait Ingest: Send + Sync + 'static {
+        async fn ingest(
+            &self,
+            request: tonic::Request<super::IngestionRequest>,
+        ) -> std::result::Result<tonic::Response<super::IngestionResponse>, tonic::Status>;
+    }
+    #[derive(Debug)]
+    pub struct IngestServer<T: Ingest> {
+        inner: _Inner<T>,
+        accept_compression_encodings: EnabledCompressionEncodings,
+        send_compression_encodings: EnabledCompressionEncodings,
+        max_decoding_message_size: Option<usize>,
+        max_encoding_message_size: Option<usize>,
+    }
+    struct _Inner<T>(Arc<T>);
+    impl<T: Ingest> IngestServer<T> {
+        pub fn new(inner: T) -> Self {
+            Self::from_arc(Arc::new(inner))
+        }
+        pub fn from_arc(inner: Arc<T>) -> Self {
+            let inner = _Inner(inner);
+            Self {
+                inner,
+                accept_compression_encodings: Default::default(),
+                send_compression_encodings: Default::default(),
+                max_decoding_message_size: None,
+                max_encoding_message_size: None,
+            }
+        }
+        pub fn with_interceptor<F>(inner: T, interceptor: F) -> InterceptedService<Self, F>
+        where
+            F: tonic::service::Interceptor,
+        {
+            InterceptedService::new(Self::new(inner), interceptor)
+        }
+        /// Enable decompressing requests with the given encoding.
+        #[must_use]
+        pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
+            self.accept_compression_encodings.enable(encoding);
+            self
+        }
+        /// Compress responses with the given encoding, if the client supports it.
+        #[must_use]
+        pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
+            self.send_compression_encodings.enable(encoding);
+            self
+        }
+        /// Limits the maximum size of a decoded message.
+        ///
+        /// Default: `4MB`
+        #[must_use]
+        pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
+            self.max_decoding_message_size = Some(limit);
+            self
+        }
+        /// Limits the maximum size of an encoded message.
+        ///
+        /// Default: `usize::MAX`
+        #[must_use]
+        pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
+            self.max_encoding_message_size = Some(limit);
+            self
+        }
+    }
+    impl<T, B> tonic::codegen::Service<http::Request<B>> for IngestServer<T>
+    where
+        T: Ingest,
+        B: Body + Send + 'static,
+        B::Error: Into<StdError> + Send + 'static,
+    {
+        type Response = http::Response<tonic::body::BoxBody>;
+        type Error = std::convert::Infallible;
+        type Future = BoxFuture<Self::Response, Self::Error>;
+        fn poll_ready(
+            &mut self,
+            _cx: &mut Context<'_>,
+        ) -> Poll<std::result::Result<(), Self::Error>> {
+            Poll::Ready(Ok(()))
+        }
+        fn call(&mut self, req: http::Request<B>) -> Self::Future {
+            let inner = self.inner.clone();
+            match req.uri().path() {
+                "/cluster.Ingest/Ingest" => {
+                    #[allow(non_camel_case_types)]
+                    struct IngestSvc<T: Ingest>(pub Arc<T>);
+                    impl<T: Ingest> tonic::server::UnaryService<super::IngestionRequest> for IngestSvc<T> {
+                        type Response = super::IngestionResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::IngestionRequest>,
+                        ) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as Ingest>::ingest(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = IngestSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            )
+                            .apply_max_message_size_config(
+                                max_decoding_message_size,
+                                max_encoding_message_size,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                _ => Box::pin(async move {
+                    Ok(http::Response::builder()
+                        .status(200)
+                        .header("grpc-status", "12")
+                        .header("content-type", "application/grpc")
+                        .body(empty_body())
+                        .unwrap())
+                }),
+            }
+        }
+    }
+    impl<T: Ingest> Clone for IngestServer<T> {
+        fn clone(&self) -> Self {
+            let inner = self.inner.clone();
+            Self {
+                inner,
+                accept_compression_encodings: self.accept_compression_encodings,
+                send_compression_encodings: self.send_compression_encodings,
+                max_decoding_message_size: self.max_decoding_message_size,
+                max_encoding_message_size: self.max_encoding_message_size,
+            }
+        }
+    }
+    impl<T: Ingest> Clone for _Inner<T> {
+        fn clone(&self) -> Self {
+            Self(Arc::clone(&self.0))
+        }
+    }
+    impl<T: std::fmt::Debug> std::fmt::Debug for _Inner<T> {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "{:?}", self.0)
+        }
+    }
+    impl<T: Ingest> tonic::server::NamedService for IngestServer<T> {
+        const NAME: &'static str = "cluster.Ingest";
+    }
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
 pub struct UsageData {
     #[prost(bytes = "vec", tag = "1")]
     pub data: ::prost::alloc::vec::Vec<u8>,

src/proto/src/lib.rs

@@ -25,3 +25,11 @@ impl From<Vec<serde_json::Value>> for cluster_rpc::UsageData {
         }
     }
 }
+
+impl From<Vec<serde_json::Value>> for cluster_rpc::IngestionData {
+    fn from(usages: Vec<serde_json::Value>) -> Self {
+        Self {
+            data: serde_json::to_vec(&usages).unwrap(),
+        }
+    }
+}

src/service/alerts/alert.rs

@@ -0,0 +1,910 @@
+// Copyright 2024 Zinc Labs Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+    collections::{HashMap, HashSet},
+    str::FromStr,
+};
+
+use actix_web::http;
+use chrono::{Duration, Local, TimeZone, Timelike, Utc};
+use config::{
+    get_config,
+    meta::stream::StreamType,
+    utils::{
+        base64,
+        json::{Map, Value},
+    },
+    SMTP_CLIENT,
+};
+use cron::Schedule;
+use lettre::{message::SinglePart, AsyncTransport, Message};
+
+use crate::{
+    common::{
+        meta::{
+            alerts::{
+                alert::Alert,
+                destinations::{DestinationType, DestinationWithTemplate, HTTPType},
+                FrequencyType, Operator, QueryType,
+            },
+            authz::Authz,
+            stream::StreamParams,
+        },
+        utils::auth::{remove_ownership, set_ownership},
+    },
+    service::{
+        alerts::{build_sql, destinations},
+        db,
+    },
+};
+
+pub async fn save(
+    org_id: &str,
+    stream_name: &str,
+    name: &str,
+    mut alert: Alert,
+    create: bool,
+) -> Result<(), anyhow::Error> {
+    if !name.is_empty() {
+        alert.name = name.to_string();
+    }
+    alert.name = alert.name.trim().to_string();
+    alert.org_id = org_id.to_string();
+    let stream_type = alert.stream_type;
+    alert.stream_name = stream_name.to_string();
+    alert.row_template = alert.row_template.trim().to_string();
+
+    match db::alerts::alert::get(org_id, stream_type, stream_name, &alert.name).await {
+        Ok(Some(_)) => {
+            if create {
+                return Err(anyhow::anyhow!("Alert already exists"));
+            }
+        }
+        Ok(None) => {
+            if !create {
+                return Err(anyhow::anyhow!("Alert not found"));
+            }
+        }
+        Err(e) => {
+            return Err(e);
+        }
+    }
+
+    if alert.trigger_condition.frequency_type == FrequencyType::Cron {
+        let cron_exp = alert.trigger_condition.cron.clone();
+        if cron_exp.starts_with("* ") {
+            let (_, rest) = cron_exp.split_once(" ").unwrap();
+            let now = Utc::now().second().to_string();
+            alert.trigger_condition.cron = format!("{now} {rest}");
+            log::debug!(
+                "New cron expression for alert {}: {}",
+                alert.name,
+                alert.trigger_condition.cron
+            );
+        }
+        // Check the cron expression
+        Schedule::from_str(&alert.trigger_condition.cron)?;
+    } else if alert.trigger_condition.frequency == 0 {
+        // default frequency is 60 seconds
+        alert.trigger_condition.frequency =
+            std::cmp::max(10, get_config().limit.alert_schedule_interval);
+    }
+
+    if alert.name.is_empty() || alert.stream_name.is_empty() {
+        return Err(anyhow::anyhow!("Alert name is required"));
+    }
+    if alert.name.contains('/') {
+        return Err(anyhow::anyhow!("Alert name cannot contain '/'"));
+    }
+
+    if let Some(vrl) = alert.query_condition.vrl_function.as_ref() {
+        match base64::decode_url(vrl) {
+            Ok(vrl) => {
+                if !vrl.ends_with('.') {
+                    let vrl = base64::encode_url(&format!("{vrl} \n ."));
+                    alert.query_condition.vrl_function = Some(vrl);
+                }
+            }
+            Err(e) => {
+                return Err(anyhow::anyhow!(
+                    "Error decoding vrl function for alert: {e}"
+                ));
+            }
+        }
+    }
+
+    // before saving alert check alert destination
+    if alert.destinations.is_empty() {
+        return Err(anyhow::anyhow!("Alert destinations is required"));
+    }
+    for dest in alert.destinations.iter() {
+        if db::alerts::destinations::get(org_id, dest).await.is_err() {
+            return Err(anyhow::anyhow!("Alert destination {dest} not found"));
+        };
+    }
+
+    // before saving alert check alert context attributes
+    if alert.context_attributes.is_some() {
+        let attrs = alert.context_attributes.as_ref().unwrap();
+        let mut new_attrs = hashbrown::HashMap::with_capacity(attrs.len());
+        for key in attrs.keys() {
+            let new_key = key.trim().to_string();
+            if !new_key.is_empty() {
+                new_attrs.insert(new_key, attrs.get(key).unwrap().to_string());
+            }
+        }
+        alert.context_attributes = Some(new_attrs);
+    }
+
+    // before saving alert check column type to decide numeric condition
+    let schema = infra::schema::get(org_id, stream_name, stream_type).await?;
+    if stream_name.is_empty() || schema.fields().is_empty() {
+        return Err(anyhow::anyhow!("Stream {stream_name} not found"));
+    }
+
+    if alert.is_real_time && alert.query_condition.query_type != QueryType::Custom {
+        return Err(anyhow::anyhow!(
+            "Realtime alert should use Custom query type"
+        ));
+    }
+
+    match alert.query_condition.query_type {
+        QueryType::Custom => {
+            if alert.query_condition.aggregation.is_some() {
+                // if it has result we should fire the alert when enable aggregation
+                alert.trigger_condition.operator = Operator::GreaterThanEquals;
+                alert.trigger_condition.threshold = 1;
+            }
+        }
+        QueryType::SQL => {
+            if alert.query_condition.sql.is_none()
+                || alert.query_condition.sql.as_ref().unwrap().is_empty()
+            {
+                return Err(anyhow::anyhow!("Alert with SQL mode should have a query"));
+            }
+        }
+        QueryType::PromQL => {
+            if alert.query_condition.promql.is_none()
+                || alert.query_condition.promql.as_ref().unwrap().is_empty()
+                || alert.query_condition.promql_condition.is_none()
+            {
+                return Err(anyhow::anyhow!(
+                    "Alert with PromQL mode should have a query"
+                ));
+            }
+        }
+    }
+
+    // test the alert
+    if let Err(e) = &alert.evaluate(None).await {
+        return Err(anyhow::anyhow!("Alert test failed: {}", e));
+    }
+
+    // save the alert
+    match db::alerts::alert::set(org_id, stream_type, stream_name, &alert, create).await {
+        Ok(_) => {
+            if name.is_empty() {
+                set_ownership(org_id, "alerts", Authz::new(&alert.name)).await;
+            }
+            Ok(())
+        }
+        Err(e) => Err(e),
+    }
+}
+
+pub async fn get(
+    org_id: &str,
+    stream_type: StreamType,
+    stream_name: &str,
+    name: &str,
+) -> Result<Option<Alert>, anyhow::Error> {
+    db::alerts::alert::get(org_id, stream_type, stream_name, name).await
+}
+
+pub async fn list(
+    org_id: &str,
+    stream_type: Option<StreamType>,
+    stream_name: Option<&str>,
+    permitted: Option<Vec<String>>,
+) -> Result<Vec<Alert>, anyhow::Error> {
+    match db::alerts::alert::list(org_id, stream_type, stream_name).await {
+        Ok(alerts) => {
+            let mut result = Vec::new();
+            for alert in alerts {
+                if permitted.is_none()
+                    || permitted
+                        .as_ref()
+                        .unwrap()
+                        .contains(&format!("alert:{}", alert.name))
+                    || permitted
+                        .as_ref()
+                        .unwrap()
+                        .contains(&format!("alert:_all_{}", org_id))
+                {
+                    result.push(alert);
+                }
+            }
+            Ok(result)
+        }
+        Err(e) => Err(e),
+    }
+}
+
+pub async fn delete(
+    org_id: &str,
+    stream_type: StreamType,
+    stream_name: &str,
+    name: &str,
+) -> Result<(), (http::StatusCode, anyhow::Error)> {
+    if db::alerts::alert::get(org_id, stream_type, stream_name, name)
+        .await
+        .is_err()
+    {
+        return Err((
+            http::StatusCode::NOT_FOUND,
+            anyhow::anyhow!("Alert not found"),
+        ));
+    }
+    match db::alerts::alert::delete(org_id, stream_type, stream_name, name).await {
+        Ok(_) => {
+            remove_ownership(org_id, "alerts", Authz::new(name)).await;
+            Ok(())
+        }
+        Err(e) => Err((http::StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+pub async fn enable(
+    org_id: &str,
+    stream_type: StreamType,
+    stream_name: &str,
+    name: &str,
+    value: bool,
+) -> Result<(), (http::StatusCode, anyhow::Error)> {
+    let mut alert = match db::alerts::alert::get(org_id, stream_type, stream_name, name).await {
+        Ok(Some(alert)) => alert,
+        _ => {
+            return Err((
+                http::StatusCode::NOT_FOUND,
+                anyhow::anyhow!("Alert not found"),
+            ));
+        }
+    };
+    alert.enabled = value;
+    db::alerts::alert::set(org_id, stream_type, stream_name, &alert, false)
+        .await
+        .map_err(|e| (http::StatusCode::INTERNAL_SERVER_ERROR, e))
+}
+
+pub async fn trigger(
+    org_id: &str,
+    stream_type: StreamType,
+    stream_name: &str,
+    name: &str,
+) -> Result<(bool, String), (http::StatusCode, anyhow::Error)> {
+    let alert = match db::alerts::alert::get(org_id, stream_type, stream_name, name).await {
+        Ok(Some(alert)) => alert,
+        _ => {
+            return Err((
+                http::StatusCode::NOT_FOUND,
+                anyhow::anyhow!("Alert not found"),
+            ));
+        }
+    };
+    alert
+        .send_notification(&[])
+        .await
+        .map_err(|e| (http::StatusCode::INTERNAL_SERVER_ERROR, e))
+}
+
+impl Alert {
+    /// Returns the evaluated row data and the end time of the search timerange,
+    /// for realtime this is 0
+    pub async fn evaluate(
+        &self,
+        row: Option<&Map<String, Value>>,
+    ) -> Result<(Option<Vec<Map<String, Value>>>, i64), anyhow::Error> {
+        if self.is_real_time {
+            self.query_condition.evaluate_realtime(row).await
+        } else {
+            self.query_condition
+                .evaluate_scheduled(
+                    &self.get_stream_params(),
+                    &self.trigger_condition,
+                    &self.query_condition,
+                )
+                .await
+        }
+    }
+
+    /// Returns a tuple containing a boolean - if all the send notification jobs succeeded
+    /// and the error message if any
+    pub async fn send_notification(
+        &self,
+        rows: &[Map<String, Value>],
+    ) -> Result<(bool, String), anyhow::Error> {
+        let mut message = "".to_string();
+        let mut no_of_error = 0;
+        for dest in self.destinations.iter() {
+            let dest = destinations::get_with_template(&self.org_id, dest).await?;
+            if let Err(e) = send_notification(self, &dest, rows).await {
+                log::error!(
+                    "Error sending notification for {}/{}/{}/{} for destination {} err: {}",
+                    self.org_id,
+                    self.stream_type,
+                    self.stream_name,
+                    self.name,
+                    dest.name,
+                    e
+                );
+                no_of_error += 1;
+                message = format!(
+                    "{message} Error sending notification for destination {} err: {e};",
+                    dest.name
+                );
+            }
+        }
+        if no_of_error == self.destinations.len() {
+            Err(anyhow::anyhow!(message))
+        } else if no_of_error != 0 {
+            // Some send notification job failed
+            Ok((false, message))
+        } else {
+            Ok((true, "".to_owned()))
+        }
+    }
+
+    pub fn get_stream_params(&self) -> StreamParams {
+        StreamParams {
+            org_id: self.org_id.clone().into(),
+            stream_name: self.stream_name.clone().into(),
+            stream_type: self.stream_type,
+        }
+    }
+}
+
+pub async fn send_notification(
+    alert: &Alert,
+    dest: &DestinationWithTemplate,
+    rows: &[Map<String, Value>],
+) -> Result<(), anyhow::Error> {
+    let rows_tpl_val = if alert.row_template.is_empty() {
+        vec!["".to_string()]
+    } else {
+        process_row_template(&alert.row_template, alert, rows)
+    };
+    let msg: String = process_dest_template(&dest.template.body, alert, rows, &rows_tpl_val).await;
+
+    match dest.destination_type {
+        DestinationType::Http => send_http_notification(dest, msg.clone()).await,
+        DestinationType::Email => send_email_notification(&alert.name, dest, msg).await,
+    }
+}
+
+pub async fn send_http_notification(
+    dest: &DestinationWithTemplate,
+    msg: String,
+) -> Result<(), anyhow::Error> {
+    let client = if dest.skip_tls_verify {
+        reqwest::Client::builder()
+            .danger_accept_invalid_certs(true)
+            .build()?
+    } else {
+        reqwest::Client::new()
+    };
+    let url = url::Url::parse(&dest.url)?;
+    let mut req = match dest.method {
+        HTTPType::POST => client.post(url),
+        HTTPType::PUT => client.put(url),
+        HTTPType::GET => client.get(url),
+    };
+
+    // Add additional headers if any from destination description
+    let mut has_context_type = false;
+    if let Some(headers) = &dest.headers {
+        for (key, value) in headers.iter() {
+            if !key.is_empty() && !value.is_empty() {
+                if key.to_lowercase().trim() == "content-type" {
+                    has_context_type = true;
+                }
+                req = req.header(key, value);
+            }
+        }
+    };
+    // set default content type
+    if !has_context_type {
+        req = req.header("Content-type", "application/json");
+    }
+
+    let resp = req.body(msg.clone()).send().await?;
+    if !resp.status().is_success() {
+        log::error!("Alert body: {}", msg);
+        return Err(anyhow::anyhow!(
+            "sent error status: {}, err: {:?}",
+            resp.status(),
+            resp.bytes().await
+        ));
+    }
+
+    Ok(())
+}
+
+pub async fn send_email_notification(
+    alert_name: &str,
+    dest: &DestinationWithTemplate,
+    msg: String,
+) -> Result<(), anyhow::Error> {
+    let cfg = get_config();
+    if !cfg.smtp.smtp_enabled {
+        return Err(anyhow::anyhow!("SMTP configuration not enabled"));
+    }
+
+    let mut recepients = vec![];
+    for recepient in &dest.emails {
+        recepients.push(recepient);
+    }
+
+    let mut email = Message::builder()
+        .from(cfg.smtp.smtp_from_email.parse()?)
+        .subject(format!("Openobserve Alert - {}", alert_name));
+
+    for recepient in recepients {
+        email = email.to(recepient.parse()?);
+    }
+
+    if !cfg.smtp.smtp_reply_to.is_empty() {
+        email = email.reply_to(cfg.smtp.smtp_reply_to.parse()?);
+    }
+
+    let email = email.singlepart(SinglePart::html(msg)).unwrap();
+
+    // Send the email
+    match SMTP_CLIENT.as_ref().unwrap().send(email).await {
+        Ok(_) => Ok(()),
+        Err(e) => Err(anyhow::anyhow!("Error sending email: {e}")),
+    }
+}
+
+fn process_row_template(tpl: &String, alert: &Alert, rows: &[Map<String, Value>]) -> Vec<String> {
+    let alert_type = if alert.is_real_time {
+        "realtime"
+    } else {
+        "scheduled"
+    };
+    let alert_count = rows.len();
+    let mut rows_tpl = Vec::with_capacity(rows.len());
+    for row in rows.iter() {
+        let mut resp = tpl.to_string();
+        let mut alert_start_time = 0;
+        let mut alert_end_time = 0;
+        for (key, value) in row.iter() {
+            let value = if value.is_string() {
+                value.as_str().unwrap_or_default().to_string()
+            } else if value.is_f64() {
+                format!("{:.2}", value.as_f64().unwrap_or_default())
+            } else {
+                value.to_string()
+            };
+            process_variable_replace(&mut resp, key, &VarValue::Str(&value));
+
+            // calculate start and end time
+            if key == &get_config().common.column_timestamp {
+                let val = value.parse::<i64>().unwrap_or_default();
+                if alert_start_time == 0 || val < alert_start_time {
+                    alert_start_time = val;
+                }
+                if alert_end_time == 0 || val > alert_end_time {
+                    alert_end_time = val;
+                }
+            }
+            if key == "zo_sql_min_time" {
+                let val = value.parse::<i64>().unwrap_or_default();
+                if alert_start_time == 0 || val < alert_start_time {
+                    alert_start_time = val;
+                }
+            }
+            if key == "zo_sql_max_time" {
+                let val = value.parse::<i64>().unwrap_or_default();
+                if alert_end_time == 0 || val > alert_end_time {
+                    alert_end_time = val;
+                }
+            }
+        }
+        let alert_start_time_str = if alert_start_time > 0 {
+            Local
+                .timestamp_nanos(alert_start_time * 1000)
+                .format("%Y-%m-%dT%H:%M:%S")
+                .to_string()
+        } else {
+            String::from("N/A")
+        };
+        let alert_end_time_str = if alert_end_time > 0 {
+            Local
+                .timestamp_nanos(alert_end_time * 1000)
+                .format("%Y-%m-%dT%H:%M:%S")
+                .to_string()
+        } else {
+            String::from("N/A")
+        };
+
+        resp = resp
+            .replace("{org_name}", &alert.org_id)
+            .replace("{stream_type}", &alert.stream_type.to_string())
+            .replace("{stream_name}", &alert.stream_name)
+            .replace("{alert_name}", &alert.name)
+            .replace("{alert_type}", alert_type)
+            .replace(
+                "{alert_period}",
+                &alert.trigger_condition.period.to_string(),
+            )
+            .replace(
+                "{alert_operator}",
+                &alert.trigger_condition.operator.to_string(),
+            )
+            .replace(
+                "{alert_threshold}",
+                &alert.trigger_condition.threshold.to_string(),
+            )
+            .replace("{alert_count}", &alert_count.to_string())
+            .replace("{alert_start_time}", &alert_start_time_str)
+            .replace("{alert_end_time}", &alert_end_time_str);
+
+        if let Some(contidion) = &alert.query_condition.promql_condition {
+            resp = resp
+                .replace("{alert_promql_operator}", &contidion.operator.to_string())
+                .replace("{alert_promql_value}", &contidion.value.to_string());
+        }
+
+        if let Some(attrs) = &alert.context_attributes {
+            for (key, value) in attrs.iter() {
+                process_variable_replace(&mut resp, key, &VarValue::Str(value));
+            }
+        }
+
+        rows_tpl.push(resp);
+    }
+
+    rows_tpl
+}
+
+async fn process_dest_template(
+    tpl: &str,
+    alert: &Alert,
+    rows: &[Map<String, Value>],
+    rows_tpl_val: &[String],
+) -> String {
+    let cfg = get_config();
+    // format values
+    let alert_count = rows.len();
+    let mut vars = HashMap::with_capacity(rows.len());
+    for row in rows.iter() {
+        for (key, value) in row.iter() {
+            let value = if value.is_string() {
+                value.as_str().unwrap_or_default().to_string()
+            } else if value.is_f64() {
+                format!("{:.2}", value.as_f64().unwrap_or_default())
+            } else {
+                value.to_string()
+            };
+            let entry = vars.entry(key.to_string()).or_insert_with(HashSet::new);
+            entry.insert(value);
+        }
+    }
+
+    // calculate start and end time
+    let (alert_start_time, alert_end_time) =
+        get_alert_start_end_time(&vars, alert.trigger_condition.period);
+
+    let alert_start_time_str = if alert_start_time > 0 {
+        Local
+            .timestamp_nanos(alert_start_time * 1000)
+            .format("%Y-%m-%dT%H:%M:%S")
+            .to_string()
+    } else {
+        String::from("N/A")
+    };
+    let alert_end_time_str = if alert_end_time > 0 {
+        Local
+            .timestamp_nanos(alert_end_time * 1000)
+            .format("%Y-%m-%dT%H:%M:%S")
+            .to_string()
+    } else {
+        String::from("N/A")
+    };
+
+    let alert_type = if alert.is_real_time {
+        "realtime"
+    } else {
+        "scheduled"
+    };
+
+    let mut alert_query = String::new();
+    let alert_url = if alert.query_condition.query_type == QueryType::PromQL {
+        if let Some(promql) = &alert.query_condition.promql {
+            let condition = alert.query_condition.promql_condition.as_ref().unwrap();
+            alert_query = format!(
+                "({}) {} {}",
+                promql,
+                match condition.operator {
+                    Operator::EqualTo => "==".to_string(),
+                    _ => condition.operator.to_string(),
+                },
+                to_float(&condition.value)
+            );
+        }
+        // http://localhost:5080/web/metrics?stream=zo_http_response_time_bucket&from=1705248000000000&to=1705334340000000&query=em9faHR0cF9yZXNwb25zZV90aW1lX2J1Y2tldHt9&org_identifier=default
+        format!(
+            "{}{}/web/metrics?stream_type={}&stream={}&stream_value={}&from={}&to={}&query={}&org_identifier={}",
+            cfg.common.web_url,
+            cfg.common.base_uri,
+            alert.stream_type,
+            alert.stream_name,
+            alert.stream_name,
+            alert_start_time,
+            alert_end_time,
+            base64::encode_url(&alert_query).replace('+', "%2B"),
+            alert.org_id,
+        )
+    } else {
+        match alert.query_condition.query_type {
+            QueryType::SQL => {
+                if let Some(sql) = &alert.query_condition.sql {
+                    alert_query = sql.clone();
+                }
+            }
+            QueryType::Custom => {
+                if let Some(conditions) = &alert.query_condition.conditions {
+                    if let Ok(v) = build_sql(
+                        &alert.get_stream_params(),
+                        &alert.query_condition,
+                        conditions,
+                    )
+                    .await
+                    {
+                        alert_query = v;
+                    }
+                }
+            }
+            _ => unreachable!(),
+        };
+        // http://localhost:5080/web/logs?stream_type=logs&stream=test&from=1708416534519324&to=1708416597898186&sql_mode=true&query=U0VMRUNUICogRlJPTSAidGVzdCIgd2hlcmUgbGV2ZWwgPSAnaW5mbyc=&org_identifier=default
+        format!(
+            "{}{}/web/logs?stream_type={}&stream={}&stream_value={}&from={}&to={}&sql_mode=true&query={}&org_identifier={}",
+            cfg.common.web_url,
+            cfg.common.base_uri,
+            alert.stream_type,
+            alert.stream_name,
+            alert.stream_name,
+            alert_start_time,
+            alert_end_time,
+            base64::encode_url(&alert_query),
+            alert.org_id,
+        )
+    };
+
+    let mut resp = tpl
+        .replace("{org_name}", &alert.org_id)
+        .replace("{stream_type}", &alert.stream_type.to_string())
+        .replace("{stream_name}", &alert.stream_name)
+        .replace("{alert_name}", &alert.name)
+        .replace("{alert_type}", alert_type)
+        .replace(
+            "{alert_period}",
+            &alert.trigger_condition.period.to_string(),
+        )
+        .replace(
+            "{alert_operator}",
+            &alert.trigger_condition.operator.to_string(),
+        )
+        .replace(
+            "{alert_threshold}",
+            &alert.trigger_condition.threshold.to_string(),
+        )
+        .replace("{alert_count}", &alert_count.to_string())
+        .replace("{alert_start_time}", &alert_start_time_str)
+        .replace("{alert_end_time}", &alert_end_time_str)
+        .replace("{alert_url}", &alert_url);
+
+    if let Some(contidion) = &alert.query_condition.promql_condition {
+        resp = resp
+            .replace("{alert_promql_operator}", &contidion.operator.to_string())
+            .replace("{alert_promql_value}", &contidion.value.to_string());
+    }
+
+    process_variable_replace(&mut resp, "rows", &VarValue::Vector(rows_tpl_val));
+    for (key, value) in vars.iter() {
+        if resp.contains(&format!("{{{key}}}")) {
+            let val = value.iter().cloned().collect::<Vec<_>>();
+            process_variable_replace(&mut resp, key, &VarValue::Str(&val.join(", ")));
+        }
+    }
+    if let Some(attrs) = &alert.context_attributes {
+        for (key, value) in attrs.iter() {
+            process_variable_replace(&mut resp, key, &VarValue::Str(value));
+        }
+    }
+
+    resp
+}
+
+fn process_variable_replace(tpl: &mut String, var_name: &str, var_val: &VarValue) {
+    let pattern = "{".to_owned() + var_name + "}";
+    if tpl.contains(&pattern) {
+        *tpl = tpl.replace(&pattern, &var_val.to_string_with_length(0));
+        return;
+    }
+    let pattern = "{".to_owned() + var_name + ":";
+    if let Some(start) = tpl.find(&pattern) {
+        // find } start from position v
+        let p = start + pattern.len();
+        if let Some(end) = tpl[p..].find('}') {
+            let len = tpl[p..p + end].parse::<usize>().unwrap_or_default();
+            if len > 0 {
+                *tpl = tpl.replace(
+                    &tpl[start..p + end + 1],
+                    &var_val.to_string_with_length(len),
+                );
+            }
+        }
+    }
+}
+
+pub fn get_row_column_map(rows: &[Map<String, Value>]) -> HashMap<String, HashSet<String>> {
+    let mut vars = HashMap::with_capacity(rows.len());
+    for row in rows.iter() {
+        for (key, value) in row.iter() {
+            let value = if value.is_string() {
+                value.as_str().unwrap_or_default().to_string()
+            } else if value.is_f64() {
+                format!("{:.2}", value.as_f64().unwrap_or_default())
+            } else {
+                value.to_string()
+            };
+            let entry = vars.entry(key.to_string()).or_insert_with(HashSet::new);
+            entry.insert(value);
+        }
+    }
+    vars
+}
+
+pub fn get_alert_start_end_time(
+    vars: &HashMap<String, HashSet<String>>,
+    period: i64,
+) -> (i64, i64) {
+    let cfg = get_config();
+
+    // calculate start and end time
+    let mut alert_start_time = 0;
+    let mut alert_end_time = 0;
+    if let Some(values) = vars.get(&cfg.common.column_timestamp) {
+        for val in values {
+            let val = val.parse::<i64>().unwrap_or_default();
+            if alert_start_time == 0 || val < alert_start_time {
+                alert_start_time = val;
+            }
+            if alert_end_time == 0 || val > alert_end_time {
+                alert_end_time = val;
+            }
+        }
+    }
+    if let Some(values) = vars.get("zo_sql_min_time") {
+        for val in values {
+            let val = val.parse::<i64>().unwrap_or_default();
+            if alert_start_time == 0 || val < alert_start_time {
+                alert_start_time = val;
+            }
+        }
+    }
+    if let Some(values) = vars.get("zo_sql_max_time") {
+        for val in values {
+            let val = val.parse::<i64>().unwrap_or_default();
+            if alert_end_time == 0 || val > alert_end_time {
+                alert_end_time = val;
+            }
+        }
+    }
+
+    // Hack time range for alert url
+    alert_end_time = if alert_end_time == 0 {
+        Utc::now().timestamp_micros()
+    } else {
+        // the frontend will drop the second, so we add 1 minute to the end time
+        alert_end_time
+            + Duration::try_minutes(1)
+                .unwrap()
+                .num_microseconds()
+                .unwrap()
+    };
+    if alert_start_time == 0 {
+        alert_start_time = alert_end_time
+            - Duration::try_minutes(period)
+                .unwrap()
+                .num_microseconds()
+                .unwrap();
+    }
+    if alert_end_time - alert_start_time
+        < Duration::try_minutes(1)
+            .unwrap()
+            .num_microseconds()
+            .unwrap()
+    {
+        alert_start_time = alert_end_time
+            - Duration::try_minutes(period)
+                .unwrap()
+                .num_microseconds()
+                .unwrap();
+    }
+    (alert_start_time, alert_end_time)
+}
+
+fn format_variable_value(val: String) -> String {
+    val.replace('\n', "\\n")
+        .replace('\r', "\\r")
+        .replace('\"', "\\\"")
+}
+
+pub(super) fn to_float(val: &Value) -> f64 {
+    if val.is_number() {
+        val.as_f64().unwrap_or_default()
+    } else {
+        val.as_str().unwrap_or_default().parse().unwrap_or_default()
+    }
+}
+
+enum VarValue<'a> {
+    Str(&'a str),
+    Vector(&'a [String]),
+}
+
+impl<'a> VarValue<'a> {
+    fn len(&self) -> usize {
+        match self {
+            VarValue::Str(v) => v.chars().count(),
+            VarValue::Vector(v) => v.len(),
+        }
+    }
+
+    fn to_string_with_length(&self, n: usize) -> String {
+        let n = if n > 0 && n < self.len() {
+            n
+        } else {
+            self.len()
+        };
+        match self {
+            VarValue::Str(v) => format_variable_value(v.chars().take(n).collect()),
+            VarValue::Vector(v) => v[0..n].join("\\n"),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_alert_create() {
+        let org_id = "default";
+        let stream_name = "default";
+        let alert_name = "abc/alert";
+        let alert = Alert {
+            name: alert_name.to_string(),
+            ..Default::default()
+        };
+        let ret = save(org_id, stream_name, alert_name, alert, true).await;
+        // alert name should not contain /
+        assert!(ret.is_err());
+    }
+}

src/service/alerts/derived_streams.rs

@@ -0,0 +1,197 @@
+// Copyright 2024 Zinc Labs Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::str::FromStr;
+
+use chrono::{Duration, Utc};
+use config::{
+    get_config,
+    utils::json::{Map, Value},
+};
+use cron::Schedule;
+use hashbrown::HashMap;
+
+use crate::{
+    common::{
+        meta::{
+            // authz::Authz,
+            alerts::{derived_streams::DerivedStreamMeta, FrequencyType, QueryType},
+        },
+        // utils::auth::{remove_ownership, set_ownership},
+    },
+    service::db,
+};
+
+pub async fn save(
+    mut derived_stream: DerivedStreamMeta,
+    pipeline_name: &str,
+) -> Result<(), anyhow::Error> {
+    derived_stream.name = derived_stream.name.trim().to_string();
+    // 1. Start validate DerivedStream
+    if !derived_stream.is_valid() {
+        return Err(anyhow::anyhow!(
+            "DerivedStream Name, destination, and Trigger period required"
+        ));
+    }
+
+    // query_type for realtime
+    if derived_stream.is_real_time && derived_stream.query_condition.query_type != QueryType::Custom
+    {
+        return Err(anyhow::anyhow!(
+            "Realtime DerivedStream should use Custom query type"
+        ));
+    }
+
+    // other checks for query type
+    match derived_stream.query_condition.query_type {
+        QueryType::SQL => {
+            if derived_stream
+                .query_condition
+                .sql
+                .as_ref()
+                .map_or(false, |sql| sql.is_empty())
+            {
+                return Err(anyhow::anyhow!(
+                    "DerivedStreams with SQL mode should have a query"
+                ));
+            }
+        }
+        QueryType::PromQL => {
+            if derived_stream
+                .query_condition
+                .promql
+                .as_ref()
+                .map_or(false, |promql| promql.is_empty())
+                || derived_stream.query_condition.promql_condition.is_none()
+            {
+                return Err(anyhow::anyhow!(
+                    "DerivedStreams with SQL mode should have a query"
+                ));
+            }
+        }
+        _ => {}
+    }
+
+    // check source stream schema
+    let schema = infra::schema::get(
+        &derived_stream.source.org_id,
+        &derived_stream.source.stream_name,
+        derived_stream.source.stream_type,
+    )
+    .await?;
+    if schema.fields().is_empty() {
+        return Err(anyhow::anyhow!(
+            "Source Stream {}/{} schema is empty.",
+            derived_stream.source.org_id,
+            derived_stream.source.stream_name,
+        ));
+    }
+    // End input validation
+
+    // 2. update the frequency
+    if derived_stream.trigger_condition.frequency_type == FrequencyType::Cron {
+        // Check the cron expression
+        Schedule::from_str(&derived_stream.trigger_condition.cron)?;
+    } else if derived_stream.trigger_condition.frequency == 0 {
+        // default 3 mins, set min at 1 minutes
+        derived_stream.trigger_condition.frequency =
+            std::cmp::max(1, get_config().limit.derived_stream_schedule_interval / 60);
+    }
+
+    // 3. clean up DerivedStream context attributes
+    if let Some(attrs) = &derived_stream.context_attributes {
+        let mut new_attrs = HashMap::with_capacity(attrs.len());
+        for (key, val) in attrs.iter() {
+            new_attrs.insert(key.trim().to_string(), val.to_string());
+        }
+        derived_stream.context_attributes = Some(new_attrs);
+    }
+
+    // test derived_stream
+    if let Err(e) = &derived_stream.evaluate(None).await {
+        return Err(anyhow::anyhow!(
+            "DerivedStream not saved due to failed test run caused by {}",
+            e.to_string()
+        ));
+    };
+
+    // Save the trigger to db
+    let next_run_at = Utc::now().timestamp_micros()
+        + Duration::try_minutes(derived_stream.trigger_condition.frequency)
+            .unwrap()
+            .num_microseconds()
+            .unwrap();
+    let trigger = db::scheduler::Trigger {
+        org: derived_stream.source.org_id.to_string(),
+        module: db::scheduler::TriggerModule::DerivedStream,
+        module_key: derived_stream.get_scheduler_module_key(pipeline_name),
+        next_run_at,
+        is_realtime: derived_stream.is_real_time,
+        is_silenced: false,
+        ..Default::default()
+    };
+
+    match db::scheduler::get(&trigger.org, trigger.module.clone(), &trigger.module_key).await {
+        Ok(_) => db::scheduler::update_trigger(trigger)
+            .await
+            .map_err(|_| anyhow::anyhow!("Trigger already exists, but failed to update")),
+        Err(_) => db::scheduler::push(trigger)
+            .await
+            .map_err(|e| anyhow::anyhow!("Error save DerivedStream trigger: {}", e)),
+    }
+}
+
+pub async fn delete(
+    derived_stream: DerivedStreamMeta,
+    pipeline_name: &str,
+) -> Result<(), anyhow::Error> {
+    db::scheduler::delete(
+        &derived_stream.source.org_id,
+        db::scheduler::TriggerModule::DerivedStream,
+        &derived_stream.get_scheduler_module_key(pipeline_name),
+    )
+    .await
+    .map_err(|e| anyhow::anyhow!("Error deleting derived stream trigger: {e}"))
+}
+
+impl DerivedStreamMeta {
+    pub fn is_valid(&self) -> bool {
+        !self.name.is_empty()
+            && !self.is_real_time // TODO(taiming): support realtime DerivedStream
+            && self.source.is_valid()
+            && self.destination.is_valid()
+            && self.trigger_condition.period != 0
+    }
+
+    pub fn get_scheduler_module_key(&self, pipeline_name: &str) -> String {
+        format!(
+            "{}/{}/{}/{}",
+            self.source.stream_type, self.source.stream_name, pipeline_name, self.name
+        )
+    }
+
+    pub async fn evaluate(
+        &self,
+        row: Option<&Map<String, Value>>,
+    ) -> Result<(Option<Vec<Map<String, Value>>>, i64), anyhow::Error> {
+        if self.is_real_time {
+            self.query_condition.evaluate_realtime(row).await
+        } else {
+            self.query_condition
+                .evaluate_scheduled(&self.source, &self.trigger_condition, &self.query_condition)
+                .await
+        }
+    }
+}

src/service/alerts/scheduler.rs

@@ -22,15 +22,18 @@ use config::{
         stream::StreamType,
         usage::{TriggerData, TriggerDataStatus, TriggerDataType},
     },
+    utils::json,
 };
 use cron::Schedule;
 use futures::future::try_join_all;
+use proto::cluster_rpc;
 
 use crate::{
-    common::meta::{alerts::AlertFrequencyType, dashboards::reports::ReportFrequencyType},
+    common::meta::{alerts::FrequencyType, dashboards::reports::ReportFrequencyType},
     service::{
-        alerts::{get_alert_start_end_time, get_row_column_map},
+        alerts::alert::{get_alert_start_end_time, get_row_column_map},
         db,
+        ingestion::ingestion_service,
         usage::publish_triggers_usage,
     },
 };
@@ -55,13 +58,13 @@ pub async fn run() -> Result<(), anyhow::Error> {
     for trigger in triggers {
         let task = tokio::task::spawn(async move {
             if let Err(e) = handle_triggers(trigger).await {
-                log::error!("[ALERT_MANAGER] Error handling trigger: {}", e);
+                log::error!("[SCHEDULER] Error handling trigger: {}", e);
             }
         });
         tasks.push(task);
     }
     if let Err(e) = try_join_all(tasks).await {
-        log::error!("[ALERT_MANAGER] Error handling triggers: {}", e);
+        log::error!("[SCHEDULER] Error handling triggers: {}", e);
     }
     Ok(())
 }
@@ -70,6 +73,9 @@ pub async fn handle_triggers(trigger: db::scheduler::Trigger) -> Result<(), anyh
     match trigger.module {
         db::scheduler::TriggerModule::Report => handle_report_triggers(trigger).await,
         db::scheduler::TriggerModule::Alert => handle_alert_triggers(trigger).await,
+        db::scheduler::TriggerModule::DerivedStream => {
+            handle_derived_stream_triggers(trigger).await
+        }
     }
 }
 
@@ -105,7 +111,7 @@ async fn handle_alert_triggers(trigger: db::scheduler::Trigger) -> Result<(), an
         return Ok(());
     }
 
-    let alert = match super::get(org_id, stream_type, stream_name, alert_name).await? {
+    let alert = match super::alert::get(org_id, stream_type, stream_name, alert_name).await? {
         Some(alert) => alert,
         None => {
             return Err(anyhow::anyhow!(
@@ -145,7 +151,7 @@ async fn handle_alert_triggers(trigger: db::scheduler::Trigger) -> Result<(), an
         );
     }
     if ret.is_some() && alert.trigger_condition.silence > 0 {
-        if alert.trigger_condition.frequency_type == AlertFrequencyType::Cron {
+        if alert.trigger_condition.frequency_type == FrequencyType::Cron {
             let schedule = Schedule::from_str(&alert.trigger_condition.cron)?;
             let silence =
                 Utc::now() + Duration::try_minutes(alert.trigger_condition.silence).unwrap();
@@ -163,7 +169,7 @@ async fn handle_alert_triggers(trigger: db::scheduler::Trigger) -> Result<(), an
                 .unwrap();
         }
         new_trigger.is_silenced = true;
-    } else if alert.trigger_condition.frequency_type == AlertFrequencyType::Cron {
+    } else if alert.trigger_condition.frequency_type == FrequencyType::Cron {
         let schedule = Schedule::from_str(&alert.trigger_condition.cron)?;
         // tz_offset is in minutes
         let tz_offset = FixedOffset::east_opt(alert.tz_offset * 60).unwrap();
@@ -425,3 +431,207 @@ async fn handle_report_triggers(trigger: db::scheduler::Trigger) -> Result<(), a
 
     Ok(())
 }
+
+async fn handle_derived_stream_triggers(
+    trigger: db::scheduler::Trigger,
+) -> Result<(), anyhow::Error> {
+    log::debug!(
+        "Inside handle_derived_stream_triggers processing trigger: {}",
+        trigger.module_key
+    );
+
+    // module_key format: stream_type/stream_name/pipeline_name/derived_stream_name
+    let columns = trigger.module_key.split('/').collect::<Vec<_>>();
+    assert_eq!(columns.len(), 4);
+    let org_id = &trigger.org;
+    let stream_type: StreamType = columns[0].into();
+    let stream_name = columns[1];
+    let pipeline_name = columns[2];
+    let name = columns[3];
+
+    let is_real_time = trigger.is_realtime;
+    let is_silenced = trigger.is_silenced;
+    if is_real_time && is_silenced {
+        log::debug!(
+            "Realtime derived_stream needs to wake up, {}/{}",
+            org_id,
+            trigger.module_key
+        );
+        let new_trigger = db::scheduler::Trigger {
+            next_run_at: Utc::now().timestamp_micros(),
+            is_silenced: false,
+            status: db::scheduler::TriggerStatus::Waiting,
+            ..trigger.clone()
+        };
+        db::scheduler::update_trigger(new_trigger).await?;
+        return Ok(());
+    }
+
+    let Ok(pipeline) = db::pipelines::get(org_id, stream_type, stream_name, pipeline_name).await
+    else {
+        return Err(anyhow::anyhow!(
+            "Pipeline associated with trigger not found: {}/{}/{}/{}",
+            org_id,
+            stream_name,
+            stream_type,
+            pipeline_name
+        ));
+    };
+
+    let Some(derived_stream) = pipeline
+        .derived_streams
+        .and_then(|ds| ds.into_iter().find(|ds| ds.name == name))
+    else {
+        return Err(anyhow::anyhow!(
+            "DerivedStream associated with the trigger not found in pipeline: {}/{}/{}/{}",
+            org_id,
+            stream_name,
+            stream_type,
+            name,
+        ));
+    };
+
+    let mut new_trigger = db::scheduler::Trigger {
+        next_run_at: Utc::now().timestamp_micros(),
+        is_silenced: false,
+        status: db::scheduler::TriggerStatus::Waiting,
+        ..trigger.clone()
+    };
+
+    // evaluate trigger and configure trigger next run time
+    let (ret, _) = derived_stream.evaluate(None).await?;
+    if ret.is_some() {
+        log::info!(
+            "DerivedStream conditions satisfied, org: {}, module_key: {}",
+            new_trigger.org,
+            new_trigger.module_key
+        );
+    }
+    if ret.is_some() && derived_stream.trigger_condition.silence > 0 {
+        if derived_stream.trigger_condition.frequency_type == FrequencyType::Cron {
+            let schedule = Schedule::from_str(&derived_stream.trigger_condition.cron)?;
+            let silence = Utc::now()
+                + Duration::try_minutes(derived_stream.trigger_condition.silence).unwrap();
+            let silence = silence.with_timezone(
+                FixedOffset::east_opt(derived_stream.tz_offset * 60)
+                    .as_ref()
+                    .unwrap(),
+            );
+            // Check for the cron timestamp after the silence period
+            new_trigger.next_run_at = schedule.after(&silence).next().unwrap().timestamp_micros();
+        } else {
+            new_trigger.next_run_at +=
+                Duration::try_minutes(derived_stream.trigger_condition.silence)
+                    .unwrap()
+                    .num_microseconds()
+                    .unwrap();
+        }
+        new_trigger.is_silenced = true;
+    } else if derived_stream.trigger_condition.frequency_type == FrequencyType::Cron {
+        let schedule = Schedule::from_str(&derived_stream.trigger_condition.cron)?;
+        // tz_offset is in minutes
+        let tz_offset = FixedOffset::east_opt(derived_stream.tz_offset * 60).unwrap();
+        new_trigger.next_run_at = schedule
+            .upcoming(tz_offset)
+            .next()
+            .unwrap()
+            .timestamp_micros();
+    } else {
+        new_trigger.next_run_at +=
+            Duration::try_minutes(derived_stream.trigger_condition.frequency)
+                .unwrap()
+                .num_microseconds()
+                .unwrap();
+    }
+
+    let mut trigger_data_stream = TriggerData {
+        _timestamp: trigger.start_time.unwrap_or_default(),
+        org: trigger.org,
+        module: TriggerDataType::DerivedStream,
+        key: trigger.module_key.clone(),
+        next_run_at: new_trigger.next_run_at,
+        is_realtime: trigger.is_realtime,
+        is_silenced: trigger.is_silenced,
+        status: TriggerDataStatus::Completed,
+        start_time: trigger.start_time.unwrap_or_default(),
+        end_time: trigger.end_time.unwrap_or_default(),
+        retries: trigger.retries,
+        error: None,
+    };
+
+    // ingest evaluation result into destination
+    if let Some(data) = ret {
+        let local_val = data
+            .into_iter()
+            .map(json::Value::Object)
+            .collect::<Vec<_>>();
+        // Ingest result into destination stream
+        let (org_id, stream_name, stream_type): (String, String, i32) = {
+            (
+                derived_stream.destination.org_id.into(),
+                derived_stream.destination.stream_name.into(),
+                cluster_rpc::StreamType::from(derived_stream.destination.stream_type).into(),
+            )
+        };
+        let req = cluster_rpc::IngestionRequest {
+            org_id: org_id.clone(),
+            stream_name: stream_name.clone(),
+            stream_type,
+            data: Some(cluster_rpc::IngestionData::from(local_val)),
+            ingestion_type: Some(cluster_rpc::IngestionType::Json.into()), /* TODO(taiming): finalize IngestionType for derived_stream */
+        };
+        match ingestion_service::ingest(&org_id, req).await {
+            Ok(_) => {
+                log::info!(
+                    "DerivedStream result ingested to destination {org_id}/{stream_name}/{stream_type}",
+                );
+                db::scheduler::update_trigger(new_trigger).await?;
+            }
+            Err(e) => {
+                log::error!(
+                    "Error in ingesting DerivedStream result to destination {:?}, org: {}, module_key: {}",
+                    e,
+                    new_trigger.org,
+                    new_trigger.module_key
+                );
+                if trigger.retries + 1 >= get_config().limit.scheduler_max_retries {
+                    // It has been tried the maximum time, just update the
+                    // next_run_at to the next expected trigger time
+                    log::debug!(
+                        "This DerivedStream trigger: {}/{} has reached maximum retries",
+                        &new_trigger.org,
+                        &new_trigger.module_key
+                    );
+                    db::scheduler::update_trigger(new_trigger).await?;
+                } else {
+                    // Otherwise update its status only
+                    db::scheduler::update_status(
+                        &new_trigger.org,
+                        new_trigger.module,
+                        &new_trigger.module_key,
+                        db::scheduler::TriggerStatus::Waiting,
+                        trigger.retries + 1,
+                    )
+                    .await?;
+                }
+                trigger_data_stream.status = TriggerDataStatus::Failed;
+                trigger_data_stream.error =
+                    Some(format!("error sending notification for alert: {e}"));
+            }
+        }
+    } else {
+        log::debug!(
+            "DerivedStream conditions not satisfied, org: {}, module_key: {}",
+            &new_trigger.org,
+            &new_trigger.module_key
+        );
+        db::scheduler::update_trigger(new_trigger).await?;
+        trigger_data_stream.status = TriggerDataStatus::ConditionNotSatisfied;
+    }
+
+    // publish the triggers as stream
+    trigger_data_stream.end_time = Utc::now().timestamp_micros();
+    publish_triggers_usage(trigger_data_stream).await;
+
+    Ok(())
+}

src/service/db/alerts/alert.rs

@@ -0,0 +1,257 @@
+// Copyright 2024 Zinc Labs Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+
+use config::{meta::stream::StreamType, utils::json};
+
+use crate::{
+    common::{infra::config::STREAM_ALERTS, meta::alerts::alert::Alert},
+    service::db,
+};
+
+pub async fn get(
+    org_id: &str,
+    stream_type: StreamType,
+    stream_name: &str,
+    name: &str,
+) -> Result<Option<Alert>, anyhow::Error> {
+    let stream_key = format!("{org_id}/{stream_type}/{stream_name}");
+    let value: Option<Alert> = if let Some(v) = STREAM_ALERTS.read().await.get(&stream_key) {
+        v.iter().find(|x| x.name.eq(name)).cloned()
+    } else {
+        let key = format!("/alerts/{org_id}/{stream_type}/{stream_name}/{name}");
+        match db::get(&key).await {
+            Ok(val) => json::from_slice(&val)?,
+            Err(_) => None,
+        }
+    };
+    if value.is_none() { Ok(None) } else { Ok(value) }
+}
+
+pub async fn set(
+    org_id: &str,
+    stream_type: StreamType,
+    stream_name: &str,
+    alert: &Alert,
+    create: bool,
+) -> Result<(), anyhow::Error> {
+    let schedule_key = format!("{stream_type}/{stream_name}/{}", alert.name);
+    let key = format!("/alerts/{org_id}/{}", &schedule_key);
+    match db::put(
+        &key,
+        json::to_vec(alert).unwrap().into(),
+        db::NEED_WATCH,
+        None,
+    )
+    .await
+    {
+        Ok(_) => {
+            let trigger = db::scheduler::Trigger {
+                org: org_id.to_string(),
+                module_key: schedule_key,
+                next_run_at: chrono::Utc::now().timestamp_micros(),
+                is_realtime: alert.is_real_time,
+                is_silenced: false,
+                ..Default::default()
+            };
+            if create {
+                match db::scheduler::push(trigger).await {
+                    Ok(_) => Ok(()),
+                    Err(e) => {
+                        log::error!("Failed to save trigger: {}", e);
+                        Ok(())
+                    }
+                }
+            } else {
+                match db::scheduler::update_trigger(trigger).await {
+                    Ok(_) => Ok(()),
+                    Err(e) => {
+                        log::error!("Failed to update trigger: {}", e);
+                        Ok(())
+                    }
+                }
+            }
+        }
+        Err(e) => Err(anyhow::anyhow!("Error save alert: {}", e)),
+    }
+}
+
+pub async fn delete(
+    org_id: &str,
+    stream_type: StreamType,
+    stream_name: &str,
+    name: &str,
+) -> Result<(), anyhow::Error> {
+    let schedule_key = format!("{stream_type}/{stream_name}/{name}");
+    let key = format!("/alerts/{org_id}/{}", &schedule_key);
+    match db::delete(&key, false, db::NEED_WATCH, None).await {
+        Ok(_) => {
+            match db::scheduler::delete(org_id, db::scheduler::TriggerModule::Alert, &schedule_key)
+                .await
+            {
+                Ok(_) => Ok(()),
+                Err(e) => {
+                    log::error!("Failed to delete trigger: {}", e);
+                    Ok(())
+                }
+            }
+        }
+        Err(e) => Err(anyhow::anyhow!("Error deleting alert: {e}")),
+    }
+}
+
+pub async fn list(
+    org_id: &str,
+    stream_type: Option<StreamType>,
+    stream_name: Option<&str>,
+) -> Result<Vec<Alert>, anyhow::Error> {
+    let loc_stream_type = stream_type.unwrap_or_default();
+    let key = match stream_name {
+        Some(stream_name) => format!("/alerts/{org_id}/{loc_stream_type}/{stream_name}"),
+        None => format!("/alerts/{org_id}/"),
+    };
+    let ret = db::list_values(&key).await?;
+    let mut items: Vec<Alert> = Vec::with_capacity(ret.len());
+    for item_value in ret {
+        let json_val = json::from_slice(&item_value)?;
+        items.push(json_val)
+    }
+    items.sort_by(|a, b| a.name.cmp(&b.name));
+    Ok(items)
+}
+
+pub async fn watch() -> Result<(), anyhow::Error> {
+    let key = "/alerts/";
+    let cluster_coordinator = db::get_coordinator().await;
+    let mut events = cluster_coordinator.watch(key).await?;
+    let events = Arc::get_mut(&mut events).unwrap();
+    log::info!("Start watching alerts");
+    loop {
+        let ev = match events.recv().await {
+            Some(ev) => ev,
+            None => {
+                log::error!("watch_alerts: event channel closed");
+                break;
+            }
+        };
+        match ev {
+            db::Event::Put(ev) => {
+                let item_key = ev.key.strip_prefix(key).unwrap();
+                let stream_key = item_key[0..item_key.rfind('/').unwrap()].to_string();
+                let item_value: Alert = if config::get_config().common.meta_store_external {
+                    match db::get(&ev.key).await {
+                        Ok(val) => match json::from_slice(&val) {
+                            Ok(val) => val,
+                            Err(e) => {
+                                log::error!("Error getting value: {}", e);
+                                continue;
+                            }
+                        },
+                        Err(e) => {
+                            log::error!("Error getting value: {}", e);
+                            continue;
+                        }
+                    }
+                } else {
+                    json::from_slice(&ev.value.unwrap()).unwrap()
+                };
+                let mut cacher = STREAM_ALERTS.write().await;
+                let group = cacher.entry(stream_key.to_string()).or_default();
+                if group.contains(&item_value) {
+                    let idx = group.iter().position(|x| x.eq(&item_value)).unwrap();
+                    let _ = std::mem::replace(&mut group[idx], item_value);
+                } else {
+                    group.push(item_value);
+                }
+                drop(cacher);
+            }
+            db::Event::Delete(ev) => {
+                let item_key = ev.key.strip_prefix(key).unwrap();
+                let stream_key = item_key[0..item_key.rfind('/').unwrap()].to_string();
+                let alert_name = item_key[item_key.rfind('/').unwrap() + 1..].to_string();
+                let mut cacher = STREAM_ALERTS.write().await;
+                if stream_key.contains('/') {
+                    let group = match cacher.get_mut(&stream_key) {
+                        Some(v) => v,
+                        None => continue,
+                    };
+                    group.retain(|v| !v.name.eq(&alert_name));
+                } else {
+                    cacher.remove(&stream_key);
+                }
+                drop(cacher);
+            }
+            db::Event::Empty => {}
+        }
+    }
+    Ok(())
+}
+
+pub async fn cache() -> Result<(), anyhow::Error> {
+    let key = "/alerts/";
+    let ret = db::list(key).await?;
+    for (item_key, item_value) in ret {
+        let new_key = item_key.strip_prefix(key).unwrap();
+        let alert: Alert = match json::from_slice(&item_value) {
+            Ok(v) => v,
+            Err(_) => {
+                // HACK: for old version, write it back to up
+                let data: json::Value = json::from_slice(&item_value).unwrap();
+                let data = data.as_object().unwrap();
+                let destinations = if let Some(dest) = data.get("destination") {
+                    vec![dest.as_str().unwrap().to_string()]
+                } else {
+                    vec![]
+                };
+                let alert = Alert {
+                    name: data.get("name").unwrap().as_str().unwrap().to_string(),
+                    stream_type: data.get("stream_type").unwrap().as_str().unwrap().into(),
+                    stream_name: match data.get("stream") {
+                        Some(v) => v.as_str().unwrap().to_string(),
+                        None => data
+                            .get("stream_name")
+                            .unwrap()
+                            .as_str()
+                            .unwrap()
+                            .to_string(),
+                    },
+                    destinations,
+                    ..Default::default()
+                };
+                _ = db::put(
+                    &item_key,
+                    json::to_vec(&alert).unwrap().into(),
+                    db::NO_NEED_WATCH,
+                    None,
+                )
+                .await;
+                alert
+            }
+        };
+        let stream_key = &new_key[0..new_key.rfind('/').unwrap()];
+
+        let mut cacher = STREAM_ALERTS.write().await;
+        let group = cacher.entry(stream_key.to_string()).or_default();
+        group.push(alert);
+    }
+    log::info!("Alerts Cached");
+    Ok(())
+}
+
+pub async fn reset() -> Result<(), anyhow::Error> {
+    let key = "/alerts/";
+    Ok(db::delete(key, true, db::NO_NEED_WATCH, None).await?)
+}

src/service/db/alerts/mod.rs

@@ -13,249 +13,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::sync::Arc;
-
-use config::{meta::stream::StreamType, utils::json};
-
-use crate::{
-    common::{infra::config::STREAM_ALERTS, meta::alerts::Alert},
-    service::db,
-};
-
+pub mod alert;
 pub mod destinations;
 pub mod realtime_triggers;
 pub mod templates;
-
-pub async fn get(
-    org_id: &str,
-    stream_type: StreamType,
-    stream_name: &str,
-    name: &str,
-) -> Result<Option<Alert>, anyhow::Error> {
-    let stream_key = format!("{org_id}/{stream_type}/{stream_name}");
-    let value: Option<Alert> = if let Some(v) = STREAM_ALERTS.read().await.get(&stream_key) {
-        v.iter().find(|x| x.name.eq(name)).cloned()
-    } else {
-        let key = format!("/alerts/{org_id}/{stream_type}/{stream_name}/{name}");
-        match db::get(&key).await {
-            Ok(val) => json::from_slice(&val)?,
-            Err(_) => None,
-        }
-    };
-    if value.is_none() { Ok(None) } else { Ok(value) }
-}
-
-pub async fn set(
-    org_id: &str,
-    stream_type: StreamType,
-    stream_name: &str,
-    alert: &Alert,
-    create: bool,
-) -> Result<(), anyhow::Error> {
-    let schedule_key = format!("{stream_type}/{stream_name}/{}", alert.name);
-    let key = format!("/alerts/{org_id}/{}", &schedule_key);
-    match db::put(
-        &key,
-        json::to_vec(alert).unwrap().into(),
-        db::NEED_WATCH,
-        None,
-    )
-    .await
-    {
-        Ok(_) => {
-            let trigger = db::scheduler::Trigger {
-                org: org_id.to_string(),
-                module_key: schedule_key,
-                next_run_at: chrono::Utc::now().timestamp_micros(),
-                is_realtime: alert.is_real_time,
-                is_silenced: false,
-                ..Default::default()
-            };
-            if create {
-                match db::scheduler::push(trigger).await {
-                    Ok(_) => Ok(()),
-                    Err(e) => {
-                        log::error!("Failed to save trigger: {}", e);
-                        Ok(())
-                    }
-                }
-            } else {
-                match db::scheduler::update_trigger(trigger).await {
-                    Ok(_) => Ok(()),
-                    Err(e) => {
-                        log::error!("Failed to update trigger: {}", e);
-                        Ok(())
-                    }
-                }
-            }
-        }
-        Err(e) => Err(anyhow::anyhow!("Error save alert: {}", e)),
-    }
-}
-
-pub async fn delete(
-    org_id: &str,
-    stream_type: StreamType,
-    stream_name: &str,
-    name: &str,
-) -> Result<(), anyhow::Error> {
-    let schedule_key = format!("{stream_type}/{stream_name}/{name}");
-    let key = format!("/alerts/{org_id}/{}", &schedule_key);
-    match db::delete(&key, false, db::NEED_WATCH, None).await {
-        Ok(_) => {
-            match db::scheduler::delete(org_id, db::scheduler::TriggerModule::Alert, &schedule_key)
-                .await
-            {
-                Ok(_) => Ok(()),
-                Err(e) => {
-                    log::error!("Failed to delete trigger: {}", e);
-                    Ok(())
-                }
-            }
-        }
-        Err(e) => Err(anyhow::anyhow!("Error deleting alert: {e}")),
-    }
-}
-
-pub async fn list(
-    org_id: &str,
-    stream_type: Option<StreamType>,
-    stream_name: Option<&str>,
-) -> Result<Vec<Alert>, anyhow::Error> {
-    let loc_stream_type = stream_type.unwrap_or_default();
-    let key = match stream_name {
-        Some(stream_name) => format!("/alerts/{org_id}/{loc_stream_type}/{stream_name}"),
-        None => format!("/alerts/{org_id}/"),
-    };
-    let ret = db::list_values(&key).await?;
-    let mut items: Vec<Alert> = Vec::with_capacity(ret.len());
-    for item_value in ret {
-        let json_val = json::from_slice(&item_value)?;
-        items.push(json_val)
-    }
-    items.sort_by(|a, b| a.name.cmp(&b.name));
-    Ok(items)
-}
-
-pub async fn watch() -> Result<(), anyhow::Error> {
-    let key = "/alerts/";
-    let cluster_coordinator = db::get_coordinator().await;
-    let mut events = cluster_coordinator.watch(key).await?;
-    let events = Arc::get_mut(&mut events).unwrap();
-    log::info!("Start watching alerts");
-    loop {
-        let ev = match events.recv().await {
-            Some(ev) => ev,
-            None => {
-                log::error!("watch_alerts: event channel closed");
-                break;
-            }
-        };
-        match ev {
-            db::Event::Put(ev) => {
-                let item_key = ev.key.strip_prefix(key).unwrap();
-                let stream_key = item_key[0..item_key.rfind('/').unwrap()].to_string();
-                let item_value: Alert = if config::get_config().common.meta_store_external {
-                    match db::get(&ev.key).await {
-                        Ok(val) => match json::from_slice(&val) {
-                            Ok(val) => val,
-                            Err(e) => {
-                                log::error!("Error getting value: {}", e);
-                                continue;
-                            }
-                        },
-                        Err(e) => {
-                            log::error!("Error getting value: {}", e);
-                            continue;
-                        }
-                    }
-                } else {
-                    json::from_slice(&ev.value.unwrap()).unwrap()
-                };
-                let mut cacher = STREAM_ALERTS.write().await;
-                let group = cacher.entry(stream_key.to_string()).or_default();
-                if group.contains(&item_value) {
-                    let idx = group.iter().position(|x| x.eq(&item_value)).unwrap();
-                    let _ = std::mem::replace(&mut group[idx], item_value);
-                } else {
-                    group.push(item_value);
-                }
-                drop(cacher);
-            }
-            db::Event::Delete(ev) => {
-                let item_key = ev.key.strip_prefix(key).unwrap();
-                let stream_key = item_key[0..item_key.rfind('/').unwrap()].to_string();
-                let alert_name = item_key[item_key.rfind('/').unwrap() + 1..].to_string();
-                let mut cacher = STREAM_ALERTS.write().await;
-                if stream_key.contains('/') {
-                    let group = match cacher.get_mut(&stream_key) {
-                        Some(v) => v,
-                        None => continue,
-                    };
-                    group.retain(|v| !v.name.eq(&alert_name));
-                } else {
-                    cacher.remove(&stream_key);
-                }
-                drop(cacher);
-            }
-            db::Event::Empty => {}
-        }
-    }
-    Ok(())
-}
-
-pub async fn cache() -> Result<(), anyhow::Error> {
-    let key = "/alerts/";
-    let ret = db::list(key).await?;
-    for (item_key, item_value) in ret {
-        let new_key = item_key.strip_prefix(key).unwrap();
-        let alert: Alert = match json::from_slice(&item_value) {
-            Ok(v) => v,
-            Err(_) => {
-                // HACK: for old version, write it back to up
-                let data: json::Value = json::from_slice(&item_value).unwrap();
-                let data = data.as_object().unwrap();
-                let destinations = if let Some(dest) = data.get("destination") {
-                    vec![dest.as_str().unwrap().to_string()]
-                } else {
-                    vec![]
-                };
-                let alert = Alert {
-                    name: data.get("name").unwrap().as_str().unwrap().to_string(),
-                    stream_type: data.get("stream_type").unwrap().as_str().unwrap().into(),
-                    stream_name: match data.get("stream") {
-                        Some(v) => v.as_str().unwrap().to_string(),
-                        None => data
-                            .get("stream_name")
-                            .unwrap()
-                            .as_str()
-                            .unwrap()
-                            .to_string(),
-                    },
-                    destinations,
-                    ..Default::default()
-                };
-                _ = db::put(
-                    &item_key,
-                    json::to_vec(&alert).unwrap().into(),
-                    db::NO_NEED_WATCH,
-                    None,
-                )
-                .await;
-                alert
-            }
-        };
-        let stream_key = &new_key[0..new_key.rfind('/').unwrap()];
-
-        let mut cacher = STREAM_ALERTS.write().await;
-        let group = cacher.entry(stream_key.to_string()).or_default();
-        group.push(alert);
-    }
-    log::info!("Alerts Cached");
-    Ok(())
-}
-
-pub async fn reset() -> Result<(), anyhow::Error> {
-    let key = "/alerts/";
-    Ok(db::delete(key, true, db::NO_NEED_WATCH, None).await?)
-}

src/service/db/pipelines.rs

@@ -22,10 +22,10 @@ use crate::{
     service::db,
 };
 
-pub async fn set(org_id: &str, name: &str, pipeline: &PipeLine) -> Result<(), anyhow::Error> {
+pub async fn set(org_id: &str, pipeline: &PipeLine) -> Result<(), anyhow::Error> {
     let key = format!(
-        "/pipeline/{org_id}/{}/{}/{name}",
-        pipeline.stream_type, pipeline.stream_name
+        "/pipeline/{org_id}/{}/{}/{}",
+        pipeline.stream_type, pipeline.stream_name, pipeline.name
     );
     match db::put(
         &key,

src/service/enrichment_table/mod.rs

@@ -56,7 +56,7 @@ pub mod geoip;
 pub async fn save_enrichment_data(
     org_id: &str,
     table_name: &str,
-    mut payload: Multipart,
+    payload: Vec<json::Map<String, json::Value>>,
     append_data: bool,
 ) -> Result<HttpResponse, Error> {
     let start = std::time::Instant::now();
@@ -118,77 +118,44 @@ pub async fn save_enrichment_data(
             .parse::<i64>()
             .unwrap()
     };
-    while let Ok(Some(mut field)) = payload.try_next().await {
-        let content_disposition = field.content_disposition();
-        let filename = content_disposition.get_filename();
-        let mut data = bytes::Bytes::new();
 
-        if filename.is_some() {
-            while let Some(chunk) = field.next().await {
-                let chunked_data = chunk.unwrap();
-                // Reconstruct entire CSV data bytes here to prevent fragmentation of values.
-                data = Bytes::from([data.as_ref(), chunked_data.as_ref()].concat());
-            }
-            let mut rdr = csv::Reader::from_reader(data.as_ref());
-            let headers: csv::StringRecord = rdr
-                .headers()?
-                .iter()
-                .map(|x| {
-                    let mut x = x.trim().to_string();
-                    format_key(&mut x);
-                    x
-                })
-                .collect::<Vec<_>>()
-                .into();
+    for mut json_record in payload {
+        json_record.insert(
+            get_config().common.column_timestamp.clone(),
+            json::Value::Number(timestamp.into()),
+        );
 
-            for result in rdr.records() {
-                // The iterator yields Result<StringRecord, Error>, so we check the
-                // error here.
-                let record = result?;
-                // Transform the record to a JSON value
-                let mut json_record = json::Map::new();
-
-                for (header, field) in headers.iter().zip(record.iter()) {
-                    json_record.insert(header.into(), json::Value::String(field.into()));
-                }
-                json_record.insert(
-                    get_config().common.column_timestamp.clone(),
-                    json::Value::Number(timestamp.into()),
-                );
-
-                // check for schema evolution
-                if !schema_evolved
-                    && check_for_schema(
-                        org_id,
-                        stream_name,
-                        StreamType::EnrichmentTables,
-                        &mut stream_schema_map,
-                        vec![&json_record],
-                        timestamp,
-                    )
-                    .await
-                    .is_ok()
-                {
-                    schema_evolved = true;
-                }
-
-                if records.is_empty() {
-                    let schema = stream_schema_map.get(stream_name).unwrap();
-                    let schema_key = schema.hash_key();
-                    hour_key = super::ingestion::get_wal_time_key(
-                        timestamp,
-                        &vec![],
-                        PartitionTimeLevel::Unset,
-                        &json_record,
-                        Some(schema_key),
-                    );
-                }
-                let record = json::Value::Object(json_record);
-                let record_size = json::estimate_json_bytes(&record);
-                records.push(Arc::new(record));
-                records_size += record_size;
-            }
+        // check for schema evolution
+        if !schema_evolved
+            && check_for_schema(
+                org_id,
+                stream_name,
+                StreamType::EnrichmentTables,
+                &mut stream_schema_map,
+                vec![&json_record],
+                timestamp,
+            )
+            .await
+            .is_ok()
+        {
+            schema_evolved = true;
         }
+
+        if records.is_empty() {
+            let schema = stream_schema_map.get(stream_name).unwrap();
+            let schema_key = schema.hash_key();
+            hour_key = super::ingestion::get_wal_time_key(
+                timestamp,
+                &vec![],
+                PartitionTimeLevel::Unset,
+                &json_record,
+                Some(schema_key),
+            );
+        }
+        let record = json::Value::Object(json_record);
+        let record_size = json::estimate_json_bytes(&record);
+        records.push(Arc::new(record));
+        records_size += record_size;
     }
 
     if records.is_empty() {
@@ -280,3 +247,51 @@ async fn delete_enrichment_table(org_id: &str, stream_name: &str, stream_type: S
     stats::remove_stream_stats(org_id, stream_name, stream_type);
     log::info!("deleted enrichment table  {stream_name}");
 }
+
+pub async fn extract_multipart(
+    mut payload: Multipart,
+) -> Result<Vec<json::Map<String, json::Value>>, Error> {
+    let mut records = Vec::new();
+    while let Ok(Some(mut field)) = payload.try_next().await {
+        let content_disposition = field.content_disposition();
+        let filename = content_disposition.get_filename();
+        let mut data = bytes::Bytes::new();
+
+        if filename.is_some() {
+            while let Some(chunk) = field.next().await {
+                let chunked_data = chunk.unwrap();
+                // Reconstruct entire CSV data bytes here to prevent fragmentation of values.
+                data = Bytes::from([data.as_ref(), chunked_data.as_ref()].concat());
+            }
+            let mut rdr = csv::Reader::from_reader(data.as_ref());
+            let headers: csv::StringRecord = rdr
+                .headers()?
+                .iter()
+                .map(|x| {
+                    let mut x = x.trim().to_string();
+                    format_key(&mut x);
+                    x
+                })
+                .collect::<Vec<_>>()
+                .into();
+
+            for result in rdr.records() {
+                // The iterator yields Result<StringRecord, Error>, so we check the
+                // error here.
+                let record = result?;
+                // Transform the record to a JSON value
+                let mut json_record = json::Map::new();
+
+                for (header, field) in headers.iter().zip(record.iter()) {
+                    json_record.insert(header.into(), json::Value::String(field.into()));
+                }
+
+                if !json_record.is_empty() {
+                    records.push(json_record);
+                }
+            }
+        }
+    }
+
+    Ok(records)
+}

src/service/ingestion/ingestion_service.rs

@@ -0,0 +1,68 @@
+// Copyright 2024 Zinc Labs Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use anyhow::Error;
+use proto::cluster_rpc;
+use tonic::{
+    codec::CompressionEncoding,
+    metadata::{MetadataKey, MetadataValue},
+    Request,
+};
+
+use crate::{common::infra::cluster, router::grpc::ingest::get_ingester_channel};
+
+pub async fn ingest(
+    dest_org_id: &str,
+    req: cluster_rpc::IngestionRequest,
+) -> Result<cluster_rpc::IngestionResponse, Error> {
+    let cfg = config::get_config();
+    let org_header_key: MetadataKey<_> = cfg
+        .grpc
+        .org_header_key
+        .parse()
+        .map_err(|_| Error::msg("invalid org_header_key".to_string()))?;
+    let token: MetadataValue<_> = cluster::get_internal_grpc_token()
+        .parse()
+        .map_err(|_| Error::msg("invalid token".to_string()))?;
+    let channel = get_ingester_channel().await?;
+    let mut client = cluster_rpc::ingest_client::IngestClient::with_interceptor(
+        channel,
+        move |mut req: Request<()>| {
+            req.metadata_mut().insert("authorization", token.clone());
+            req.metadata_mut()
+                .insert(org_header_key.clone(), dest_org_id.parse().unwrap());
+            Ok(req)
+        },
+    );
+    client = client
+        .send_compressed(CompressionEncoding::Gzip)
+        .accept_compressed(CompressionEncoding::Gzip)
+        .max_decoding_message_size(cfg.grpc.max_message_size * 1024 * 1024)
+        .max_encoding_message_size(cfg.grpc.max_message_size * 1024 * 1024);
+    let res: cluster_rpc::IngestionResponse = match client.ingest(req).await {
+        Ok(res) => res.into_inner(),
+        Err(err) => {
+            log::error!(
+                "[InternalIngestion] export partial_success response: {:?}",
+                err
+            );
+            if err.code() == tonic::Code::Internal {
+                return Err(err.into());
+            }
+            return Err(Error::msg("Ingest node error"));
+        }
+    };
+    Ok(res)
+}

src/service/ingestion/mod.rs

@@ -30,6 +30,7 @@ use config::{
     utils::{flatten, json::*},
     SIZE_IN_MB,
 };
+use proto::cluster_rpc::IngestionType;
 use vector_enrichment::TableRegistry;
 use vrl::{
     compiler::{runtime::Runtime, CompilationResult, TargetValueRef},
@@ -43,8 +44,9 @@ use crate::{
             REALTIME_ALERT_TRIGGERS, STREAM_ALERTS, STREAM_FUNCTIONS, STREAM_PIPELINES,
         },
         meta::{
-            alerts::Alert,
+            alerts::alert::Alert,
             functions::{StreamTransform, VRLResultResolver, VRLRuntimeConfig},
+            ingestion::IngestionRequest,
             stream::{SchemaRecords, StreamParams},
         },
         utils::functions::get_vrl_compiler_config,
@@ -53,6 +55,7 @@ use crate::{
 };
 
 pub mod grpc;
+pub mod ingestion_service;
 
 pub type TriggerAlertData = Vec<(Alert, Vec<Map<String, Value>>)>;
 
@@ -575,6 +578,19 @@ pub async fn get_user_defined_schema(
     }
 }
 
+pub fn create_log_ingestion_req(
+    ingestion_type: i32,
+    data: &actix_web::web::Bytes,
+) -> Result<IngestionRequest> {
+    match IngestionType::try_from(ingestion_type) {
+        Ok(IngestionType::Json) => Ok(IngestionRequest::JSON(data)),
+        Ok(IngestionType::Multi) => Ok(IngestionRequest::Multi(data)),
+        Ok(IngestionType::Usage) => Ok(IngestionRequest::Usage(data)),
+        Ok(IngestionType::Rum) => Ok(IngestionRequest::RUM(data)),
+        _ => Err(anyhow::anyhow!("Not yet supported")),
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use infra::schema::{unwrap_stream_settings, STREAM_SETTINGS};

src/service/logs/mod.rs

@@ -43,7 +43,7 @@ use super::{
 };
 use crate::{
     common::meta::{
-        alerts::Alert,
+        alerts::alert::Alert,
         ingestion::IngestionStatus,
         stream::{SchemaRecords, StreamParams},
     },

src/service/metrics/otlp_grpc.rs

@@ -39,7 +39,7 @@ use prost::Message;
 
 use crate::{
     common::meta::{
-        alerts,
+        alerts::alert,
         http::HttpResponse as MetaHttpResponse,
         prom::*,
         stream::{SchemaRecords, StreamParams},
@@ -96,7 +96,7 @@ pub async fn handle_grpc_request(
     let mut metric_data_map: HashMap<String, HashMap<String, SchemaRecords>> = HashMap::new();
     let mut metric_schema_map: HashMap<String, SchemaCache> = HashMap::new();
     let mut schema_evolved: HashMap<String, bool> = HashMap::new();
-    let mut stream_alerts_map: HashMap<String, Vec<alerts::Alert>> = HashMap::new();
+    let mut stream_alerts_map: HashMap<String, Vec<alert::Alert>> = HashMap::new();
     let mut stream_trigger_map: HashMap<String, Option<TriggerAlertData>> = HashMap::new();
     let mut stream_partitioning_map: HashMap<String, PartitioningDetails> = HashMap::new();
 

src/service/metrics/otlp_http.rs

@@ -40,7 +40,7 @@ use prost::Message;
 use crate::{
     common::meta::{
         self,
-        alerts::Alert,
+        alerts::alert::Alert,
         http::HttpResponse as MetaHttpResponse,
         prom::{self, MetricType, HASH_LABEL, NAME_LABEL, VALUE_LABEL},
         stream::{SchemaRecords, StreamParams},

src/service/metrics/prom.rs

@@ -43,7 +43,7 @@ use crate::{
     common::{
         infra::config::{METRIC_CLUSTER_LEADER, METRIC_CLUSTER_MAP},
         meta::{
-            alerts,
+            alerts::alert,
             functions::StreamTransform,
             prom::*,
             stream::{SchemaRecords, StreamParams},
@@ -95,7 +95,7 @@ pub async fn remote_write(
     let mut metric_data_map: HashMap<String, HashMap<String, SchemaRecords>> = HashMap::new();
     let mut metric_schema_map: HashMap<String, SchemaCache> = HashMap::new();
     let mut schema_evolved: HashMap<String, bool> = HashMap::new();
-    let mut stream_alerts_map: HashMap<String, Vec<alerts::Alert>> = HashMap::new();
+    let mut stream_alerts_map: HashMap<String, Vec<alert::Alert>> = HashMap::new();
     let mut stream_trigger_map: HashMap<String, Option<TriggerAlertData>> = HashMap::new();
     let mut stream_transform_map: HashMap<String, Vec<StreamTransform>> = HashMap::new();
     let mut stream_partitioning_map: HashMap<String, PartitioningDetails> = HashMap::new();

src/service/organization.rs

@@ -35,7 +35,7 @@ use crate::{
 pub async fn get_summary(org_id: &str) -> OrgSummary {
     let streams = get_streams(org_id, None, false, None).await;
     let functions = db::functions::list(org_id).await.unwrap();
-    let alerts = db::alerts::list(org_id, None, None).await.unwrap();
+    let alerts = db::alerts::alert::list(org_id, None, None).await.unwrap();
     let mut num_streams = 0;
     let mut total_storage_size = 0.00;
     let mut total_compressed_size = 0.00;

src/service/pipelines.rs

@@ -19,7 +19,7 @@ use actix_web::{
     http::{self, StatusCode},
     HttpResponse,
 };
-use config::meta::stream::StreamType;
+use config::meta::{search::SearchEventType, stream::StreamType};
 
 use super::db;
 use crate::common::{
@@ -27,24 +27,52 @@ use crate::common::{
     meta::{
         http::HttpResponse as MetaHttpResponse,
         pipelines::{PipeLine, PipeLineList},
+        stream::StreamParams,
     },
 };
 
 #[tracing::instrument(skip(pipeline))]
-pub async fn save_pipeline(org_id: String, pipeline: PipeLine) -> Result<HttpResponse, Error> {
-    if let Some(_existing_pipeline) = check_existing_pipeline(
+pub async fn save_pipeline(org_id: String, mut pipeline: PipeLine) -> Result<HttpResponse, Error> {
+    if check_existing_pipeline(
         &org_id,
         pipeline.stream_type,
         &pipeline.stream_name,
         &pipeline.name,
     )
     .await
+    .is_some()
     {
-        Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error(
+        return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error(
             StatusCode::BAD_REQUEST.into(),
             "Pipeline already exits".to_string(),
-        )))
-    } else if let Err(error) = db::pipelines::set(&org_id, &pipeline.name, &pipeline).await {
+        )));
+    }
+
+    // Save DerivedStream details if there's any
+    if let Some(ref mut derived_streams) = &mut pipeline.derived_streams {
+        for derived_stream in derived_streams {
+            derived_stream.source =
+                StreamParams::new(&org_id, &pipeline.stream_name, pipeline.stream_type);
+            derived_stream.query_condition.search_event_type = Some(SearchEventType::DerivedStream);
+            if !derived_stream.is_valid() {
+                return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error(
+                    http::StatusCode::BAD_REQUEST.into(),
+                    "Invalid DerivedStream details. Name, destination, and Trigger Period required"
+                        .to_string(),
+                )));
+            }
+            if let Err(e) =
+                super::alerts::derived_streams::save(derived_stream.clone(), &pipeline.name).await
+            {
+                return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error(
+                    http::StatusCode::BAD_REQUEST.into(),
+                    format!("Failed to save DerivedStream details error: {}", e),
+                )));
+            }
+        }
+    }
+
+    if let Err(error) = db::pipelines::set(&org_id, &pipeline).await {
         return Ok(
             HttpResponse::InternalServerError().json(MetaHttpResponse::message(
                 http::StatusCode::INTERNAL_SERVER_ERROR.into(),
@@ -60,16 +88,12 @@ pub async fn save_pipeline(org_id: String, pipeline: PipeLine) -> Result<HttpRes
 }
 
 #[tracing::instrument(skip(pipeline))]
-pub async fn update_pipeline(
-    org_id: &str,
-    pipeline_name: &str,
-    pipeline: PipeLine,
-) -> Result<HttpResponse, Error> {
+pub async fn update_pipeline(org_id: &str, mut pipeline: PipeLine) -> Result<HttpResponse, Error> {
     let existing_pipeline = match check_existing_pipeline(
         org_id,
         pipeline.stream_type,
         &pipeline.stream_name,
-        pipeline_name,
+        &pipeline.name,
     )
     .await
     {
@@ -85,7 +109,32 @@ pub async fn update_pipeline(
         return Ok(HttpResponse::Ok().json(pipeline));
     }
 
-    if let Err(error) = db::pipelines::set(org_id, &pipeline.name, &pipeline).await {
+    // Update DerivedStream details if there's any
+    if let Some(ref mut derived_streams) = &mut pipeline.derived_streams {
+        for derived_stream in derived_streams {
+            derived_stream.source =
+                StreamParams::new(org_id, &pipeline.stream_name, pipeline.stream_type);
+            derived_stream.query_condition.search_event_type = Some(SearchEventType::DerivedStream);
+            if !derived_stream.is_valid() {
+                return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error(
+                    http::StatusCode::BAD_REQUEST.into(),
+                    "Invalid DerivedStream details. Name, destination, and Trigger Period required"
+                        .to_string(),
+                )));
+            }
+            // derived_streams::save updates existing triggers when found
+            if let Err(e) =
+                super::alerts::derived_streams::save(derived_stream.clone(), &pipeline.name).await
+            {
+                return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error(
+                    http::StatusCode::BAD_REQUEST.into(),
+                    format!("Failed to update DerivedStream details with error {}", e),
+                )));
+            }
+        }
+    }
+
+    if let Err(error) = db::pipelines::set(org_id, &pipeline).await {
         return Ok(
             HttpResponse::InternalServerError().json(MetaHttpResponse::message(
                 http::StatusCode::INTERNAL_SERVER_ERROR.into(),
@@ -135,12 +184,50 @@ pub async fn list_pipelines(
 
 #[tracing::instrument]
 pub async fn delete_pipeline(
-    org_id: &str,
-    stream_type: StreamType,
-    stream_name: &str,
     pipeline_name: &str,
+    source: StreamParams,
 ) -> Result<HttpResponse, Error> {
-    let result = db::pipelines::delete(org_id, stream_type, stream_name, pipeline_name).await;
+    let existing_pipeline = match check_existing_pipeline(
+        &source.org_id,
+        source.stream_type,
+        &source.stream_name,
+        pipeline_name,
+    )
+    .await
+    {
+        Some(pipeline) => pipeline,
+        None => {
+            return Ok(HttpResponse::NotFound().json(MetaHttpResponse::error(
+                StatusCode::NOT_FOUND.into(),
+                "Pipeline not found".to_string(),
+            )));
+        }
+    };
+
+    // delete DerivedStream details if there's any
+    if let Some(derived_streams) = existing_pipeline.derived_streams {
+        for derived_stream in derived_streams {
+            if let Err(error) =
+                super::alerts::derived_streams::delete(derived_stream, &existing_pipeline.name)
+                    .await
+            {
+                return Ok(
+                    HttpResponse::InternalServerError().json(MetaHttpResponse::message(
+                        http::StatusCode::INTERNAL_SERVER_ERROR.into(),
+                        error.to_string(),
+                    )),
+                );
+            }
+        }
+    }
+
+    let result = db::pipelines::delete(
+        &source.org_id,
+        source.stream_type,
+        &source.stream_name,
+        pipeline_name,
+    )
+    .await;
     match result {
         Ok(_) => Ok(HttpResponse::Ok().json(MetaHttpResponse::message(
             http::StatusCode::OK.into(),

src/service/search/datafusion/plan.rs

@@ -55,7 +55,8 @@ pub fn get_partial_plan(
     if cplan.name() == "RepartitionExec" {
         let plan = cplan.as_any().downcast_ref::<RepartitionExec>().unwrap();
         if let Partitioning::RoundRobinBatch(_) = plan.partitioning() {
-            return Ok(None);
+            let child = *cplan.children().first().unwrap();
+            return get_partial_plan(child);
         }
     }
     if DISTRIBUTED_PLAN_NAMES.contains(&cplan.name()) {

src/service/search/mod.rs

@@ -160,6 +160,7 @@ pub async fn search(
                     search::SearchEventType::Dashboards => (true, in_req.search_type),
                     search::SearchEventType::Reports => (true, in_req.search_type),
                     search::SearchEventType::Alerts => (true, in_req.search_type),
+                    search::SearchEventType::DerivedStream => (true, in_req.search_type),
                     search::SearchEventType::RUM => (true, in_req.search_type),
                     search::SearchEventType::Values => (false, None),
                     search::SearchEventType::Other => (false, None),

src/service/traces/mod.rs

@@ -41,7 +41,7 @@ use prost::Message;
 
 use crate::{
     common::meta::{
-        alerts::Alert,
+        alerts::alert::Alert,
         http::HttpResponse as MetaHttpResponse,
         stream::{SchemaRecords, StreamParams},
         traces::{Event, Span, SpanLink, SpanLinkContext, SpanRefType},

web/src/components/alerts/FieldsInput.vue

@@ -1,3 +1,19 @@
+<!-- Copyright 2023 Zinc Labs Inc.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-->
+
 <template>
   <div>
     <div data-test="alert-conditions-text" class="text-bold">
@@ -23,7 +39,7 @@
     </template>
     <template v-else>
       <div
-        v-for="(field, index) in (fields as any)"
+        v-for="(field, index) in fields as any"
         :key="field.uuid"
         class="flex justify-start items-end q-col-gutter-sm q-pb-sm"
         :data-test="`alert-conditions-${index + 1}`"
@@ -187,7 +203,7 @@ const filterColumns = (val: string, update: Function) => {
   update(() => {
     const value = val.toLowerCase();
     filteredFields.value = props.streamFields.filter(
-      (column: any) => column.value.toLowerCase().indexOf(value) > -1
+      (column: any) => column.value.toLowerCase().indexOf(value) > -1,
     );
   });
 };

web/src/components/alerts/PreviewAlert.vue

@@ -1,3 +1,19 @@
+<!-- Copyright 2023 Zinc Labs Inc.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-->
+
 <template>
   <div ref="chartPanelRef" style="height: 100%; position: relative">
     <div style="height: 200px" data-test="alert-preview-chart">
@@ -150,7 +166,9 @@ const refreshData = () => {
 
   const endTime = new Date().getTime() * 1000;
   let new_relative_time = 5;
-  if (relativeTime < 5) { new_relative_time = relativeTime };
+  if (relativeTime < 5) {
+    new_relative_time = relativeTime;
+  }
 
   const startTime = endTime - new_relative_time * 60 * 1000000;
 

web/src/components/alerts/RealTimeAlert.vue

@@ -1,3 +1,19 @@
+<!-- Copyright 2023 Zinc Labs Inc.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-->
+
 <template>
   <div>
     <fields-input
@@ -6,7 +22,9 @@
       :fields="conditions"
       @add="addField"
       @remove="removeField"
-      @input:update="(name:string, field: any) => emits('input:update', name, field)"
+      @input:update="
+        (name: string, field: any) => emits('input:update', name, field)
+      "
     />
   </div>
 </template>

web/src/components/alerts/ScheduledAlert.vue

@@ -1,6 +1,22 @@
+<!-- Copyright 2023 Zinc Labs Inc.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-->
+
 <template>
   <div class="scheduled-alerts">
-    <div class="scheduled-alert-tabs q-mb-lg">
+    <div v-if="!disableQueryTypeSelection" class="scheduled-alert-tabs q-mb-lg">
       <q-tabs
         data-test="scheduled-alert-tabs"
         v-model="tab"
@@ -45,6 +61,7 @@
           {{ tab === "promql" ? "Promql" : "SQL" }}
         </div>
         <q-toggle
+          v-if="!disableVrlFunction"
           data-test="logs-search-bar-show-query-toggle-btn"
           v-model="isVrlFunctionEnabled"
           :icon="'img:' + getImageURL('images/common/function.svg')"
@@ -89,7 +106,7 @@
 
         <div
           data-test="logs-vrl-function-editor"
-          v-show="isVrlFunctionEnabled && tab === 'sql'"
+          v-show="!disableVrlFunction && isVrlFunctionEnabled && tab === 'sql'"
         >
           <div style="height: 40px; width: 100%">
             <div style="display: flex; height: 40px">
@@ -280,7 +297,10 @@
           />
         </div>
       </div>
-      <div class="flex justify-start items-center q-mb-xs no-wrap q-pb-md">
+      <div
+        v-if="!disableThreshold"
+        class="flex justify-start items-center q-mb-xs no-wrap q-pb-md"
+      >
         <div
           data-test="scheduled-alert-threshold-title"
           class="text-bold flex items-center"
@@ -710,7 +730,7 @@
 
 <script setup lang="ts">
 import { ref, watch, computed, type Ref, defineAsyncComponent } from "vue";
-import FieldsInput from "./FieldsInput.vue";
+import FieldsInput from "@/components/alerts/FieldsInput.vue";
 import { useI18n } from "vue-i18n";
 import {
   outlinedDelete,
@@ -720,6 +740,7 @@ import { useStore } from "vuex";
 import { getImageURL } from "@/utils/zincutils";
 import useQuery from "@/composables/useQuery";
 import searchService from "@/services/search";
+import { useQuasar } from "quasar";
 
 const QueryEditor = defineAsyncComponent(
   () => import("@/components/QueryEditor.vue"),
@@ -739,6 +760,9 @@ const props = defineProps([
   "vrl_function",
   "showVrlFunction",
   "isValidSqlQuery",
+  "disableThreshold",
+  "disableVrlFunction",
+  "disableQueryTypeSelection",
 ]);
 
 const emits = defineEmits([
@@ -767,6 +791,8 @@ const promqlQuery = ref(props.promql);
 
 const tab = ref(props.query_type || "custom");
 
+const q = useQuasar();
+
 const store = useStore();
 
 const functionEditorPlaceholderFlag = ref(true);
@@ -977,8 +1003,60 @@ const onBlurQueryEditor = () => {
   emits("validate-sql");
 };
 
+const validateInputs = (notify: boolean = true) => {
+  if (
+    Number(triggerData.value.period) < 1 ||
+    isNaN(Number(triggerData.value.period))
+  ) {
+    notify &&
+      q.notify({
+        type: "negative",
+        message: "Period should be greater than 0",
+        timeout: 1500,
+      });
+    return false;
+  }
+
+  if (aggregationData.value) {
+    if (
+      !props.disableThreshold &&
+      (isNaN(triggerData.value.threshold) ||
+        !aggregationData.value.having.value.toString().trim().length ||
+        !aggregationData.value.having.column ||
+        !aggregationData.value.having.operator)
+    ) {
+      notify &&
+        q.notify({
+          type: "negative",
+          message: "Threshold should not be empty",
+          timeout: 1500,
+        });
+      return false;
+    }
+
+    return true;
+  }
+
+  if (
+    !props.disableThreshold &&
+    (isNaN(triggerData.value.threshold) ||
+      triggerData.value.threshold < 1 ||
+      !triggerData.value.operator)
+  ) {
+    notify &&
+      q.notify({
+        type: "negative",
+        message: "Threshold should not be empty",
+        timeout: 1500,
+      });
+    return false;
+  }
+
+  return true;
+};
 defineExpose({
   tab,
+  validateInputs,
 });
 </script>
 

web/src/components/alerts/VariablesInput.vue

@@ -1,3 +1,19 @@
+<!-- Copyright 2023 Zinc Labs Inc.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-->
+
 <template>
   <div class="col-12 q-py-sm variables-input">
     <template v-if="!variables.length">
@@ -18,7 +34,7 @@
     </template>
     <template v-else>
       <div
-        v-for="(variable, index) in (variables as any)"
+        v-for="(variable, index) in variables as any"
         :key="variable.uuid"
         class="q-col-gutter-sm q-pb-sm flex items-center"
         :data-test="`alert-variables-${index + 1}`"

web/src/components/pipeline/AssociateFunction.vue

@@ -1,3 +1,19 @@
+<!-- Copyright 2023 Zinc Labs Inc.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-->
+
 <template>
   <div
     data-test="add-stream-routing-section"
@@ -78,7 +94,9 @@
               filled
               dense
               type="number"
-              :rules="[(val: any) => (!!val && val > -1) || 'Field is required!']"
+              :rules="[
+                (val: any) => (!!val && val > -1) || 'Field is required!',
+              ]"
               tabindex="0"
               style="min-width: 220px"
             />
@@ -169,7 +187,7 @@ interface StreamRoute {
 }
 
 const AddFunction = defineAsyncComponent(
-  () => import("../functions/AddFunction.vue")
+  () => import("../functions/AddFunction.vue"),
 );
 
 const props = defineProps({
@@ -251,7 +269,7 @@ watch(
   {
     deep: true,
     immediate: true,
-  }
+  },
 );
 
 onBeforeMount(() => {

web/src/components/pipeline/PipelineEditor.vue

@@ -1,3 +1,19 @@
+<!-- Copyright 2023 Zinc Labs Inc.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-->
+
 <template>
   <div class="flex justify-between items-center q-pb-sm">
     <div class="flex items-center">
@@ -164,7 +180,7 @@ const streamRouteImage = getImageURL("images/pipeline/route.svg");
 const conditionImage = getImageURL("images/pipeline/condition.svg");
 
 const ChartRenderer = defineAsyncComponent(
-  () => import("@/components/dashboards/panels/ChartRenderer.vue")
+  () => import("@/components/dashboards/panels/ChartRenderer.vue"),
 );
 
 interface Routing {
@@ -191,6 +207,7 @@ interface Pipeline {
   stream_type: string;
   routing: Routing;
   functions: Function[];
+  derived_streams: any[];
 }
 
 interface Node {
@@ -246,6 +263,7 @@ const pipeline = ref<Pipeline>({
   stream_name: "",
   routing: {},
   functions: [],
+  derived_streams: [],
 });
 
 const router = useRouter();
@@ -337,7 +355,7 @@ const getPipeline = () => {
         (pipeline: Pipeline) =>
           pipeline.name === route.query.name &&
           pipeline.stream_name === route.query.stream &&
-          pipeline.stream_type === route.query.stream_type
+          pipeline.stream_type === route.query.stream_type,
       );
 
       if (!_pipeline) {
@@ -391,7 +409,7 @@ const setupNodes = () => {
       createFunctionNode(
         _function.name,
         pipeline.value.stream_name,
-        _function.order
+        _function.order,
       );
 
       functions.value[_function.name] = _function;
@@ -409,9 +427,17 @@ const setupNodes = () => {
       streamRoutes.value[stream] = {
         name: stream,
         conditions: pipeline.value?.routing[stream],
+        is_real_time: true,
       };
     });
 
+  (pipeline.value.derived_streams || []).forEach((stream: any) => {
+    createStreamRouteNode({
+      name: stream.name,
+    });
+    streamRoutes.value[stream.name] = { ...stream };
+  });
+
   updateGraph();
 };
 
@@ -436,7 +462,7 @@ const getFunctionFrom = (stream: string) => {
   let fromNodeName = stream;
 
   const functions = nodes.value.filter(
-    (node) => node.type === "function" && node.stream === stream
+    (node) => node.type === "function" && node.stream === stream,
   );
 
   if (functions.length) {
@@ -522,7 +548,7 @@ const addFunctionNode = (data: { data: Function }) => {
 
   if (editingFunctionName.value) {
     const editedNode = nodes.value.find(
-      (node) => node.name === editingFunctionName.value
+      (node) => node.name === editingFunctionName.value,
     );
 
     if (editedNode) editedNode.order = data.data.order;
@@ -533,8 +559,8 @@ const addFunctionNode = (data: { data: Function }) => {
       createFunctionNode(
         nodeName,
         pipeline.value.stream_name,
-        data.data.order
-      ) as Node
+        data.data.order,
+      ) as Node,
     );
   }
 
@@ -544,13 +570,13 @@ const addFunctionNode = (data: { data: Function }) => {
     .sort((a, b) =>
       a.order?.toString() && b.order?.toString()
         ? Number(a.order) - Number(b.order)
-        : 0
+        : 0,
     )
     .forEach((node) => {
       createFunctionNode(
         node.name,
         node.stream as string,
-        node.order as number
+        node.order as number,
       );
     });
 
@@ -572,7 +598,7 @@ const addFunctionNode = (data: { data: Function }) => {
 const createFunctionNode = (
   nodeName: string,
   streamName: string,
-  order: number
+  order: number,
 ) => {
   const position = getNodePosition("function");
 
@@ -604,7 +630,7 @@ const updateFunctionNodesOrder = () => {
     .sort((a, b) =>
       a.order?.toString() && b.order?.toString()
         ? Number(a.order) - Number(b.order)
-        : 0
+        : 0,
     );
 
   sortedFunctionNodes.forEach((node, index) => {
@@ -626,15 +652,10 @@ const addStreamNode = (data: { data: Node }) => {
   streamRoutes.value[nodeName] = data.data;
 
   if (editingStreamRouteName.value) {
-    nodes.value = nodes.value.filter(
-      (node) =>
-        node.name !== editingFunctionName.value &&
-        node.name !== editingStreamRouteName.value + ":condition"
-    );
-
     editingStreamRouteName.value = "";
+  } else {
+    createStreamRouteNode(data.data);
   }
-  createStreamRouteNode(data.data);
 
   updateGraph();
 };
@@ -747,7 +768,7 @@ const getFunctions = () => {
       "name",
       false,
       "",
-      store.state.selectedOrganization.identifier
+      store.state.selectedOrganization.identifier,
     )
     .then((res) => {
       functions.value = {};
@@ -764,11 +785,12 @@ const getFunctions = () => {
 
 const deleteNode = (node: { data: Node; type: string }) => {
   nodes.value = nodes.value.filter((n) => n.name !== node.data.name);
+
   delete nodeLinks.value[node.data.name];
 
   if (node.type === "function") {
     associatedFunctions.value = associatedFunctions.value.filter(
-      (func) => func !== node.data.name
+      (func) => func !== node.data.name,
     );
     editingFunctionName.value = "";
   }
@@ -778,6 +800,10 @@ const deleteNode = (node: { data: Node; type: string }) => {
     editingStreamRouteName.value = "";
   }
 
+  nodeRows.value.forEach((row, index) => {
+    if (row === node.data.name) nodeRows.value[index] = null;
+  });
+
   updateFunctionNodesOrder();
 
   updateGraph();
@@ -797,9 +823,16 @@ const getPipelinePayload = () => {
     stream_name: pipeline.value.stream_name,
     stream_type: pipeline.value.stream_type,
     routing: {} as Routing,
+    derived_streams: [] as any[],
   };
+
   Object.values(streamRoutes.value).forEach((route: any) => {
-    payload.routing[route.name] = route.conditions;
+    if (!route.is_real_time) {
+      payload.derived_streams.push(route);
+      return;
+    } else {
+      payload.routing[route.name] = route.conditions;
+    }
   });
 
   return payload;
@@ -809,12 +842,12 @@ const associateFunctions = async () => {
   const associateFunctionPromises = [];
 
   const previousAssociatedFunctions = pipeline.value.functions.map(
-    (func) => func.name
+    (func) => func.name,
   );
 
   for (let i = 0; i < associatedFunctions.value.length; i++) {
     const functionIndex = previousAssociatedFunctions.indexOf(
-      associatedFunctions.value[i]
+      associatedFunctions.value[i],
     );
 
     // If function is already associated with the pipeline and order is same, skip.
@@ -836,8 +869,8 @@ const associateFunctions = async () => {
         associatedFunctions.value[i],
         {
           order: Number(functions.value[associatedFunctions.value[i]].order),
-        }
-      )
+        },
+      ),
     );
   }
   return Promise.all(associateFunctionPromises);
@@ -848,7 +881,7 @@ const deleteFunctionAssociation = (name: string) => {
     store.state.selectedOrganization.identifier,
     pipeline.value.stream_name,
     pipeline.value.stream_type,
-    name
+    name,
   );
 };
 
@@ -866,7 +899,7 @@ const savePipeline = async () => {
     for (let i = 0; i < pipeline.value.functions.length; i++) {
       if (!associatedFunctions.value.includes(pipeline.value.functions[i].name))
         deleteFunctionPromises.push(
-          deleteFunctionAssociation(pipeline.value.functions[i].name)
+          deleteFunctionAssociation(pipeline.value.functions[i].name),
         );
     }
 

web/src/components/pipeline/PipelinesList.vue

@@ -1,3 +1,19 @@
+<!-- Copyright 2023 Zinc Labs Inc.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-->
+
 <template>
   <div v-if="currentRouteName === 'pipelines'">
     <div class="full-wdith">
@@ -247,7 +263,7 @@ const getPipelines = () => {
             ...pipeline,
             "#": index + 1,
           };
-        }
+        },
       );
     })
     .catch((error) => {

web/src/components/pipeline/StreamRouting.vue

@@ -1,7 +1,23 @@
+<!-- Copyright 2023 Zinc Labs Inc.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-->
+
 <template>
   <div
     data-test="add-stream-routing-section"
-    class="full-width full-height"
+    class="full-width stream-routing-section"
     :class="store.state.theme === 'dark' ? 'bg-dark' : 'bg-white'"
   >
     <div class="stream-routing-title q-pb-sm q-pl-md">
@@ -10,7 +26,7 @@
     <q-separator />
 
     <div class="stream-routing-container q-px-md q-pt-md q-pr-xl">
-      <q-form @submit="saveRouting">
+      <q-form ref="routeFormRef" @submit="saveRouting">
         <div
           data-test="stream-routing-name-input"
           class="o2-input"
@@ -36,7 +52,7 @@
                   : t('common.nameRequired'),
             ]"
             tabindex="0"
-            style="width: 480px"
+            style="width: 400px"
             :error-message="
               streamRoute.name && isValidStreamName
                 ? 'Stream name already exists'
@@ -47,12 +63,96 @@
           />
         </div>
 
-        <real-time-alert
-          :columns="filteredColumns"
-          :conditions="streamRoute.conditions"
-          @field:add="addField"
-          @field:remove="removeField"
-        />
+        <div class="q-gutter-sm">
+          <q-radio
+            data-test="add-alert-realtime-alert-radio"
+            v-bind:readonly="isUpdating"
+            v-bind:disable="isUpdating"
+            v-model="streamRoute.is_real_time"
+            :checked="!streamRoute.is_real_time"
+            :val="true"
+            :label="t('alerts.realTime')"
+            class="q-ml-none"
+          />
+          <q-radio
+            data-test="add-alert-scheduled-alert-radio"
+            v-bind:readonly="isUpdating"
+            v-bind:disable="isUpdating"
+            v-model="streamRoute.is_real_time"
+            :checked="streamRoute.is_real_time"
+            :val="false"
+            :label="t('alerts.standard')"
+            class="q-ml-none"
+          />
+        </div>
+
+        <div
+          v-if="streamRoute.is_real_time"
+          class="q-py-sm showLabelOnTop text-bold text-h7"
+          data-test="add-alert-query-input-title"
+        >
+          <real-time-alert
+            v-if="streamRoute.is_real_time"
+            :columns="filteredColumns"
+            :conditions="streamRoute.conditions"
+            @field:add="addField"
+            @field:remove="removeField"
+          />
+        </div>
+        <div v-else>
+          <div
+            data-test="stream-route-stream-type-select"
+            class="stream-route-stream-type o2-input q-mr-sm q-my-md"
+            style="padding-top: 0"
+          >
+            <q-select
+              v-model="streamRoute.destination.stream_type"
+              :options="streamTypes"
+              :label="t('alerts.streamType') + ' *'"
+              :popup-content-style="{ textTransform: 'lowercase' }"
+              color="input-border"
+              bg-color="input-bg"
+              class="q-py-sm showLabelOnTop no-case"
+              stack-label
+              outlined
+              filled
+              dense
+              v-bind:readonly="isUpdating"
+              v-bind:disable="isUpdating"
+              :rules="[(val: any) => !!val || 'Field is required!']"
+              style="width: 400px"
+            />
+          </div>
+          <scheduled-alert
+            ref="scheduledAlertRef"
+            :columns="filteredColumns"
+            :conditions="[]"
+            :alertData="streamRoute"
+            :disableThreshold="true"
+            :disableVrlFunction="true"
+            :isValidSqlQuery="isValidSqlQuery"
+            :disableQueryTypeSelection="true"
+            v-model:trigger="streamRoute.trigger_condition"
+            v-model:sql="streamRoute.query_condition.sql"
+            v-model:query_type="streamRoute.query_condition.type"
+            v-model:aggregation="streamRoute.query_condition.aggregation"
+            v-model:isAggregationEnabled="isAggregationEnabled"
+            @field:add="addField"
+            @field:remove="removeField"
+            @validate-sql="validateSqlQuery"
+            class="q-mt-sm"
+          />
+
+          <div class="q-mt-md">
+            <div class="text-bold">{{ t("alerts.additionalVariables") }}</div>
+            <variables-input
+              class="o2-input"
+              :variables="streamRoute.context_attributes"
+              @add:variable="addVariable"
+              @remove:variable="removeVariable"
+            />
+          </div>
+        </div>
 
         <div
           class="flex justify-start q-mt-lg q-py-sm full-width"
@@ -99,7 +199,7 @@
   />
 </template>
 <script lang="ts" setup>
-import { computed, onMounted, ref, type Ref } from "vue";
+import { computed, defineAsyncComponent, onMounted, ref, type Ref } from "vue";
 import { useI18n } from "vue-i18n";
 import RealTimeAlert from "../alerts/RealTimeAlert.vue";
 import { getUUID } from "@/utils/zincutils";
@@ -108,6 +208,13 @@ import { useRouter } from "vue-router";
 import useStreams from "@/composables/useStreams";
 import ConfirmDialog from "../ConfirmDialog.vue";
 import { useQuasar } from "quasar";
+import ScheduledAlert from "@/components/alerts/ScheduledAlert.vue";
+import useQuery from "@/composables/useQuery";
+import searchService from "@/services/search";
+
+const VariablesInput = defineAsyncComponent(
+  () => import("@/components/alerts/VariablesInput.vue"),
+);
 
 interface RouteCondition {
   column: string;
@@ -119,6 +226,29 @@ interface RouteCondition {
 interface StreamRoute {
   name: string;
   conditions: RouteCondition[];
+  destination: {
+    // should be part of payload
+    org_id: string;
+    stream_name: string;
+    stream_type: string;
+  };
+  is_real_time: boolean;
+  query_condition: {
+    sql: string;
+    type: string;
+    aggregation: {
+      group_by: string[];
+    } | null;
+  };
+  trigger_condition: {
+    period: number;
+    frequency_type: string;
+    frequency: number;
+    cron: string;
+  };
+  context_attributes: any;
+  description: string;
+  enabled: boolean;
 }
 
 const props = defineProps({
@@ -152,25 +282,31 @@ const store = useStore();
 
 const { getStream, getStreams } = useStreams();
 
+const { buildQueryPayload } = useQuery();
+
 const emit = defineEmits(["update:node", "cancel:hideform", "delete:node"]);
 
 const isUpdating = ref(false);
 
 const filteredColumns: any = ref([]);
 
-const isFetchingStreams = ref(false);
+const isValidSqlQuery = ref(true);
+
+const validateSqlQueryPromise = ref<Promise<unknown>>();
+
+const scheduledAlertRef = ref<any>(null);
 
 const filteredStreams: Ref<any[]> = ref([]);
 
-const streams: any = ref({});
-
 const indexOptions = ref([]);
 
 const originalStreamFields: Ref<any[]> = ref([]);
 
 const isValidName: Ref<boolean> = ref(true);
 
-let existingStreamNames: any;
+const isAggregationEnabled = ref(false);
+
+const routeFormRef = ref<any>(null);
 
 const nodeLink = ref({
   from: "",
@@ -188,28 +324,67 @@ const getDefaultStreamRoute = () => {
   return {
     name: "",
     conditions: [{ column: "", operator: "", value: "", id: getUUID() }],
+    destination: {
+      org_id: "",
+      stream_name: "",
+      stream_type: "logs",
+    },
+    is_real_time: true,
+    query_condition: {
+      sql: "",
+      type: "sql",
+      aggregation: null,
+    },
+    trigger_condition: {
+      period: 10,
+      frequency_type: "minutes",
+      cron: "",
+      frequency: 2,
+    },
+    context_attributes: [
+      {
+        key: "",
+        value: "",
+        id: getUUID(),
+      },
+    ],
+    description: "",
+    enabled: true,
   };
 };
 
 onMounted(() => {
   if (props.editingRoute) {
     isUpdating.value = true;
+
     streamRoute.value = JSON.parse(
-      JSON.stringify(props.editingRoute)
+      JSON.stringify(props.editingRoute),
     ) as StreamRoute;
+
+    if (!streamRoute.value.is_real_time) {
+      // If aggregation was present enable aggregation toggle
+      isAggregationEnabled.value =
+        !!props.editingRoute.query_condition.aggregation;
+
+      // If context attributes are present, convert them to array
+      streamRoute.value.context_attributes = Object.keys(
+        streamRoute.value.context_attributes,
+      ).map((attr: string) => {
+        return {
+          key: attr,
+          value: streamRoute.value.context_attributes[attr],
+          id: getUUID(),
+        };
+      });
+    }
   }
 
   originalStreamRouting.value = JSON.parse(JSON.stringify(streamRoute.value));
 
-  existingStreamNames = new Set(
-    ...Object.values(props.streamRoutes).map((route: any) => route.name),
-    props.streamName
-  );
-
   updateStreamFields();
 });
 
-const streamTypes = ["logs", "metrics", "traces"];
+const streamTypes = ["logs", "enrichment_tables"];
 
 const streamRoute: Ref<StreamRoute> = ref(getDefaultStreamRoute());
 
@@ -226,7 +401,7 @@ const filterColumns = (options: any[], val: String, update: Function) => {
   update(() => {
     const value = val.toLowerCase();
     filteredOptions = options.filter(
-      (column: any) => column.toLowerCase().indexOf(value) > -1
+      (column: any) => column.toLowerCase().indexOf(value) > -1,
     );
   });
   return filteredOptions;
@@ -247,7 +422,7 @@ const updateStreamFields = async () => {
   const streams: any = await getStream(
     props.streamName,
     props.streamType,
-    true
+    true,
   );
 
   if (streams && Array.isArray(streams.schema)) {
@@ -262,18 +437,22 @@ const updateStreamFields = async () => {
 };
 
 const addField = () => {
-  streamRoute.value.conditions.push({
-    column: "",
-    operator: "=",
-    value: "",
-    id: getUUID(),
-  });
+  if (streamRoute.value.is_real_time) {
+    streamRoute.value.conditions.push({
+      column: "",
+      operator: "",
+      value: "",
+      id: getUUID(),
+    });
+  }
 };
 
 const removeField = (field: any) => {
-  streamRoute.value.conditions = streamRoute.value.conditions.filter(
-    (_field: any) => _field.id !== field.id
-  );
+  if (streamRoute.value.is_real_time) {
+    streamRoute.value.conditions = streamRoute.value.conditions.filter(
+      (_field: any) => _field.id !== field.id,
+    );
+  }
 };
 
 const closeDialog = () => {
@@ -296,7 +475,7 @@ const openCancelDialog = () => {
 };
 
 // TODO OK : Add check for duplicate routing name
-const saveRouting = () => {
+const saveRouting = async () => {
   isValidName.value = true;
 
   if (!isUpdating.value) validateStreamName();
@@ -305,10 +484,28 @@ const saveRouting = () => {
     return;
   }
 
+  if (!streamRoute.value.is_real_time) {
+    if (!scheduledAlertRef.value.validateInputs()) {
+      return false;
+    }
+  }
+
+  try {
+    await validateSqlQueryPromise.value;
+  } catch (e) {
+    return false;
+  }
+
+  routeFormRef.value.validate().then((valid: any) => {
+    if (!valid) {
+      return false;
+    }
+  });
+
   // Save routing
   emit("update:node", {
     data: {
-      ...streamRoute.value,
+      ...getRoutePayload(),
       name: streamRoute.value.name,
     },
     link: nodeLink.value,
@@ -317,10 +514,6 @@ const saveRouting = () => {
   emit("cancel:hideform");
 };
 
-const saveUpdatedLink = (link: { from: string; to: string }) => {
-  nodeLink.value = link;
-};
-
 const openDeleteDialog = () => {
   dialog.value.show = true;
   dialog.value.title = "Delete Node";
@@ -359,6 +552,126 @@ const validateStreamName = () => {
     }
   });
 };
+
+const addVariable = () => {
+  streamRoute.value.context_attributes.push({
+    key: "",
+    value: "",
+    id: getUUID(),
+  });
+};
+
+const removeVariable = (variable: any) => {
+  streamRoute.value.context_attributes =
+    streamRoute.value.context_attributes.filter(
+      (_variable: any) => _variable.id !== variable.id,
+    );
+};
+
+const getRoutePayload = () => {
+  let payload = JSON.parse(JSON.stringify(streamRoute.value));
+
+  if (payload.uuid) delete payload.uuid;
+
+  if (payload.is_real_time) {
+    payload = {
+      name: payload.name,
+      conditions: payload.conditions,
+      is_real_time: payload.is_real_time,
+    };
+  } else {
+    // Deleting uuid from payload as it was added for reference of frontend
+    payload.destination.org_id = store.state.selectedOrganization.identifier;
+
+    payload.destination.stream_name = payload.name;
+
+    payload.context_attributes = {};
+
+    payload.query_condition.type = payload.is_real_time
+      ? "custom"
+      : streamRoute.value.query_condition.type;
+
+    streamRoute.value.context_attributes.forEach((attr: any) => {
+      if (attr.key?.trim() && attr.value?.trim())
+        payload.context_attributes[attr.key] = attr.value;
+    });
+
+    payload.trigger_condition.period = Number(
+      streamRoute.value.trigger_condition.period,
+    );
+
+    payload.trigger_condition.frequency = Number(
+      streamRoute.value.trigger_condition.frequency,
+    );
+
+    payload.description = streamRoute.value.description.trim();
+
+    if (
+      !isAggregationEnabled.value ||
+      streamRoute.value.query_condition.type !== "custom"
+    ) {
+      payload.query_condition.aggregation = null;
+    }
+
+    if (payload.query_condition.aggregation?.having) {
+      delete payload.query_condition?.aggregation?.having;
+    }
+
+    delete payload?.conditions;
+
+    delete payload?.query_condition?.conditions;
+  }
+
+  if (isUpdating.value) {
+    payload.updatedAt = new Date().toISOString();
+    payload.lastEditedBy = store.state.userInfo.email;
+  } else {
+    payload.createdAt = new Date().toISOString();
+    payload.owner = store.state.userInfo.email;
+    payload.lastTriggeredAt = new Date().getTime();
+    payload.lastEditedBy = store.state.userInfo.email;
+    payload.updatedAt = new Date().toISOString();
+  }
+
+  return payload;
+};
+
+const validateSqlQuery = () => {
+  const query = buildQueryPayload({
+    sqlMode: true,
+    streamName: streamRoute.value.name as string,
+  });
+
+  delete query.aggs;
+
+  query.query.sql = streamRoute.value.query_condition.sql;
+
+  validateSqlQueryPromise.value = new Promise((resolve, reject) => {
+    searchService
+      .search({
+        org_identifier: store.state.selectedOrganization.identifier,
+        query,
+        page_type: "logs",
+      })
+      .then((res: any) => {
+        isValidSqlQuery.value = true;
+        resolve("");
+      })
+      .catch((err: any) => {
+        if (err.response.data.code === 500) {
+          isValidSqlQuery.value = false;
+          q.notify({
+            type: "negative",
+            message: "Invalid SQL Query : " + err.response.data.message,
+            timeout: 3000,
+          });
+          reject("");
+        } else isValidSqlQuery.value = true;
+
+        resolve("");
+      });
+  });
+};
 </script>
 
 <style scoped>
@@ -367,8 +680,12 @@ const validateStreamName = () => {
   padding-top: 16px;
 }
 .stream-routing-container {
-  width: fit-content;
+  width: 720px;
   border-radius: 8px;
   /* box-shadow: 0px 0px 10px 0px #d2d1d1; */
 }
+
+.stream-routing-section {
+  min-height: 100%;
+}
 </style>

web/src/components/pipeline/StreamSelection.vue

@@ -1,3 +1,19 @@
+<!-- Copyright 2023 Zinc Labs Inc.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-->
+
 <template>
   <div
     class="full-height"
@@ -227,7 +243,7 @@ const filterColumns = (options: any[], val: String, update: Function) => {
   update(() => {
     const value = val.toLowerCase();
     filteredOptions = options.filter(
-      (column: any) => column.toLowerCase().indexOf(value) > -1
+      (column: any) => column.toLowerCase().indexOf(value) > -1,
     );
   });
   return filteredOptions;

web/src/composables/useQuery.ts

@@ -22,7 +22,7 @@ interface BuildQueryPayload {
     query: string;
     offset: number;
   };
-  streamName: "";
+  streamName: string;
 }
 
 const getTimeInterval = (start_time: number, end_time: number) => {

web/src/locales/languages/en.json

@@ -405,7 +405,8 @@
     "search": "Search Pipeline",
     "routing": "Routing",
     "nodes": "Nodes",
-    "associateFunction": "Associate Function"
+    "associateFunction": "Associate Function",
+    "derivedStream": "Derived Stream"
   },
   "function": {
     "header": "Functions",