From 0ebaf21f33e8bfdd3b8ab0a7e9eba62bb5ec32b0 Mon Sep 17 00:00:00 2001 From: Sai Nikhil Date: Thu, 31 Oct 2024 10:23:28 +0530 Subject: [PATCH] feat: Pipeline integration v2 (#4732) This PR implements the new design of **Pipeline** introduced by #4432 --- **2 types of pipelines differentiates by the source** Type I: **Realtime** - source is a stream (Logs, Metrics, & Traces) - takes effect when the source stream is being ingested Type II: **Scheduled** (aka DerivedSteram) - source is a SQL query - takes effect based on the given schedule The new pipeline is represented as a graph consists of different types of Nodes that are connected by edges. **Pipeline Data Model** ```rs pub struct Pipeline { pub id: String, pub version: i32, pub enabled: bool, pub org: String, pub name: String, pub description: String, pub source: PipelineSource, pub nodes: Vec, pub edges: Vec, } ``` **4 types of node** I. StreamNode: either used for source node of realtime pipeline or for destination nodes II. QueryNode: used for source node of scheduled pipeline III. FunctionNode: used for executing vrl functions IV. ConditionNode: used for checking routing conditions Rules applied to validating a pipeline when it's created: 1. non-empty nodes list 2. non-empty edges list 3. 1st node in nodes list is either StreamNode or QueryNode 4. non-empty `conditions` in all ConditionNode nodes in nodes list 5. all leaf nodes are of type StreamNode 6. In the same branch, unchecked `after_flattened` FunctionNode can't follow checked `after_flattened` checked FunctionNode Pipeline execution is implemented with the struct `ExecutablePipeline`, a ready-execute pipeline object cached in memory based on the Pipeline objects modified on the UI side. `ExecutablePipeline` object - builds the relationships among all the nodes based on the edges - topologically sorts the nodes based on the level to determine which node to process first - registers all the vrl function node once `ExecutablePipeline` object processes ingested records in batch. Starting from the source node, each connecting edge has a channel connecting the parent and child node, used for passing the records. This approach enables batch processing in parallel. --- **Deprecated Features** 1. previous implementation of pipeline 2. Function x Stream associations - Functions can still be added/edited/removed as before - Needs to create a pipeline in order to apply chosen functions to the desired stream The new release when this pr is merged will automatically migrate **old pipelines** and **Function x Stream associations** to new pipeline format and can be found in `Stream Pipelines` tab with `Migrated-` prefix names. **_Note: Auto generated pipelines are paused by default. Please verify the pipelines before enabling them_** Co-authored-by: Taiming Liu Co-authored-by: Bhargav --- Cargo.lock | 2 + src/cli/basic/cli.rs | 13 + src/common/infra/config.rs | 26 +- src/common/meta/mod.rs | 2 - src/common/meta/organization.rs | 3 +- src/common/meta/pipelines.rs | 48 +- src/common/meta/telemetry.rs | 6 - src/common/migration/mod.rs | 13 + src/common/migration/pipeline_func.rs | 364 +++++++ src/common/migration/schema.rs | 13 +- src/common/utils/functions.rs | 6 +- src/config/Cargo.toml | 2 + .../src}/meta/alerts/alert.rs | 6 +- .../src}/meta/alerts/derived_streams.rs | 6 +- .../src}/meta/alerts/destinations.rs | 0 src/{common => config/src}/meta/alerts/mod.rs | 3 +- .../src}/meta/alerts/templates.rs | 0 .../src/meta/function.rs} | 58 +- src/config/src/meta/mod.rs | 3 + src/config/src/meta/pipeline/components.rs | 238 +++++ src/config/src/meta/pipeline/mod.rs | 486 +++++++++ src/config/src/meta/stream.rs | 112 ++- src/config/src/utils/flatten.rs | 2 +- src/handler/http/request/alerts/alert.rs | 7 +- .../http/request/alerts/destinations.rs | 6 +- src/handler/http/request/alerts/templates.rs | 6 +- src/handler/http/request/functions/mod.rs | 136 +-- src/handler/http/request/mod.rs | 2 +- src/handler/http/request/pipeline.rs | 208 ++++ src/handler/http/request/pipelines.rs | 221 ---- .../http/request/search/multi_streams.rs | 9 +- src/handler/http/request/status/mod.rs | 3 +- src/handler/http/router/mod.rs | 14 +- src/handler/http/router/openapi.rs | 41 +- src/infra/src/lib.rs | 2 + src/infra/src/pipeline/mod.rs | 133 +++ src/infra/src/pipeline/mysql.rs | 375 +++++++ src/infra/src/pipeline/postgres.rs | 375 +++++++ src/infra/src/pipeline/sqlite.rs | 387 +++++++ src/job/mod.rs | 9 +- src/service/alerts/alert.rs | 64 +- src/service/alerts/derived_streams.rs | 119 +-- src/service/alerts/destinations.rs | 6 +- src/service/alerts/mod.rs | 99 +- src/service/alerts/scheduler.rs | 225 +++-- src/service/alerts/templates.rs | 3 +- src/service/db/alerts/alert.rs | 10 +- src/service/db/alerts/destinations.rs | 7 +- src/service/db/alerts/templates.rs | 7 +- src/service/db/functions.rs | 55 +- src/service/db/mod.rs | 2 +- src/service/db/pipeline.rs | 231 +++++ src/service/db/pipelines.rs | 162 --- src/service/functions.rs | 147 +-- src/service/ingestion/mod.rs | 164 +-- src/service/logs/bulk.rs | 479 +++++---- src/service/logs/ingest.rs | 331 +++--- src/service/logs/mod.rs | 9 +- src/service/logs/otlp_grpc.rs | 296 +++--- src/service/logs/otlp_http.rs | 292 +++--- src/service/logs/syslog.rs | 294 +++--- src/service/metrics/json.rs | 453 ++++++--- src/service/metrics/otlp_grpc.rs | 445 ++++---- src/service/metrics/otlp_http.rs | 433 ++++---- src/service/metrics/prom.rs | 194 ++-- src/service/mod.rs | 2 +- src/service/pipeline/batch_execution.rs | 547 ++++++++++ src/service/pipeline/mod.rs | 286 ++++++ src/service/pipelines.rs | 255 ----- src/service/search/cluster/http.rs | 11 +- src/service/search/datafusion/udf/mod.rs | 4 +- src/service/search/mod.rs | 14 +- src/service/traces/mod.rs | 279 ++++-- tests/api-testing/tests/test_functions.py | 45 +- tests/integration_test.rs | 100 +- .../playwright-tests/pipeline.spec.js | 16 +- web/package-lock.json | 218 ++++ web/package.json | 3 + web/src/assets/images/pipeline/condition.svg | 5 +- web/src/assets/images/pipeline/function.svg | 5 +- .../assets/images/pipeline/outputStream.svg | 41 + web/src/assets/images/pipeline/query.svg | 37 + web/src/assets/images/pipeline/stream.svg | 5 +- web/src/components/alerts/FieldsInput.vue | 12 +- web/src/components/alerts/RealTimeAlert.vue | 3 +- web/src/components/logstream/AddStream.vue | 21 +- web/src/components/pipeline/FlowChart.vue | 46 + .../{ => NodeForm}/AssociateFunction.vue | 240 +++-- .../pipeline/NodeForm/Condition.vue | 558 +++++++++++ .../{StreamRouting.vue => NodeForm/Query.vue} | 421 ++------ .../{ => NodeForm}/ScheduledPipeline.vue | 23 +- .../components/pipeline/NodeForm/Stream.vue | 441 ++++++++ web/src/components/pipeline/NodeSidebar.vue | 67 ++ .../components/pipeline/PipelineEditor.vue | 948 +++++++----------- web/src/components/pipeline/PipelineView.vue | 198 ++++ web/src/components/pipeline/PipelinesList.vue | 529 ++++++++-- .../components/pipeline/StreamSelection.vue | 3 +- web/src/composables/shared/router.ts | 17 +- web/src/composables/useStreams.ts | 2 +- web/src/locales/languages/en.json | 13 +- web/src/plugins/pipelines/CustomNode.vue | 610 +++++++++++ .../plugins/pipelines/DropzoneBackground.vue | 31 + web/src/plugins/pipelines/EdgeWithButton.vue | 92 ++ web/src/plugins/pipelines/PipelineFlow.vue | 251 +++++ web/src/plugins/pipelines/useDnD.ts | 707 +++++++++++++ web/src/services/pipelines.ts | 45 +- web/src/views/Functions.vue | 14 - web/src/views/LogStream.vue | 1 + 108 files changed, 10612 insertions(+), 4427 deletions(-) create mode 100644 src/common/migration/pipeline_func.rs rename src/{common => config/src}/meta/alerts/alert.rs (97%) rename src/{common => config/src}/meta/alerts/derived_streams.rs (95%) rename src/{common => config/src}/meta/alerts/destinations.rs (100%) rename src/{common => config/src}/meta/alerts/mod.rs (99%) rename src/{common => config/src}/meta/alerts/templates.rs (100%) rename src/{common/meta/functions.rs => config/src/meta/function.rs} (73%) create mode 100644 src/config/src/meta/pipeline/components.rs create mode 100644 src/config/src/meta/pipeline/mod.rs create mode 100644 src/handler/http/request/pipeline.rs delete mode 100644 src/handler/http/request/pipelines.rs create mode 100644 src/infra/src/pipeline/mod.rs create mode 100644 src/infra/src/pipeline/mysql.rs create mode 100644 src/infra/src/pipeline/postgres.rs create mode 100644 src/infra/src/pipeline/sqlite.rs create mode 100644 src/service/db/pipeline.rs delete mode 100644 src/service/db/pipelines.rs create mode 100644 src/service/pipeline/batch_execution.rs create mode 100644 src/service/pipeline/mod.rs delete mode 100644 src/service/pipelines.rs create mode 100644 web/src/assets/images/pipeline/outputStream.svg create mode 100644 web/src/assets/images/pipeline/query.svg create mode 100644 web/src/components/pipeline/FlowChart.vue rename web/src/components/pipeline/{ => NodeForm}/AssociateFunction.vue (60%) create mode 100644 web/src/components/pipeline/NodeForm/Condition.vue rename web/src/components/pipeline/{StreamRouting.vue => NodeForm/Query.vue} (50%) rename web/src/components/pipeline/{ => NodeForm}/ScheduledPipeline.vue (98%) create mode 100644 web/src/components/pipeline/NodeForm/Stream.vue create mode 100644 web/src/components/pipeline/NodeSidebar.vue create mode 100644 web/src/components/pipeline/PipelineView.vue create mode 100644 web/src/plugins/pipelines/CustomNode.vue create mode 100644 web/src/plugins/pipelines/DropzoneBackground.vue create mode 100644 web/src/plugins/pipelines/EdgeWithButton.vue create mode 100644 web/src/plugins/pipelines/PipelineFlow.vue create mode 100644 web/src/plugins/pipelines/useDnD.ts diff --git a/Cargo.lock b/Cargo.lock index f9313e8e26..235c101c52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2515,6 +2515,7 @@ dependencies = [ "serde", "serde_json", "sqlparser", + "sqlx", "svix-ksuid", "sysinfo", "tokio", @@ -2523,6 +2524,7 @@ dependencies = [ "tracing-subscriber", "urlencoding", "utoipa", + "vrl", "zstd", ] diff --git a/src/cli/basic/cli.rs b/src/cli/basic/cli.rs index a42e4f817d..dedc481db0 100644 --- a/src/cli/basic/cli.rs +++ b/src/cli/basic/cli.rs @@ -99,6 +99,14 @@ pub async fn cli() -> Result { .help("migrate to: sqlite, etcd, mysql, postgresql"), ]), clap::Command::new("migrate-dashboards").about("migrate-dashboards"), + clap::Command::new("migrate-pipeline").about("migrate pipelines") + .arg( + clap::Arg::new("drop-table") + .long("drop-table") + .required(false) + .num_args(0) + .help("Drop existing Pipeline table first before migrating") + ), clap::Command::new("delete-parquet") .about("delete parquet files from s3 and file_list") .arg( @@ -246,6 +254,11 @@ pub async fn cli() -> Result { println!("Running migration dashboard"); migration::dashboards::run().await? } + "migrate-pipeline" => { + println!("Running migration pipeline"); + let drop_table = command.get_flag("drop-table"); + migration::pipeline_func::run(drop_table).await?; + } "delete-parquet" => { let file = command.get_one::("file").unwrap(); match file_list::delete_parquet_file(file, true).await { diff --git a/src/common/infra/config.rs b/src/common/infra/config.rs index 0635ae2f1e..c1c631260f 100644 --- a/src/common/infra/config.rs +++ b/src/common/infra/config.rs @@ -15,7 +15,14 @@ use std::sync::Arc; -use config::{RwAHashMap, RwHashMap}; +use config::{ + meta::{ + alerts::{alert::Alert, destinations::Destination, templates::Template}, + function::Transform, + stream::StreamParams, + }, + RwAHashMap, RwHashMap, +}; use dashmap::DashMap; use hashbrown::HashMap; use infra::table::short_urls::ShortUrlRecord; @@ -25,18 +32,12 @@ use vector_enrichment::TableRegistry; use crate::{ common::meta::{ - alerts::{alert::Alert, destinations::Destination, templates::Template}, - dashboards::reports, - functions::{StreamFunctionsList, Transform}, - maxmind::MaxmindClient, - organization::OrganizationSetting, - pipelines::PipeLine, - prom::ClusterLeader, - syslog::SyslogRoute, - user::User, + dashboards::reports, maxmind::MaxmindClient, organization::OrganizationSetting, + prom::ClusterLeader, syslog::SyslogRoute, user::User, }, service::{ db::scheduler as db_scheduler, enrichment::StreamTable, enrichment_table::geoip::Geoip, + pipeline::batch_execution::ExecutablePipeline, }, }; @@ -47,8 +48,6 @@ pub static BUILD_DATE: &str = env!("GIT_BUILD_DATE"); // global cache variables pub static KVS: Lazy> = Lazy::new(Default::default); -pub static STREAM_FUNCTIONS: Lazy> = - Lazy::new(DashMap::default); pub static QUERY_FUNCTIONS: Lazy> = Lazy::new(DashMap::default); pub static USERS: Lazy> = Lazy::new(DashMap::default); pub static USERS_RUM_TOKEN: Lazy>> = @@ -83,6 +82,7 @@ pub static GEOIP_CITY_TABLE: Lazy>>> = pub static GEOIP_ASN_TABLE: Lazy>>> = Lazy::new(|| Arc::new(RwLock::new(None))); +pub static STREAM_EXECUTABLE_PIPELINES: Lazy> = + Lazy::new(Default::default); pub static USER_SESSIONS: Lazy> = Lazy::new(Default::default); -pub static STREAM_PIPELINES: Lazy> = Lazy::new(DashMap::default); pub static SHORT_URLS: Lazy> = Lazy::new(DashMap::default); diff --git a/src/common/meta/mod.rs b/src/common/meta/mod.rs index a93514a7fe..f64546d016 100644 --- a/src/common/meta/mod.rs +++ b/src/common/meta/mod.rs @@ -13,10 +13,8 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -pub mod alerts; pub mod authz; pub mod dashboards; -pub mod functions; pub mod http; pub mod ingestion; pub mod maxmind; diff --git a/src/common/meta/organization.rs b/src/common/meta/organization.rs index 98da0b5d78..b848d66495 100644 --- a/src/common/meta/organization.rs +++ b/src/common/meta/organization.rs @@ -13,11 +13,10 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +use config::meta::{alerts::alert::Alert, function::Transform}; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; -use super::{alerts::alert::Alert, functions::Transform}; - pub const DEFAULT_ORG: &str = "default"; pub const CUSTOM: &str = "custom"; pub const THRESHOLD: i64 = 9383939382; diff --git a/src/common/meta/pipelines.rs b/src/common/meta/pipelines.rs index 300715afad..c38343729f 100644 --- a/src/common/meta/pipelines.rs +++ b/src/common/meta/pipelines.rs @@ -16,16 +16,15 @@ use std::collections::HashMap; use config::{ - meta::stream::{RoutingCondition, StreamType}, + meta::{ + alerts::derived_streams::DerivedStreamMeta, + stream::{RoutingCondition, StreamType}, + }, utils::json::Value, }; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; -use crate::common::meta::{ - alerts::derived_streams::DerivedStreamMeta, functions::StreamFunctionsList, -}; - #[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)] pub struct PipeLine { pub name: String, @@ -42,42 +41,3 @@ pub struct PipeLine { #[serde(skip_serializing_if = "Option::is_none")] pub meta: Option>, } - -impl PipeLine { - pub fn into_response(self, functions: Option) -> PipeLineResponse { - PipeLineResponse { - name: self.name, - description: self.description, - stream_name: self.stream_name, - stream_type: self.stream_type, - routing: self.routing, - derived_streams: self.derived_streams, - functions, - meta: self.meta, - } - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)] -pub struct PipeLineResponse { - pub name: String, - #[serde(default)] - pub description: String, - #[serde(default)] - pub stream_name: String, - #[serde(default)] - pub stream_type: StreamType, - #[serde(skip_serializing_if = "Option::is_none")] - pub routing: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub functions: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub derived_streams: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub meta: Option>, -} - -#[derive(Clone, Debug, Serialize, Deserialize, ToSchema)] -pub struct PipeLineList { - pub list: Vec, -} diff --git a/src/common/meta/telemetry.rs b/src/common/meta/telemetry.rs index e56cc95808..c8279db2f0 100644 --- a/src/common/meta/telemetry.rs +++ b/src/common/meta/telemetry.rs @@ -251,12 +251,6 @@ pub async fn add_zo_info(mut data: HashMap) -> HashMap Result<(), anyhow::Error> { @@ -46,6 +47,11 @@ pub async fn check_upgrade(old_ver: &str, new_ver: &str) -> Result<(), anyhow::E upgrade_092_093().await?; } + let v131 = Version::from("v0.13.1").unwrap(); + if old_ver < v131 { + upgrade_130_131().await?; + } + Ok(()) } @@ -66,6 +72,13 @@ async fn upgrade_092_093() -> Result<(), anyhow::Error> { Ok(()) } +async fn upgrade_130_131() -> Result<(), anyhow::Error> { + // migrate pipelines and function associations + pipeline_func::run(false).await?; + + Ok(()) +} + pub async fn upgrade_resource_names() -> Result<(), anyhow::Error> { // The below migration requires ofga init ready, but on Router node, // we don't initialize ofga, hence the migration should not run on router diff --git a/src/common/migration/pipeline_func.rs b/src/common/migration/pipeline_func.rs new file mode 100644 index 0000000000..af6d34f1d3 --- /dev/null +++ b/src/common/migration/pipeline_func.rs @@ -0,0 +1,364 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::collections::HashMap; + +use config::{ + ider, + meta::{ + function, + pipeline::{ + components::{ + ConditionParams, DerivedStream, Edge, FunctionParams, Node, NodeData, + PipelineSource, + }, + Pipeline, + }, + stream::{StreamParams, StreamType}, + }, + utils::json, +}; +use infra::db as infra_db; + +pub async fn run(drop_table_first: bool) -> Result<(), anyhow::Error> { + if drop_table_first { + infra::pipeline::drop_table().await?; + } + infra::pipeline::init().await?; + migrate_pipelines().await?; + Ok(()) +} + +async fn migrate_pipelines() -> Result<(), anyhow::Error> { + let mut new_pipeline_by_source: HashMap = HashMap::new(); + let mut func_to_update = vec![]; + + // load all functions from meta table + let mut stream_funcs: HashMap> = HashMap::new(); + let db = infra_db::get_db().await; + let db_key = "/function/"; + let data = db.list(db_key).await?; + for (key, val) in data { + let local_key = key.strip_prefix('/').unwrap_or(&key); + let key_col = local_key.split('/').collect::>(); + let org_id = key_col[1]; + let mut trans: function::Transform = json::from_slice(&val).unwrap(); + if let Some(stream_orders) = trans.streams.clone() { + for stream_ord in stream_orders { + if !matches!( + stream_ord.stream_type, + StreamType::Logs | StreamType::Metrics | StreamType::Traces + ) { + continue; + } + let func_params = FunctionParams { + name: trans.name.clone(), + num_args: trans.num_args, + after_flatten: !stream_ord.apply_before_flattening, + }; + let entry = stream_funcs + .entry(StreamParams::new( + org_id, + &stream_ord.stream, + stream_ord.stream_type, + )) + .or_default(); + entry.push((stream_ord.order, func_params)); + } + trans.streams = None; + func_to_update.push((org_id.to_string(), trans)); + } + } + + // load all old pipelines from meta table + let db = infra_db::get_db().await; + let db_key = "/pipeline/"; + let data = db.list(db_key).await?; + for (key, val) in data { + let local_key = key.strip_prefix('/').unwrap_or(&key); + let key_col = local_key.split('/').collect::>(); + let old_pipe: crate::common::meta::pipelines::PipeLine = json::from_slice(&val).unwrap(); + + // two scenarios: + // scenario 1: with DerivedStream info -> scheduled + if let Some(old_derived_streams) = old_pipe.derived_streams { + for old_derived_stream in old_derived_streams { + let new_derived_stream = DerivedStream { + org_id: old_derived_stream.source.org_id.to_string(), + stream_type: old_derived_stream.source.stream_type, + query_condition: old_derived_stream.query_condition, + trigger_condition: old_derived_stream.trigger_condition, + tz_offset: old_derived_stream.tz_offset, + }; + + let pipeline_source = PipelineSource::Scheduled(new_derived_stream.clone()); + + // construct the nodes and edges lists + let source_node_data = NodeData::Query(new_derived_stream); + let dest_node_data = NodeData::Stream(old_derived_stream.destination); + let (pos_x, pos_y): (f32, f32) = (50.0, 50.0); + let pos_offset: f32 = 200.0; + let source_node = Node::new( + ider::uuid(), + source_node_data, + pos_x, + pos_y, + "input".to_string(), + ); + + let dest_node = Node::new( + ider::uuid(), + dest_node_data, + pos_x + pos_offset, + pos_y + pos_offset, + "output".to_string(), + ); + + let nodes: Vec = vec![source_node, dest_node]; + let edges = nodes + .windows(2) + .map(|pair| Edge::new(pair[0].id.clone(), pair[1].id.clone())) + .collect::>(); + let pl_id = ider::uuid(); + let name = format!("Migrated-{pl_id}"); + let description = "This pipeline was generated from previous found prior to OpenObserve v0.13.1. Please check and confirm before enabling it manually".to_string(); + let pipeline = Pipeline { + id: pl_id, + version: 0, + enabled: false, + org: old_derived_stream.source.org_id.to_string(), + name, + description, + source: pipeline_source, + nodes, + edges, + }; + new_pipeline_by_source.insert( + StreamParams::new( + &old_derived_stream.source.org_id, + &old_derived_stream.name, + old_derived_stream.source.stream_type, + ), + pipeline, + ); + } + } + + // scenario 2: with functions or routing -> realtime + let source_params = StreamParams::new(key_col[1], key_col[3], StreamType::from(key_col[2])); + if stream_funcs.contains_key(&source_params) || old_pipe.routing.is_some() { + let (mut pos_x, mut pos_y): (f32, f32) = (50.0, 50.0); + let pos_offset: f32 = 200.0; + + let new_pipeline = new_pipeline_by_source.entry(source_params.clone()).or_insert_with(|| { + let pipeline_source = PipelineSource::Realtime(source_params.clone()); + let source_node = Node::new( + ider::uuid(), + NodeData::Stream(source_params.clone()), + pos_x, + pos_y, + "input".to_string(), + ); + let pl_id = ider::uuid(); + let name = format!("Migrated-{pl_id}"); + let description = "This pipeline was generated from previous found prior to OpenObserve v0.12.2. Please check and confirm before enabling it manually".to_string(); + Pipeline { + id: pl_id, + version: 0, + enabled: false, + org: key_col[1].to_string(), + name, + description, + source: pipeline_source, + nodes: vec![source_node], + edges: vec![], + } + }); + + let source_node_id = new_pipeline.nodes[0].id.clone(); + + if let Some(mut func_params) = stream_funcs.remove(&source_params) { + let dest_node = Node::new( + ider::uuid(), + NodeData::Stream(source_params.clone()), + pos_x, + pos_y + (pos_offset * (func_params.len() + 1) as f32), + "output".to_string(), + ); + + func_params.sort_by(|a, b| a.0.cmp(&b.0)); + for (idx, (_, func_param)) in func_params.into_iter().enumerate() { + let func_node_data = NodeData::Function(func_param); + let func_node = Node::new( + ider::uuid(), + func_node_data, + pos_x, + pos_y + (pos_offset * (idx + 1) as f32), + "default".to_string(), + ); + let new_edge = Edge::new( + new_pipeline.nodes.last().unwrap().id.clone(), + func_node.id.clone(), + ); + new_pipeline.edges.push(new_edge); + new_pipeline.nodes.push(func_node); + } + let new_edge = Edge::new( + new_pipeline.nodes.last().unwrap().id.clone(), + dest_node.id.clone(), + ); + new_pipeline.edges.push(new_edge); + new_pipeline.nodes.push(dest_node); + } + + if let Some(routings) = old_pipe.routing { + pos_x += pos_offset; + for (dest_stream, routing_conditions) in routings { + pos_y += pos_offset; + let condition_node = Node::new( + ider::uuid(), + NodeData::Condition(ConditionParams { + conditions: routing_conditions, + }), + pos_x, + pos_y, + "default".to_string(), + ); + pos_y += pos_offset; + let dest_node = Node::new( + ider::uuid(), + NodeData::Stream(StreamParams::new( + key_col[1], + &dest_stream, + StreamType::from(key_col[2]), + )), + pos_x, + pos_y, + "output".to_string(), + ); + + new_pipeline + .edges + .push(Edge::new(source_node_id.clone(), condition_node.id.clone())); + new_pipeline + .edges + .push(Edge::new(condition_node.id.clone(), dest_node.id.clone())); + new_pipeline.nodes.push(condition_node); + new_pipeline.nodes.push(dest_node); + pos_y += pos_offset; + } + } + } + } + + // remaining function stream associations + for (stream_params, mut func_params) in stream_funcs { + func_params.sort_by(|a, b| a.0.cmp(&b.0)); + + let (pos_x, pos_y): (f32, f32) = (50.0, 50.0); + let pos_offset: f32 = 200.0; + let new_pipeline = new_pipeline_by_source.entry(stream_params.clone()).or_insert_with(|| { + let pipeline_source = PipelineSource::Realtime(stream_params.clone()); + let source_node = Node::new( + ider::uuid(), + NodeData::Stream(stream_params.clone()), + pos_x, + pos_y, + "input".to_string(), + ); + let pl_id = ider::uuid(); + let name = format!("Migrated-{pl_id}"); + let description = "This pipeline was generated based on Function x Stream Associations found prior to OpenObserve v0.12.2. Please check the correctness of the pipeline and enabling manually".to_string(); + Pipeline { + id: pl_id, + version: 0, + enabled: false, + org: stream_params.org_id.to_string(), + name, + description, + source: pipeline_source, + nodes: vec![source_node], + edges: vec![], + } + }); + + let dest_node = Node::new( + ider::uuid(), + NodeData::Stream(stream_params.clone()), + pos_x, + pos_y + (pos_offset * (func_params.len() + 1) as f32), + "output".to_string(), + ); + for (idx, (_, func_param)) in func_params.into_iter().enumerate() { + let func_node_data = NodeData::Function(func_param); + let func_node = Node::new( + ider::uuid(), + func_node_data, + pos_x, + pos_y + (pos_offset * (idx + 1) as f32), + "default".to_string(), + ); + let new_edge = Edge::new( + new_pipeline.nodes.last().unwrap().id.clone(), + func_node.id.clone(), + ); + new_pipeline.edges.push(new_edge); + new_pipeline.nodes.push(func_node); + } + let new_edge = Edge::new( + new_pipeline.nodes.last().unwrap().id.clone(), + dest_node.id.clone(), + ); + new_pipeline.edges.push(new_edge); + new_pipeline.nodes.push(dest_node); + } + + // save generated pipeline + let mut ok_to_remove = true; + for (_, pipeline) in new_pipeline_by_source { + if infra::pipeline::put(&pipeline).await.is_err() { + log::error!( + "[Migration]: Error migrating pipelines to the new pipeline format introduced in v0.12.2. Original data kept." + ); + ok_to_remove = false; + continue; + } + } + + if ok_to_remove { + // clear the old pipelines from the meta table + if let Err(e) = db + .delete("/pipeline/", true, infra_db::NO_NEED_WATCH, None) + .await + { + log::error!( + "[Migration-Pipeline] error deleting all pipelines from meta table: {}", + e + ); + } + // update the functions by removing the stream associations + for (org_id, trans) in func_to_update { + if let Err(e) = crate::service::db::functions::set(&org_id, &trans.name, &trans).await { + log::error!( + "[Migration-Function] error saving updated version of function {}: {}", + trans.name, + e + ); + } + } + } + + Ok(()) +} diff --git a/src/common/migration/schema.rs b/src/common/migration/schema.rs index 40d5e4a137..420be661ac 100644 --- a/src/common/migration/schema.rs +++ b/src/common/migration/schema.rs @@ -14,7 +14,13 @@ // along with this program. If not, see . use chrono::Utc; -use config::{meta::stream::StreamType, utils::json}; +use config::{ + meta::{ + alerts::{alert::Alert, destinations::Destination, templates::Template}, + stream::StreamType, + }, + utils::json, +}; use datafusion::arrow::datatypes::Schema; use infra::{ db::{self as infra_db, NO_NEED_WATCH}, @@ -32,10 +38,7 @@ use o2_enterprise::enterprise::openfga::authorizer::authz::get_ownership_tuple; use crate::{ common::{ infra::config::VERSION, - meta::{ - alerts::{alert::Alert, destinations::Destination, templates::Template}, - dashboards::reports::Report, - }, + meta::dashboards::reports::Report, utils::auth::{into_ofga_supported_format, is_ofga_unsupported}, }, service::db, diff --git a/src/common/utils/functions.rs b/src/common/utils/functions.rs index 8a8664ccf5..212e4dfd34 100644 --- a/src/common/utils/functions.rs +++ b/src/common/utils/functions.rs @@ -15,12 +15,14 @@ use std::collections::HashMap; -use config::{GEO_IP_ASN_ENRICHMENT_TABLE, GEO_IP_CITY_ENRICHMENT_TABLE}; +use config::{ + meta::function::VRLCompilerConfig, GEO_IP_ASN_ENRICHMENT_TABLE, GEO_IP_CITY_ENRICHMENT_TABLE, +}; use vector_enrichment::{Table, TableRegistry}; use crate::common::{ infra::config::{ENRICHMENT_TABLES, GEOIP_ASN_TABLE, GEOIP_CITY_TABLE}, - meta::{functions::VRLCompilerConfig, organization::DEFAULT_ORG}, + meta::organization::DEFAULT_ORG, }; pub async fn get_all_transform_keys(org_id: &str) -> Vec { diff --git a/src/config/Cargo.toml b/src/config/Cargo.toml index 9e18bf55b4..76478f549e 100644 --- a/src/config/Cargo.toml +++ b/src/config/Cargo.toml @@ -59,6 +59,7 @@ serde.workspace = true serde_json.workspace = true segment.workspace = true sqlparser.workspace = true +sqlx.workspace = true svix-ksuid.workspace = true sysinfo.workspace = true tokio.workspace = true @@ -67,4 +68,5 @@ tracing-log.workspace = true tracing-subscriber.workspace = true urlencoding.workspace = true utoipa.workspace = true +vrl.workspace = true zstd.workspace = true diff --git a/src/common/meta/alerts/alert.rs b/src/config/src/meta/alerts/alert.rs similarity index 97% rename from src/common/meta/alerts/alert.rs rename to src/config/src/meta/alerts/alert.rs index 5371d6096a..ea27dacee5 100644 --- a/src/common/meta/alerts/alert.rs +++ b/src/config/src/meta/alerts/alert.rs @@ -14,12 +14,14 @@ // along with this program. If not, see . use chrono::{DateTime, FixedOffset}; -use config::meta::stream::StreamType; use hashbrown::HashMap; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; -use super::{QueryCondition, TriggerCondition}; +use crate::meta::{ + alerts::{QueryCondition, TriggerCondition}, + stream::StreamType, +}; #[derive(Clone, Debug, Serialize, Deserialize, ToSchema)] pub struct Alert { diff --git a/src/common/meta/alerts/derived_streams.rs b/src/config/src/meta/alerts/derived_streams.rs similarity index 95% rename from src/common/meta/alerts/derived_streams.rs rename to src/config/src/meta/alerts/derived_streams.rs index 328c3f53cb..3ff0832536 100644 --- a/src/common/meta/alerts/derived_streams.rs +++ b/src/config/src/meta/alerts/derived_streams.rs @@ -13,12 +13,14 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use config::meta::stream::StreamParams; use hashbrown::HashMap; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; -use crate::common::meta::alerts::{QueryCondition, TriggerCondition}; +use crate::meta::{ + alerts::{QueryCondition, TriggerCondition}, + stream::StreamParams, +}; #[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)] pub struct DerivedStreamMeta { diff --git a/src/common/meta/alerts/destinations.rs b/src/config/src/meta/alerts/destinations.rs similarity index 100% rename from src/common/meta/alerts/destinations.rs rename to src/config/src/meta/alerts/destinations.rs diff --git a/src/common/meta/alerts/mod.rs b/src/config/src/meta/alerts/mod.rs similarity index 99% rename from src/common/meta/alerts/mod.rs rename to src/config/src/meta/alerts/mod.rs index abd41af612..0c516ae99d 100644 --- a/src/common/meta/alerts/mod.rs +++ b/src/config/src/meta/alerts/mod.rs @@ -13,10 +13,11 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use config::{meta::search::SearchEventType, utils::json::Value}; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; +use crate::{meta::search::SearchEventType, utils::json::Value}; + pub mod alert; pub mod derived_streams; pub mod destinations; diff --git a/src/common/meta/alerts/templates.rs b/src/config/src/meta/alerts/templates.rs similarity index 100% rename from src/common/meta/alerts/templates.rs rename to src/config/src/meta/alerts/templates.rs diff --git a/src/common/meta/functions.rs b/src/config/src/meta/function.rs similarity index 73% rename from src/common/meta/functions.rs rename to src/config/src/meta/function.rs index c9b8ad0d45..f9ad55ee2a 100644 --- a/src/common/meta/functions.rs +++ b/src/config/src/meta/function.rs @@ -13,7 +13,6 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use config::meta::stream::StreamType; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use vrl::{ @@ -21,6 +20,8 @@ use vrl::{ prelude::Function, }; +use crate::meta::stream::StreamType; + #[derive(Clone, Debug, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct Transform { @@ -53,52 +54,6 @@ pub struct StreamOrder { pub apply_before_flattening: bool, } -#[derive(Clone, Debug, Serialize, Deserialize, ToSchema)] -pub struct StreamTransform { - #[serde(flatten)] - pub transform: Transform, - #[serde(default)] - pub stream: String, - #[serde(default)] - pub order: u8, - #[serde(default)] - pub stream_type: StreamType, - #[serde(default)] - pub is_removed: bool, - #[serde(default)] - #[serde(rename = "applyBeforeFlattening")] - pub apply_before_flattening: bool, -} - -impl PartialEq for StreamTransform { - fn eq(&self, other: &Self) -> bool { - self.stream == other.stream - && self.transform.name == other.transform.name - && self.stream_type == other.stream_type - } -} - -impl Transform { - pub fn to_stream_transform(&self) -> Vec { - let mut ret: Vec = vec![]; - if let Some(streams) = &self.streams { - let mut func = self.clone(); - func.streams = None; - for stream in streams { - ret.push(StreamTransform { - transform: func.clone(), - stream: stream.stream.clone(), - order: stream.order, - stream_type: stream.stream_type, - is_removed: stream.is_removed, - apply_before_flattening: stream.apply_before_flattening, - }) - } - } - ret - } -} - impl PartialEq for Transform { fn eq(&self, other: &Self) -> bool { self.name == other.name && self.function == other.function && self.params == other.params @@ -115,11 +70,6 @@ pub struct FunctionList { pub list: Vec, } -#[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)] -pub struct StreamFunctionsList { - pub list: Vec, -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct VRLConfig { pub runtime: VrlRuntime, @@ -141,6 +91,7 @@ pub struct VRLRuntimeConfig { pub fields: Vec, } +#[derive(Clone, Debug)] pub struct VRLResultResolver { pub program: Program, pub fields: Vec, @@ -148,9 +99,8 @@ pub struct VRLResultResolver { #[cfg(test)] mod tests { - use config::utils::json; - use super::*; + use crate::utils::json; #[test] fn test_functions() { diff --git a/src/config/src/meta/mod.rs b/src/config/src/meta/mod.rs index 9bddf273d4..547ce765ef 100644 --- a/src/config/src/meta/mod.rs +++ b/src/config/src/meta/mod.rs @@ -13,11 +13,14 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +pub mod alerts; pub mod bitvec; pub mod cluster; +pub mod function; pub mod inverted_index; pub mod logger; pub mod meta_store; +pub mod pipeline; pub mod puffin; pub mod search; pub mod short_url; diff --git a/src/config/src/meta/pipeline/components.rs b/src/config/src/meta/pipeline/components.rs new file mode 100644 index 0000000000..5fa5da39ae --- /dev/null +++ b/src/config/src/meta/pipeline/components.rs @@ -0,0 +1,238 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +use crate::meta::{ + alerts::{QueryCondition, TriggerCondition}, + stream::{RoutingCondition, StreamParams, StreamType}, +}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "source_type")] +#[serde(rename_all = "snake_case")] +pub enum PipelineSource { + Realtime(StreamParams), + Scheduled(DerivedStream), +} + +impl Default for PipelineSource { + fn default() -> Self { + Self::Realtime(StreamParams::default()) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Default)] +#[serde(rename_all = "snake_case")] +#[serde(default)] +pub struct DerivedStream { + #[serde(default)] + pub org_id: String, + pub stream_type: StreamType, + #[serde(default)] + pub query_condition: QueryCondition, + #[serde(default)] + pub trigger_condition: TriggerCondition, // Frequency type only supports minutes + /// Timezone offset in minutes. + /// The negative secs means the Western Hemisphere + #[serde(default)] + pub tz_offset: i32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct Node { + pub id: String, + pub data: NodeData, + position: Position, + io_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + style: Option, +} + +impl PartialEq for Node { + fn eq(&self, other: &Self) -> bool { + self.id == other.id && self.data == other.data && self.position == other.position + } +} + +impl Node { + pub fn new(id: String, data: NodeData, pos_x: f32, pos_y: f32, io_type: String) -> Self { + Self { + id, + data, + position: Position { x: pos_x, y: pos_y }, + io_type, + style: None, + } + } + + pub fn get_node_data(&self) -> NodeData { + self.data.clone() + } + + pub fn get_node_id(&self) -> String { + self.id.clone() + } + + pub fn is_function_node(&self) -> bool { + matches!(&self.data, NodeData::Function(_)) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Edge { + pub id: String, + pub source: String, + pub target: String, +} + +impl Edge { + pub fn new(source: String, target: String) -> Self { + let id = format!("e{source}-{target}"); + Self { id, source, target } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "node_type")] +#[serde(rename_all = "snake_case")] +pub enum NodeData { + Stream(StreamParams), + Query(DerivedStream), + Function(FunctionParams), + Condition(ConditionParams), +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSchema)] +#[serde(rename_all = "snake_case")] +pub struct FunctionParams { + #[serde(default)] + pub name: String, + #[serde(default)] + pub after_flatten: bool, + #[serde(default)] + pub num_args: u8, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ConditionParams { + pub conditions: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +struct Position { + x: f32, + y: f32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct NodeStyle { + background_color: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{meta::stream::StreamType, utils::json}; + + #[test] + fn test_pipeline_source_serialization() { + let data = json::json!({ + "source_type": "realtime", + "org_id": "default", + "stream_name": "default", + "stream_type": "logs" + }); + let from_json: PipelineSource = json::from_value(data).unwrap(); + let stream_params = StreamParams { + org_id: "default".into(), + stream_name: "default".into(), + stream_type: StreamType::Logs, + }; + let source = PipelineSource::Realtime(stream_params); + assert_eq!(from_json, source); + } + + #[test] + fn test_pipeline_empty_source_serialization() { + let data = json::json!({ + "source_type": "scheduled", + }); + let from_json: PipelineSource = json::from_value(data).unwrap(); + println!("result: {:?}", from_json); + let stream_params = DerivedStream::default(); + let source = PipelineSource::Scheduled(stream_params); + assert_eq!(from_json, source); + } + + #[test] + fn test_node_data_serialization() { + let data = json::json!({ + "node_type": "stream", + "org_id": "default", + "stream_name": "default", + "stream_type": "logs" + }); + let node: NodeData = json::from_value(data).unwrap(); + let node_data = NodeData::Stream(StreamParams { + org_id: "default".into(), + stream_name: "default".into(), + stream_type: StreamType::Logs, + }); + assert_eq!(node_data, node); + } + + #[test] + fn test_function_node_serialization() { + let func = FunctionParams { + // vrl_script: "vrl_script".to_string(), + name: "func".to_string(), + after_flatten: false, + // params: "row".to_string(), + num_args: 0, + }; + let func_node = NodeData::Function(func); + let payload = json::json!({ + "node_type": "function", + "name": "func", + "after_flatten": false, + }); + let node_data: NodeData = json::from_value(payload).unwrap(); + assert_eq!(func_node, node_data); + } + + #[test] + fn test_condition_node_serialization() { + let payload = json::json!({ + "node_type": "condition", // required + "conditions": [ // required + { + "column": "body", + "operator": ">=", + "value": { + "key": "value" + }, + "ignore_case": false // optional + } + ] + }); + + let node_data = json::from_value::(payload); + assert!(node_data.is_ok()); + } +} diff --git a/src/config/src/meta/pipeline/mod.rs b/src/config/src/meta/pipeline/mod.rs new file mode 100644 index 0000000000..2498747167 --- /dev/null +++ b/src/config/src/meta/pipeline/mod.rs @@ -0,0 +1,486 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::collections::{HashMap, HashSet}; + +use anyhow::{anyhow, Result}; +use components::{DerivedStream, Edge, Node, NodeData, PipelineSource}; +use serde::{Deserialize, Serialize}; +use sqlx::{Decode, Error, FromRow, Row, Type}; +use utoipa::ToSchema; + +use crate::{ + meta::{ + function::VRLResultResolver, + stream::{StreamParams, StreamType}, + }, + utils::json, +}; + +pub mod components; + +// (pipeline, node_map, graph, vrl_map) +pub type PipelineExecDFS = ( + Pipeline, + HashMap, + HashMap>, + HashMap, +); + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSchema)] +pub struct Pipeline { + #[serde(default)] + #[serde(rename = "pipeline_id")] + pub id: String, + #[serde(default)] + pub version: i32, + #[serde(default = "default_status")] + pub enabled: bool, + #[serde(default)] + pub org: String, // org this pipeline belongs to. diff from source stream org_id + pub name: String, + #[serde(default)] + pub description: String, + #[serde(default)] + pub source: PipelineSource, + pub nodes: Vec, + pub edges: Vec, +} + +impl Pipeline { + pub fn get_cache_key(&self) -> String { + match &self.source { + PipelineSource::Realtime(stream_params) => { + format!("{}/{}", self.org, stream_params) + } + PipelineSource::Scheduled(_) => { + format!("{}/{}", self.org, self.id) + } + } + } + + pub fn get_derived_stream(&self) -> Option { + match &self.source { + PipelineSource::Scheduled(derived_stream) => Some(derived_stream.to_owned()), + _ => None, + } + } + + /// Verifies the pipeline is valid by: + /// 1. non-empty nodes list + /// 2. non-empty edges list + /// 3. 1st node in nodes list is either StreamNode or QueryNode + /// 4. non-empty `conditions` in all ConditionNode nodes in nodes list + /// 5. every node is reachable + /// 6. all leaf nodes are of type StreamNode + /// 7. In the same branch, unchecked `after_flattened` FunctionNode can't follow checked + /// `after_flattened` checked FunctionNode + /// + /// If all satisfies, populates the [Pipeline::source] with the first node in nodes list + pub fn validate(&mut self) -> Result<()> { + // ck 1 & 2 + match (self.nodes.is_empty(), self.edges.is_empty()) { + (true, true) | (true, false) => { + return Err(anyhow!( + "Empty pipeline. Please add Source/Destination nodes, or any applicable Transform Nodes" + )); + } + (false, true) => { + return Err(anyhow!( + "Please connect all Nodes to complete pipeline creation" + )); + } + _ => {} + }; + + // ck 3 + match self.nodes.first().unwrap().get_node_data() { + NodeData::Stream(stream_params) => { + self.source = PipelineSource::Realtime(stream_params); + } + NodeData::Query(derived_stream) => { + if derived_stream.trigger_condition.period == 0 { + return Err(anyhow!( + "DerivedStream source's TriggerCondition period missing or is 0" + )); + } + self.source = PipelineSource::Scheduled(derived_stream); + } + _ => return Err(anyhow!("Source must be either a StreamNode or QueryNode")), + }; + + // ck 4 + if self.nodes.iter().any(|node| { + matches!(node.get_node_data(), NodeData::Condition(condition_params) if condition_params.conditions.is_empty()) + }) { + return Err(anyhow!("ConditionNode must have non-empty conditions")); + } + + // ck 5 + if self.edges.len() < self.nodes.len() - 1 { + return Err(anyhow!( + "Insufficient number of edges to connect all nodes. Need at least {} for {} nodes, but got {}.", + self.nodes.len() - 1, + self.nodes.len(), + self.edges.len() + )); + } + + // build adjacency list for ck 6 & 7 + let source_node_id = self.nodes[0].id.as_str(); + let node_map = self.get_node_map(); + let adjacency_list = self.build_adjacency_list(&node_map)?; + let mut visited = HashSet::new(); + dfs_traversal_check( + source_node_id, + &adjacency_list, + &node_map, + false, + &mut visited, + )?; + + Ok(()) + } + + /// Converts pipeline's node list to a map for quick lookup + pub fn get_node_map(&self) -> HashMap { + self.nodes + .iter() + .map(|node| (node.get_node_id(), node.get_node_data())) + .collect() + } + + /// Builds the graph representation of this pipeline's nodes structure. + /// + /// Used for pipeline validation and execution. + pub fn build_adjacency_list( + &self, + node_map: &HashMap, + ) -> Result>> { + let mut adjacency_list = HashMap::new(); + + for (idx, edge) in self.edges.iter().enumerate() { + if !node_map.contains_key(&edge.source) { + return Err(anyhow!("Edge #{idx}'s source node not found in nodes list")); + } + if !node_map.contains_key(&edge.target) { + return Err(anyhow!("Edge #{idx}'s target node not found in nodes list")); + } + adjacency_list + .entry(edge.source.clone()) + .or_insert_with(Vec::new) + .push(edge.target.clone()); + } + + Ok(adjacency_list) + } + + /// Finds all the destination streams in the pipeline. + pub fn get_all_destination_streams( + &self, + node_map: &HashMap, + graph: &HashMap>, + ) -> Vec { + node_map + .iter() + .filter_map(|(id, node_data)| { + if !graph.contains_key(id) { + if let NodeData::Stream(stream_params) = node_data { + Some(stream_params.clone()) + } else { + None + } + } else { + None + } + }) + .collect() + } + + /// Returns the number of functions nodes in this pipeline. + /// + /// Used for usage report. + pub fn num_of_func(&self) -> usize { + self.nodes + .iter() + .filter(|node| node.is_function_node()) + .count() + } +} + +impl<'r, R: Row> FromRow<'r, R> for Pipeline +where + &'r ::std::primitive::str: ::sqlx::ColumnIndex, + String: Type + Decode<'r, R::Database>, + i32: Type + Decode<'r, R::Database>, + bool: Type + Decode<'r, R::Database>, +{ + fn from_row(row: &'r R) -> Result { + let id: String = row.try_get("id")?; + let version: i32 = row.try_get("version")?; + let enabled: bool = row.try_get("enabled")?; + let org: String = row.try_get("org")?; + let name: String = row.try_get("name")?; + let description: String = row.try_get("description")?; + let source_type: String = row.try_get("source_type")?; + + let source = match source_type.as_str() { + "realtime" => { + let stream_org: String = row.try_get("stream_org")?; + let stream_name: String = row.try_get("stream_name")?; + let stream_type: String = row.try_get("stream_type")?; + let stream_params = StreamParams::new( + &stream_org, + &stream_name, + StreamType::from(stream_type.as_str()), + ); + PipelineSource::Realtime(stream_params) + } + "scheduled" => { + let derived_stream_raw: String = row.try_get("derived_stream")?; + let derived_stream: DerivedStream = json::from_str(&derived_stream_raw) + .expect("Deserializing DerivedStream from ROW error"); + PipelineSource::Scheduled(derived_stream) + } + _ => return Err(sqlx::Error::ColumnNotFound("Invalid source type".into())), + }; + + let (nodes, edges) = { + let nodes_raw: String = row.try_get("nodes")?; + let edges_raw: String = row.try_get("edges")?; + ( + json::from_str(&nodes_raw).expect("Deserializing Nodes from ROW error"), + json::from_str(&edges_raw).expect("Deserializing Edges from ROW error"), + ) + }; + + Ok(Pipeline { + id, + version, + enabled, + org, + name, + description, + source, + nodes, + edges, + }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, ToSchema)] +pub struct PipelineList { + pub list: Vec, +} + +/// DFS traversal to check: +/// 1. all leaf nodes are of StreamNode +/// 2. No `After Flattened` unchecked FunctionNode follows `After Flatten` checked FunctionNode in +/// the same branch +/// +/// `graph` and `node_map` already validated for unwrap +fn dfs_traversal_check( + current_id: &str, + graph: &HashMap>, + node_map: &HashMap, + mut flattened: bool, + visited: &mut HashSet, +) -> Result<()> { + if visited.contains(current_id) { + return Err(anyhow!("Cyclical pipeline detected.")); + } + visited.insert(current_id.to_string()); + // Check if the current node is a leaf node + if !graph.contains_key(current_id) { + // Ensure leaf nodes are Stream nodes + if let Some(node_data) = node_map.get(current_id) { + if !matches!(node_data, NodeData::Stream(_)) { + return Err(anyhow!("All leaf nodes must be StreamNode")); + } + } else { + return Err(anyhow!("Node with id {} not found in node_map", current_id)); + } + visited.remove(current_id); + return Ok(()); + } + + for next_node_id in graph.get(current_id).unwrap() { + if let NodeData::Function(func_params) = &node_map.get(next_node_id).unwrap() { + if flattened && !func_params.after_flatten { + return Err(anyhow!( + "After Flatten must be checked if a previous FunctionNode already checked it in the same branch." + )); + } + flattened |= func_params.after_flatten; + }; + dfs_traversal_check(next_node_id, graph, node_map, flattened, visited)?; + } + visited.remove(current_id); + + Ok(()) +} + +fn default_status() -> bool { + true +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::utils::json; + + #[test] + fn test_stream_pipeline_serialization() { + let payload = json::json!( + { + "pipeline_id": "uuid", + "name": "pipeline test", + "description": "with or without this field", + "org": "prod", + "nodes": [ + { + "id": "1", + "data": { + "node_type": "stream", + "org_id": "default", + "stream_name": "default", + "stream_type": "logs" + }, + "position": { + "x": 100, + "y": 100 + }, + "style": { + "backgroundColor": "#f0f0f0" + }, + "io_type": "input", + }, + { + "id": "2", + "data": { + "node_type": "stream", + "org_id": "default", + "stream_name": "default", + "stream_type": "logs" + }, + "io_type": "input", + "position": { + "x": 300, + "y": 100 + } + } + ], + "edges": [ + { + "id": "e1-2", + "source": "1", + "target": "2" + } + ] + } + ); + let from_value = json::from_value::(payload); + assert!(from_value.is_ok()); + let mut pl = from_value.unwrap(); + assert!(pl.build_adjacency_list(&pl.get_node_map()).is_ok()); + assert!(pl.validate().is_ok()); + let nodes = json::to_string(&pl.nodes); + assert!(nodes.is_ok()); + let nodes = nodes.unwrap(); + let new_nodes = json::from_str::>>(&nodes); + assert!(new_nodes.is_ok()); + } + + #[test] + fn test_query_pipeline_serialization() { + let payload = json::json!( + { + "pipeline_id": "uuid", + "name": "pipeline test", + "description": "with or without this field", + "org": "prod", + "nodes": [ + { + "id": "1", + "data": { + "node_type": "query", + "org_id": "default", + "stream_type": "logs", + "query_condition": { + "type": "sql", + "conditions": null, + "sql": "SELECT min(_timestamp) min_ts,max(_timestamp) max_ts, count(_timestamp) as counts FROM \"default1\" ", + "promql": null, + "promql_condition": null, + "aggregation": null, + "vrl_function": null, + "search_event_type": "DerivedStream" + }, + "trigger_condition": { + "period": 60, + "operator": "=", + "threshold": 0, + "frequency": 15, + "cron": "10 56 * * * *", + "frequency_type": "cron", + "silence": 0 + }, + "tz_offset": 0 + }, + "position": { + "x": 100, + "y": 100 + }, + "io_type": "input", + "style": { + "backgroundColor": "#f0f0f0" + } + }, + { + "id": "2", + "data": { + "node_type": "stream", + "org_id": "default", + "stream_name": "default", + "stream_type": "logs" + }, + "io_type": "output", + "position": { + "x": 300, + "y": 100 + }, + } + ], + "edges": [ + { + "id": "e1-2", + "source": "1", + "target": "2" + } + ] + } + ); + let from_value = json::from_value::(payload); + assert!(from_value.is_ok()); + let mut pl = from_value.unwrap(); + assert!(pl.build_adjacency_list(&pl.get_node_map()).is_ok()); + assert!(pl.validate().is_ok()); + let nodes = json::to_string(&pl.nodes); + assert!(nodes.is_ok()); + let nodes = nodes.unwrap(); + let new_nodes = json::from_str::>>(&nodes); + assert!(new_nodes.is_ok()); + } +} diff --git a/src/config/src/meta/stream.rs b/src/config/src/meta/stream.rs index 5820bcf228..49fb43666e 100644 --- a/src/config/src/meta/stream.rs +++ b/src/config/src/meta/stream.rs @@ -63,6 +63,18 @@ impl StreamType { StreamType::Logs | StreamType::Metrics | StreamType::Traces ) } + + pub fn as_str(&self) -> &str { + match self { + StreamType::Logs => "logs", + StreamType::Metrics => "metrics", + StreamType::Traces => "traces", + StreamType::EnrichmentTables => "enrichment_tables", + StreamType::Filelist => "file_list", + StreamType::Metadata => "metadata", + StreamType::Index => "index", + } + } } impl From<&str> for StreamType { @@ -108,6 +120,53 @@ impl std::fmt::Display for StreamType { } } +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[serde(default)] +pub struct StreamParams { + pub org_id: faststr::FastStr, + pub stream_name: faststr::FastStr, + pub stream_type: StreamType, +} + +impl Default for StreamParams { + fn default() -> Self { + Self { + org_id: String::default().into(), + stream_name: String::default().into(), + stream_type: StreamType::default(), + } + } +} + +impl std::fmt::Display for StreamParams { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}/{}/{}", + self.org_id, self.stream_name, self.stream_type + ) + } +} + +impl StreamParams { + pub fn new(org_id: &str, stream_name: &str, stream_type: StreamType) -> Self { + Self { + org_id: org_id.to_string().into(), + stream_name: stream_name.to_string().into(), + stream_type, + } + } + + pub fn is_valid(&self) -> bool { + !(self.org_id.is_empty() || self.stream_name.is_empty()) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, ToSchema)] +pub struct ListStreamParams { + pub list: Vec, +} + #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub struct FileKey { pub key: String, @@ -769,11 +828,6 @@ pub struct PartitioningDetails { pub partition_time_level: Option, } -pub struct Routing { - pub destination: String, - pub routing: Vec, -} - // Code Duplicated from alerts #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] pub struct RoutingCondition { @@ -786,7 +840,7 @@ pub struct RoutingCondition { } // Code Duplicated from alerts impl RoutingCondition { - pub async fn evaluate(&self, row: &Map) -> bool { + pub fn evaluate(&self, row: &Map) -> bool { let val = match row.get(&self.column) { Some(val) => val, None => { @@ -890,45 +944,6 @@ impl std::fmt::Display for Operator { } } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct StreamParams { - pub org_id: faststr::FastStr, - pub stream_name: faststr::FastStr, - pub stream_type: StreamType, -} - -impl PartialEq for StreamParams { - fn eq(&self, other: &Self) -> bool { - self.org_id == other.org_id - && self.stream_name == other.stream_name - && self.stream_type == other.stream_type - } -} - -impl Default for StreamParams { - fn default() -> Self { - Self { - org_id: String::default().into(), - stream_name: String::default().into(), - stream_type: StreamType::default(), - } - } -} - -impl StreamParams { - pub fn new(org_id: &str, stream_name: &str, stream_type: StreamType) -> Self { - Self { - org_id: org_id.to_string().into(), - stream_name: stream_name.to_string().into(), - stream_type, - } - } - - pub fn is_valid(&self) -> bool { - !(self.org_id.is_empty() || self.stream_name.is_empty()) - } -} - #[cfg(test)] mod tests { use super::*; @@ -1010,8 +1025,15 @@ mod tests { #[test] fn test_stream_params() { let params = StreamParams::new("org_id", "stream_name", StreamType::Logs); + let param2 = StreamParams::new("org_id", "stream_name", StreamType::Logs); + let param3 = StreamParams::new("org_id", "stream_name", StreamType::Index); assert_eq!(params.org_id, "org_id"); assert_eq!(params.stream_name, "stream_name"); assert_eq!(params.stream_type, StreamType::Logs); + let mut map = HashMap::new(); + map.insert(params, 1); + map.insert(param2, 2); + map.insert(param3, 2); + assert_eq!(map.len(), 2); } } diff --git a/src/config/src/utils/flatten.rs b/src/config/src/utils/flatten.rs index 9f9beb605c..fc7162f89f 100644 --- a/src/config/src/utils/flatten.rs +++ b/src/config/src/utils/flatten.rs @@ -25,7 +25,7 @@ pub fn flatten(to_flatten: Value) -> Result { /// Flattens the provided JSON object (`current`). /// /// It will return an error if flattening the object would make two keys to be -/// the same, overwriting a value. It will alre return an error if the JSON +/// the same, overwriting a value. It will also return an error if the JSON /// value passed it's not an object. /// /// # Errors diff --git a/src/handler/http/request/alerts/alert.rs b/src/handler/http/request/alerts/alert.rs index 46cf574037..18eff126df 100644 --- a/src/handler/http/request/alerts/alert.rs +++ b/src/handler/http/request/alerts/alert.rs @@ -16,14 +16,11 @@ use std::{collections::HashMap, io::Error}; use actix_web::{delete, get, http, post, put, web, HttpRequest, HttpResponse}; +use config::meta::alerts::alert::{Alert, AlertListFilter}; use crate::{ common::{ - meta::{ - alerts::alert::{Alert, AlertListFilter}, - dashboards::datetime_now, - http::HttpResponse as MetaHttpResponse, - }, + meta::{dashboards::datetime_now, http::HttpResponse as MetaHttpResponse}, utils::{auth::UserEmail, http::get_stream_type_from_request}, }, service::alerts::alert, diff --git a/src/handler/http/request/alerts/destinations.rs b/src/handler/http/request/alerts/destinations.rs index dbbfe1a210..acb6342e06 100644 --- a/src/handler/http/request/alerts/destinations.rs +++ b/src/handler/http/request/alerts/destinations.rs @@ -16,11 +16,9 @@ use std::io::Error; use actix_web::{delete, get, http, post, put, web, HttpRequest, HttpResponse}; +use config::meta::alerts::destinations::Destination; -use crate::{ - common::meta::{alerts::destinations::Destination, http::HttpResponse as MetaHttpResponse}, - service::alerts::destinations, -}; +use crate::{common::meta::http::HttpResponse as MetaHttpResponse, service::alerts::destinations}; /// CreateDestination #[utoipa::path( diff --git a/src/handler/http/request/alerts/templates.rs b/src/handler/http/request/alerts/templates.rs index 6aecac7885..2e765641d6 100644 --- a/src/handler/http/request/alerts/templates.rs +++ b/src/handler/http/request/alerts/templates.rs @@ -16,11 +16,9 @@ use std::io::Error; use actix_web::{delete, get, http, post, put, web, HttpRequest, HttpResponse}; +use config::meta::alerts::templates::Template; -use crate::{ - common::meta::{alerts::templates::Template, http::HttpResponse as MetaHttpResponse}, - service::alerts::templates, -}; +use crate::{common::meta::http::HttpResponse as MetaHttpResponse, service::alerts::templates}; /// CreateTemplate #[utoipa::path( diff --git a/src/handler/http/request/functions/mod.rs b/src/handler/http/request/functions/mod.rs index 2e00174b6a..ae98b3c6a1 100644 --- a/src/handler/http/request/functions/mod.rs +++ b/src/handler/http/request/functions/mod.rs @@ -13,15 +13,10 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{collections::HashMap, io::Error}; +use std::io::Error; -use actix_web::{delete, get, http, post, put, web, HttpRequest, HttpResponse}; - -use crate::common::{ - meta, - meta::functions::{StreamOrder, Transform}, - utils::http::get_stream_type_from_request, -}; +use actix_web::{delete, get, post, put, web, HttpRequest, HttpResponse}; +use config::meta::function::Transform; /// CreateFunction #[utoipa::path( @@ -153,128 +148,3 @@ pub async fn update_function( transform.function = transform.function.trim().to_string(); crate::service::functions::update_function(&org_id, name, transform).await } - -/// ListStreamFunctions -#[utoipa::path( - context_path = "/api", - tag = "Functions", - operation_id = "listStreamFunctions", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ("stream_name" = String, Path, description = "Stream name"), - ), - responses( - (status = 200, description = "Success", content_type = "application/json", body = StreamFunctionsList), - ) -)] -#[get("/{org_id}/streams/{stream_name}/functions")] -async fn list_stream_functions( - path: web::Path<(String, String)>, - req: HttpRequest, -) -> Result { - let (org_id, stream_name) = path.into_inner(); - let query = web::Query::>::from_query(req.query_string()).unwrap(); - let stream_type = match get_stream_type_from_request(&query) { - Ok(v) => v.unwrap_or_default(), - Err(e) => { - return Ok( - HttpResponse::BadRequest().json(meta::http::HttpResponse::error( - http::StatusCode::BAD_REQUEST.into(), - e.to_string(), - )), - ); - } - }; - crate::service::functions::list_stream_functions(&org_id, stream_type, &stream_name).await -} - -/// RemoveStreamFunction -#[utoipa::path( - context_path = "/api", - tag = "Functions", - operation_id = "removeStreamFunction", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ("stream_name" = String, Path, description = "Stream name"), - ("name" = String, Path, description = "Function name"), - ), - responses( - (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), - (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse), - ) -)] -#[delete("/{org_id}/streams/{stream_name}/functions/{name}")] -async fn delete_stream_function( - path: web::Path<(String, String, String)>, - req: HttpRequest, -) -> Result { - let (org_id, stream_name, name) = path.into_inner(); - let query = web::Query::>::from_query(req.query_string()).unwrap(); - let stream_type = match get_stream_type_from_request(&query) { - Ok(v) => v.unwrap_or_default(), - Err(e) => { - return Ok( - HttpResponse::BadRequest().json(meta::http::HttpResponse::error( - http::StatusCode::BAD_REQUEST.into(), - e.to_string(), - )), - ); - } - }; - crate::service::functions::delete_stream_function(&org_id, stream_type, &stream_name, &name) - .await -} - -/// ApplyFunctionToStream -#[utoipa::path( - context_path = "/api", - tag = "Functions", - operation_id = "applyFunctionToStream", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ("stream_name" = String, Path, description = "Stream name"), - ("name" = String, Path, description = "Function name"), - ), - request_body(content = StreamOrder, description = "Function data", content_type = "application/json"), - responses( - (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), - (status = 400, description = "Failure", content_type = "application/json", body = HttpResponse), - ) -)] -#[put("/{org_id}/streams/{stream_name}/functions/{name}")] -pub async fn add_function_to_stream( - path: web::Path<(String, String, String)>, - stream_order: web::Json, - req: HttpRequest, -) -> Result { - let (org_id, stream_name, name) = path.into_inner(); - let query = web::Query::>::from_query(req.query_string()).unwrap(); - let stream_type = match get_stream_type_from_request(&query) { - Ok(v) => v.unwrap_or_default(), - Err(e) => { - return Ok( - HttpResponse::BadRequest().json(meta::http::HttpResponse::error( - http::StatusCode::BAD_REQUEST.into(), - e.to_string(), - )), - ); - } - }; - crate::service::functions::add_function_to_stream( - &org_id, - stream_type, - &stream_name, - &name, - stream_order.into_inner(), - ) - .await -} diff --git a/src/handler/http/request/mod.rs b/src/handler/http/request/mod.rs index 02958583c9..b3e892f819 100644 --- a/src/handler/http/request/mod.rs +++ b/src/handler/http/request/mod.rs @@ -23,7 +23,7 @@ pub mod kv; pub mod logs; pub mod metrics; pub mod organization; -pub mod pipelines; +pub mod pipeline; pub mod prom; pub mod rum; pub mod search; diff --git a/src/handler/http/request/pipeline.rs b/src/handler/http/request/pipeline.rs new file mode 100644 index 0000000000..fa2a025e83 --- /dev/null +++ b/src/handler/http/request/pipeline.rs @@ -0,0 +1,208 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::io::Error; + +use actix_web::{delete, get, post, put, web, HttpRequest, HttpResponse}; +use ahash::HashMap; +use config::{ider, meta::pipeline::Pipeline}; + +/// CreatePipeline +#[utoipa::path( + context_path = "/api", + tag = "Pipeline", + operation_id = "createPipeline", + security( + ("Authorization"= []) + ), + params( + ("org_id" = String, Path, description = "Organization name"), + ), + request_body(content = Pipeline, description = "Pipeline data", content_type = "application/json"), + responses( + (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), + (status = 400, description = "Failure", content_type = "application/json", body = HttpResponse), + ) +)] +#[post("/{org_id}/pipelines")] +pub async fn save_pipeline( + path: web::Path, + pipeline: web::Json, + _req: HttpRequest, +) -> Result { + let org_id = path.into_inner(); + let mut pipeline = pipeline.into_inner(); + pipeline.name = pipeline.name.trim().to_string(); + pipeline.org = org_id; + pipeline.id = ider::generate(); + crate::service::pipeline::save_pipeline(pipeline).await +} + +/// ListPipelines +#[utoipa::path( + context_path = "/api", + tag = "Pipelines", + operation_id = "listPipelines", + security( + ("Authorization"= []) + ), + params( + ("org_id" = String, Path, description = "Organization name"), + ), + responses( + (status = 200, description = "Success", content_type = "application/json", body = PipelineList), + ) +)] +#[get("/{org_id}/pipelines")] +async fn list_pipelines( + org_id: web::Path, + _req: HttpRequest, +) -> Result { + let mut _permitted = None; + // Get List of allowed objects + #[cfg(feature = "enterprise")] + { + let user_id = _req.headers().get("user_id").unwrap(); + match crate::handler::http::auth::validator::list_objects_for_user( + &org_id, + user_id.to_str().unwrap(), + "GET", + "logs", + ) + .await + { + Ok(list) => { + _permitted = list; + } + Err(e) => { + return Ok(crate::common::meta::http::HttpResponse::forbidden( + e.to_string(), + )); + } + } + // Get List of allowed objects ends + } + + crate::service::pipeline::list_pipelines(org_id.into_inner(), _permitted).await +} + +/// GetStreamsWithPipeline +#[utoipa::path( + context_path = "/api", + tag = "Pipelines", + operation_id = "getStreamsWithPipeline", + security( + ("Authorization"= []) + ), + params( + ("org_id" = String, Path, description = "Organization name"), + ), + responses( + (status = 200, description = "Success", content_type = "application/json", body = PipelineList), + ) +)] +#[get("/{org_id}/pipelines/streams")] +async fn list_streams_with_pipeline( + path: web::Path, + _req: HttpRequest, +) -> Result { + let org_id = path.into_inner(); + crate::service::pipeline::list_streams_with_pipeline(&org_id).await +} + +/// DeletePipeline +#[utoipa::path( + context_path = "/api", + tag = "Pipelines", + operation_id = "deletePipeline", + security( + ("Authorization"= []) + ), + params( + ("org_id" = String, Path, description = "Organization name"), + ("pipeline_id" = String, Path, description = "Pipeline ID"), + ), + responses( + (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), + (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse), + ) +)] +#[delete("/{org_id}/pipelines/{pipeline_id}")] +async fn delete_pipeline( + path: web::Path<(String, String)>, + _req: HttpRequest, +) -> Result { + let (_org_id, pipeline_id) = path.into_inner(); + crate::service::pipeline::delete_pipeline(&pipeline_id).await +} + +/// UpdatePipeline +#[utoipa::path( + context_path = "/api", + tag = "Pipelines", + operation_id = "updatePipeline", + security( + ("Authorization"= []) + ), + params( + ("org_id" = String, Path, description = "Organization name"), + ), + request_body(content = Pipeline, description = "Pipeline data", content_type = "application/json"), + responses( + (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), + (status = 400, description = "Failure", content_type = "application/json", body = HttpResponse), + ) +)] +#[put("/{org_id}/pipelines")] +pub async fn update_pipeline( + pipeline: web::Json, + _req: HttpRequest, +) -> Result { + let pipeline = pipeline.into_inner(); + crate::service::pipeline::update_pipeline(pipeline).await +} + +/// EnablePipeline +#[utoipa::path( + context_path = "/api", + tag = "Pipelines", + operation_id = "enablePipeline", + security( + ("Authorization"= []) + ), + params( + ("org_id" = String, Path, description = "Organization name"), + ("pipeline_id" = String, Path, description = "Pipeline ID"), + ("value" = bool, Query, description = "Enable or disable pipeline"), + ), + responses( + (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), + (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse), + (status = 500, description = "Failure", content_type = "application/json", body = HttpResponse), + ) +)] +#[put("/{org_id}/pipelines/{pipeline_id}/enable")] +pub async fn enable_pipeline( + path: web::Path<(String, String)>, + req: HttpRequest, +) -> Result { + let (org_id, pipeline_id) = path.into_inner(); + let query = web::Query::>::from_query(req.query_string()).unwrap(); + let enable = match query.get("value") { + Some(v) => v.parse::().unwrap_or_default(), + None => false, + }; + crate::service::pipeline::enable_pipeline(&org_id, &pipeline_id, enable).await +} diff --git a/src/handler/http/request/pipelines.rs b/src/handler/http/request/pipelines.rs deleted file mode 100644 index 61f5518d94..0000000000 --- a/src/handler/http/request/pipelines.rs +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2024 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -use std::{collections::HashMap, io::Error}; - -use actix_web::{delete, get, http, post, put, web, HttpRequest, HttpResponse}; -use config::{meta::stream::StreamParams, utils::schema::format_stream_name}; - -use crate::common::{ - meta::{self, pipelines::PipeLine}, - utils::http::get_stream_type_from_request, -}; - -/// CreatePipeline -#[utoipa::path( - context_path = "/api", - tag = "Pipelines", - operation_id = "createPipeline", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ), - request_body(content = PipeLine, description = "Pipeline data", content_type = "application/json"), - responses( - (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), - (status = 400, description = "Failure", content_type = "application/json", body = HttpResponse), - ) -)] -#[post("/{org_id}/streams/{stream_name}/pipelines")] -pub async fn save_pipeline( - path: web::Path<(String, String)>, - pipeline: web::Json, - req: HttpRequest, -) -> Result { - let (org_id, stream_name) = path.into_inner(); - let query = web::Query::>::from_query(req.query_string()).unwrap(); - let mut pipeline = pipeline.into_inner(); - pipeline.name = pipeline.name.trim().to_string(); - pipeline.stream_name = stream_name; - pipeline.stream_type = match get_stream_type_from_request(&query) { - Ok(v) => v.unwrap_or_default(), - Err(e) => { - return Ok( - HttpResponse::BadRequest().json(meta::http::HttpResponse::error( - http::StatusCode::BAD_REQUEST.into(), - e.to_string(), - )), - ); - } - }; - if let Some(ref mut routing) = &mut pipeline.routing { - let keys_to_update: Vec<_> = routing.keys().cloned().collect(); - for key in keys_to_update { - let value = routing.remove(&key).unwrap(); - if value.is_empty() { - return Ok( - HttpResponse::BadRequest().json(meta::http::HttpResponse::error( - http::StatusCode::BAD_REQUEST.into(), - format!("Routing condition for {} is empty", key), - )), - ); - } - let formatted_key = format_stream_name(&key); - routing.insert(formatted_key, value); - } - } - crate::service::pipelines::save_pipeline(org_id, pipeline).await -} - -/// ListPipelines -#[utoipa::path( - context_path = "/api", - tag = "Pipelines", - operation_id = "listPipelines", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ), - responses( - (status = 200, description = "Success", content_type = "application/json", body = PipelineList), - ) -)] -#[get("/{org_id}/pipelines")] -async fn list_pipelines( - org_id: web::Path, - _req: HttpRequest, -) -> Result { - let mut _permitted = None; - // Get List of allowed objects - #[cfg(feature = "enterprise")] - { - let user_id = _req.headers().get("user_id").unwrap(); - match crate::handler::http::auth::validator::list_objects_for_user( - &org_id, - user_id.to_str().unwrap(), - "GET", - "logs", - ) - .await - { - Ok(list) => { - _permitted = list; - } - Err(e) => { - return Ok(crate::common::meta::http::HttpResponse::forbidden( - e.to_string(), - )); - } - } - // Get List of allowed objects ends - } - - crate::service::pipelines::list_pipelines(org_id.into_inner(), _permitted).await -} - -/// DeletePipeline -#[utoipa::path( - context_path = "/api", - tag = "Pipelines", - operation_id = "deletePipeline", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ("name" = String, Path, description = "Pipeline name"), - ), - responses( - (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), - (status = 404, description = "NotFound", content_type = "application/json", body = HttpResponse), - ) -)] -#[delete("/{org_id}/streams/{stream_name}/pipelines/{name}")] -async fn delete_pipeline( - path: web::Path<(String, String, String)>, - req: HttpRequest, -) -> Result { - let (org_id, stream_name, name) = path.into_inner(); - let query = web::Query::>::from_query(req.query_string()).unwrap(); - let stream_type = match get_stream_type_from_request(&query) { - Ok(v) => v.unwrap_or_default(), - Err(e) => { - return Ok(crate::common::meta::http::HttpResponse::bad_request(e)); - } - }; - let pipeline_source = StreamParams::new(&org_id, &stream_name, stream_type); - crate::service::pipelines::delete_pipeline(&name, pipeline_source).await -} - -/// UpdatePipeline -#[utoipa::path( - context_path = "/api", - tag = "Pipelines", - operation_id = "updatePipeline", - security( - ("Authorization"= []) - ), - params( - ("org_id" = String, Path, description = "Organization name"), - ("name" = String, Path, description = "Pipeline name"), - ), - request_body(content = PipeLine, description = "Pipeline data", content_type = "application/json"), - responses( - (status = 200, description = "Success", content_type = "application/json", body = HttpResponse), - (status = 400, description = "Failure", content_type = "application/json", body = HttpResponse), - ) -)] -#[put("/{org_id}/streams/{stream_name}/pipelines/{name}")] -pub async fn update_pipeline( - path: web::Path<(String, String, String)>, - pipeline: web::Json, - req: HttpRequest, -) -> Result { - let (org_id, stream_name, name) = path.into_inner(); - let query = web::Query::>::from_query(req.query_string()).unwrap(); - let stream_type = match get_stream_type_from_request(&query) { - Ok(v) => v.unwrap_or_default(), - Err(e) => { - return Ok(crate::common::meta::http::HttpResponse::bad_request(e)); - } - }; - let name = name.trim(); - let mut pipeline = pipeline.into_inner(); - pipeline.name = name.to_string(); - pipeline.stream_type = stream_type; - pipeline.stream_name = stream_name; - - if let Some(ref mut routing) = &mut pipeline.routing { - let keys_to_update: Vec<_> = routing.keys().cloned().collect(); - for key in keys_to_update { - let value = routing.remove(&key).unwrap(); - if value.is_empty() { - return Ok( - HttpResponse::BadRequest().json(meta::http::HttpResponse::error( - http::StatusCode::BAD_REQUEST.into(), - format!("Routing condition for {} is empty", key), - )), - ); - } - let formatted_key = format_stream_name(&key); - routing.insert(formatted_key, value); - } - } - crate::service::pipelines::update_pipeline(&org_id, pipeline).await -} diff --git a/src/handler/http/request/search/multi_streams.rs b/src/handler/http/request/search/multi_streams.rs index 3d9c72091d..b544d9a1ee 100644 --- a/src/handler/http/request/search/multi_streams.rs +++ b/src/handler/http/request/search/multi_streams.rs @@ -20,6 +20,7 @@ use chrono::{Duration, Utc}; use config::{ get_config, meta::{ + function::VRLResultResolver, search, sql::resolve_stream_names, stream::StreamType, @@ -471,11 +472,11 @@ pub async fn search_multi( if apply_over_hits { let ret_val = crate::service::ingestion::apply_vrl_fn( &mut runtime, - &meta::functions::VRLResultResolver { + &VRLResultResolver { program: program.program.clone(), fields: program.fields.clone(), }, - &json::Value::Array(multi_res.hits), + json::Value::Array(multi_res.hits), &org_id, &[vrl_stream_name.clone()], ); @@ -507,11 +508,11 @@ pub async fn search_multi( .filter_map(|hit| { let ret_val = crate::service::ingestion::apply_vrl_fn( &mut runtime, - &meta::functions::VRLResultResolver { + &VRLResultResolver { program: program.program.clone(), fields: program.fields.clone(), }, - &hit, + hit, &org_id, &[vrl_stream_name.clone()], ); diff --git a/src/handler/http/request/status/mod.rs b/src/handler/http/request/status/mod.rs index 7df2716fe3..a9ee1b93b0 100644 --- a/src/handler/http/request/status/mod.rs +++ b/src/handler/http/request/status/mod.rs @@ -26,7 +26,7 @@ use arrow_schema::Schema; use config::{ cluster::LOCAL_NODE, get_config, get_instance_id, - meta::cluster::NodeStatus, + meta::{cluster::NodeStatus, function::ZoFunction}, utils::{json, schema_ext::SchemaExt}, Config, QUICK_MODEL_FIELDS, SQL_FULL_TEXT_SEARCH_FIELDS, }; @@ -62,7 +62,6 @@ use crate::{ common::{ infra::{cluster, config::*}, meta::{ - functions::ZoFunction, http::HttpResponse as MetaHttpResponse, user::{AuthTokens, AuthTokensExt}, }, diff --git a/src/handler/http/router/mod.rs b/src/handler/http/router/mod.rs index d4ddab8afe..6dbc69912a 100644 --- a/src/handler/http/router/mod.rs +++ b/src/handler/http/router/mod.rs @@ -403,9 +403,6 @@ pub fn get_service_routes(cfg: &mut web::ServiceConfig) { .service(functions::list_functions) .service(functions::delete_function) .service(functions::update_function) - .service(functions::add_function_to_stream) - .service(functions::list_stream_functions) - .service(functions::delete_stream_function) .service(dashboards::create_dashboard) .service(dashboards::update_dashboard) .service(dashboards::list_dashboards) @@ -481,11 +478,12 @@ pub fn get_service_routes(cfg: &mut web::ServiceConfig) { .service(authz::fga::delete_group) .service(users::list_roles) .service(clusters::list_clusters) - .service(pipelines::save_pipeline) - .service(pipelines::list_pipelines) - .service(pipelines::delete_pipeline) - .service(pipelines::update_pipeline) - .service(pipelines::update_pipeline) + .service(pipeline::save_pipeline) + .service(pipeline::update_pipeline) + .service(pipeline::list_pipelines) + .service(pipeline::list_streams_with_pipeline) + .service(pipeline::delete_pipeline) + .service(pipeline::enable_pipeline) .service(search::multi_streams::search_multi) .service(search::multi_streams::_search_partition_multi) .service(search::multi_streams::around_multi) diff --git a/src/handler/http/router/openapi.rs b/src/handler/http/router/openapi.rs index b3811688b0..eb439d801e 100644 --- a/src/handler/http/router/openapi.rs +++ b/src/handler/http/router/openapi.rs @@ -74,9 +74,6 @@ use crate::{common::meta, handler::http::request}; request::functions::update_function, request::functions::save_function, request::functions::delete_function, - request::functions::list_stream_functions, - request::functions::add_function_to_stream, - request::functions::delete_stream_function, request::dashboards::create_dashboard, request::dashboards::update_dashboard, request::dashboards::list_dashboards, @@ -154,6 +151,24 @@ use crate::{common::meta, handler::http::request}; meta::dashboards::Folder, meta::dashboards::MoveDashboard, meta::dashboards::FolderList, + config::meta::alerts::alert::Alert, + config::meta::alerts::Aggregation, + config::meta::alerts::AggFunction, + config::meta::alerts::Condition, + config::meta::alerts::CompareHistoricData, + config::meta::alerts::destinations::Destination, + config::meta::alerts::destinations::DestinationWithTemplate, + config::meta::alerts::destinations::HTTPType, + config::meta::alerts::destinations::DestinationType, + config::meta::alerts::FrequencyType, + config::meta::alerts::Operator, + config::meta::alerts::QueryType, + config::meta::alerts::QueryCondition, + config::meta::alerts::TriggerCondition, + config::meta::alerts::templates::Template, + config::meta::function::Transform, + config::meta::function::FunctionList, + config::meta::function::StreamOrder, config::meta::sql::OrderBy, config::meta::search::Query, config::meta::search::Request, @@ -179,26 +194,6 @@ use crate::{common::meta, handler::http::request}; meta::saved_view::DeleteViewResponse, meta::saved_view::CreateViewResponse, meta::saved_view::UpdateViewRequest, - meta::alerts::alert::Alert, - meta::alerts::Condition, - meta::alerts::Operator, - meta::alerts::Aggregation, - meta::alerts::AggFunction, - meta::alerts::QueryType, - meta::alerts::TriggerCondition, - meta::alerts::FrequencyType, - meta::alerts::QueryCondition, - meta::alerts::CompareHistoricData, - meta::alerts::destinations::Destination, - meta::alerts::destinations::DestinationWithTemplate, - meta::alerts::destinations::HTTPType, - meta::alerts::destinations::DestinationType, - meta::alerts::templates::Template, - meta::functions::Transform, - meta::functions::FunctionList, - meta::functions::StreamFunctionsList, - meta::functions::StreamTransform, - meta::functions::StreamOrder, meta::user::UpdateUser, meta::user::UserRequest, meta::user::UserRole, diff --git a/src/infra/src/lib.rs b/src/infra/src/lib.rs index 4f51a37686..90fc8f7e54 100644 --- a/src/infra/src/lib.rs +++ b/src/infra/src/lib.rs @@ -18,6 +18,7 @@ pub mod db; pub mod dist_lock; pub mod errors; pub mod file_list; +pub mod pipeline; pub mod queue; pub mod scheduler; pub mod schema; @@ -30,6 +31,7 @@ pub async fn init() -> Result<(), anyhow::Error> { file_list::create_table().await?; file_list::LOCAL_CACHE.create_table().await?; file_list::local_cache_gc().await?; + pipeline::init().await?; queue::init().await?; scheduler::init().await?; schema::init().await?; diff --git a/src/infra/src/pipeline/mod.rs b/src/infra/src/pipeline/mod.rs new file mode 100644 index 0000000000..d0cbfa6bba --- /dev/null +++ b/src/infra/src/pipeline/mod.rs @@ -0,0 +1,133 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use async_trait::async_trait; +use config::meta::{ + meta_store::MetaStore, + pipeline::{components::PipelineSource, Pipeline}, + stream::StreamParams, +}; +use once_cell::sync::Lazy; + +use crate::errors::Result; + +pub mod mysql; +pub mod postgres; +pub mod sqlite; + +static CLIENT: Lazy> = Lazy::new(connect); + +pub fn connect() -> Box { + match config::get_config().common.meta_store.as_str().into() { + MetaStore::MySQL => Box::::default(), + MetaStore::PostgreSQL => Box::::default(), + _ => Box::::default(), + } +} + +#[async_trait] +pub trait PipelineTable: Sync + Send + 'static { + async fn create_table(&self) -> Result<()>; + async fn create_table_index(&self) -> Result<()>; + async fn drop_table(&self) -> Result<()>; + async fn put(&self, pipeline: &Pipeline) -> Result<()>; + async fn update(&self, pipeline: &Pipeline) -> Result<()>; + async fn get_by_stream(&self, stream_params: &StreamParams) -> Result; + async fn get_by_id(&self, pipeline_id: &str) -> Result; + async fn get_with_same_source_stream(&self, pipeline: &Pipeline) -> Result; + async fn list(&self) -> Result>; + async fn list_by_org(&self, org: &str) -> Result>; + async fn list_streams_with_pipeline(&self, org: &str) -> Result>; + async fn delete(&self, pipeline_id: &str) -> Result; +} + +/// Initializes the PipelineTable - creates table and index +pub async fn init() -> Result<()> { + CLIENT.create_table().await?; + CLIENT.create_table_index().await?; + Ok(()) +} + +/// Creates a pipeline entry in the table +#[inline] +pub async fn put(pipeline: &Pipeline) -> Result<()> { + CLIENT.put(pipeline).await +} + +/// Updates a pipeline entry by id +#[inline] +pub async fn update(pipeline: &Pipeline) -> Result<()> { + CLIENT.update(pipeline).await +} + +/// Finds the pipeline associated with the StreamParams within an organization +#[inline] +pub async fn get_by_stream(stream_params: &StreamParams) -> Result { + CLIENT.get_by_stream(stream_params).await +} + +/// Finds all streams with existing pipelines. +#[inline] +pub async fn list_streams_with_pipeline(org: &str) -> Result> { + CLIENT + .list_streams_with_pipeline(org) + .await + .map(|pipelines| { + pipelines + .into_iter() + .filter_map(|pl| match pl.source { + PipelineSource::Realtime(stream_params) => Some(stream_params), + PipelineSource::Scheduled(_) => None, + }) + .collect() + }) +} + +/// Finds the pipeline by id +#[inline] +pub async fn get_by_id(pipeline_id: &str) -> Result { + CLIENT.get_by_id(pipeline_id).await +} + +/// Finds the pipeline with the same source and structure +#[inline] +pub async fn get_with_same_source_stream(pipeline: &Pipeline) -> Result { + CLIENT.get_with_same_source_stream(pipeline).await +} + +/// Lists all pipelines +#[inline] +pub async fn list() -> Result> { + CLIENT.list().await +} + +/// Lists all pipelines within an organization +#[inline] +pub async fn list_by_org(org: &str) -> Result> { + CLIENT.list_by_org(org).await +} + +/// Deletes the pipeline by id +#[inline] +pub async fn delete(pipeline_id: &str) -> Result { + CLIENT.delete(pipeline_id).await +} + +/// DropTable first. Used for testing migration. +#[inline] +pub async fn drop_table() -> Result<()> { + CLIENT.drop_table().await?; + Ok(()) +} diff --git a/src/infra/src/pipeline/mysql.rs b/src/infra/src/pipeline/mysql.rs new file mode 100644 index 0000000000..18898f4f36 --- /dev/null +++ b/src/infra/src/pipeline/mysql.rs @@ -0,0 +1,375 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use async_trait::async_trait; +use config::{ + meta::{ + pipeline::{components::PipelineSource, Pipeline}, + stream::StreamParams, + }, + utils::json, +}; + +use crate::{ + db::mysql::CLIENT, + errors::{DbError, Error, Result}, +}; + +pub struct MySqlPipelineTable {} + +impl MySqlPipelineTable { + pub fn new() -> Self { + Self {} + } +} + +impl Default for MySqlPipelineTable { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl super::PipelineTable for MySqlPipelineTable { + async fn create_table(&self) -> Result<()> { + let pool = CLIENT.clone(); + + sqlx::query( + r#" +CREATE TABLE IF NOT EXISTS pipeline +( + id VARCHAR(256) not null primary key, + version INT not null, + enabled BOOLEAN default true not null, + name VARCHAR(256) not null, + description TEXT, + org VARCHAR(100) not null, + source_type VARCHAR(50) not null, + stream_org VARCHAR(100), + stream_name VARCHAR(256), + stream_type VARCHAR(50), + derived_stream TEXT, + nodes TEXT, + edges TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + "#, + ) + .execute(&pool) + .await?; + + Ok(()) + } + + async fn create_table_index(&self) -> Result<()> { + let pool = CLIENT.clone(); + + let queries = vec![ + "CREATE INDEX pipeline_org_idx ON pipeline (org);", + "CREATE INDEX pipeline_org_src_type_stream_params_idx ON pipeline (org, source_type, stream_org, stream_name, stream_type);", + ]; + + for query in queries { + if let Err(e) = sqlx::query(query).execute(&pool).await { + if e.to_string().contains("Duplicate key") { + // index already exists + continue; + } + log::error!("[MYSQL] create index for pipeline table error: {}", e); + return Err(e.into()); + } + } + Ok(()) + } + + async fn drop_table(&self) -> Result<()> { + let pool = CLIENT.clone(); + + sqlx::query("DROP TABLE IF EXISTS pipeline;") + .execute(&pool) + .await?; + + Ok(()) + } + + async fn put(&self, pipeline: &Pipeline) -> Result<()> { + let pool = CLIENT.clone(); + let mut tx = pool.begin().await?; + + if let Err(e) = match &pipeline.source { + PipelineSource::Realtime(stream_params) => { + let (source_type, stream_org, stream_name, stream_type): (&str, &str, &str, &str) = ( + "realtime", + stream_params.org_id.as_str(), + stream_params.stream_name.as_str(), + stream_params.stream_type.as_str(), + ); + sqlx::query( + r#" +INSERT IGNORE INTO pipeline (id, version, enabled, name, description, org, source_type, stream_org, stream_name, stream_type, nodes, edges) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + "#, + ) + .bind(&pipeline.id) + .bind(pipeline.version) + .bind(pipeline.enabled) + .bind(&pipeline.name) + .bind(&pipeline.description) + .bind(&pipeline.org) + .bind(source_type) + .bind(stream_org) + .bind(stream_name) + .bind(stream_type) + .bind(json::to_string(&pipeline.nodes).expect("Serializing pipeline nodes error")) + .bind(json::to_string(&pipeline.edges).expect("Serializing pipeline edges error")) + .execute(&mut *tx) + .await + } + PipelineSource::Scheduled(derived_stream) => { + let (source_type, derived_stream_str) = ( + "scheduled", + json::to_string(&derived_stream) + .expect("Serializing pipeline DerivedStream error"), + ); + sqlx::query( + r#" +INSERT IGNORE INTO pipeline (id, version, enabled, name, description, org, source_type, derived_stream, nodes, edges) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + "#, + ) + .bind(&pipeline.id) + .bind(pipeline.version) + .bind(pipeline.enabled) + .bind(&pipeline.name) + .bind(&pipeline.description) + .bind(&pipeline.org) + .bind(source_type) + .bind(derived_stream_str) + .bind(json::to_string(&pipeline.nodes).expect("Serializing pipeline nodes error")) + .bind(json::to_string(&pipeline.edges).expect("Serializing pipeline edges error")) + .execute(&mut *tx) + .await + } + } { + if let Err(e) = tx.rollback().await { + log::error!("[MYSQL] rollback push pipeline error: {}", e); + } + return Err(e.into()); + } + + if let Err(e) = tx.commit().await { + log::error!("[MYSQL] commit push pipeline error: {}", e); + return Err(e.into()); + } + + Ok(()) + } + + async fn update(&self, pipeline: &Pipeline) -> Result<()> { + let pool = CLIENT.clone(); + let mut tx = pool.begin().await?; + + if let Err(e) = match &pipeline.source { + PipelineSource::Realtime(stream_params) => { + let (source_type, stream_org, stream_name, stream_type): (&str, &str, &str, &str) = ( + "realtime", + stream_params.org_id.as_str(), + stream_params.stream_name.as_str(), + stream_params.stream_type.as_str(), + ); + sqlx::query( + r#" +UPDATE pipeline + SET version = ?, enabled = ?, name = ?, description = ?, org = ?, source_type = ?, stream_org = ?, stream_name = ?, stream_type = ?, nodes = ?, edges = ? + WHERE id =?; + "#, + ) + .bind(pipeline.version) + .bind(pipeline.enabled) + .bind(&pipeline.name) + .bind(&pipeline.description) + .bind(&pipeline.org) + .bind(source_type) + .bind(stream_org) + .bind(stream_name) + .bind(stream_type) + .bind(json::to_string(&pipeline.nodes).expect("Serializing pipeline nodes error")) + .bind(json::to_string(&pipeline.edges).expect("Serializing pipeline edges error")) + .bind(&pipeline.id) + .execute(&mut *tx) + .await + } + PipelineSource::Scheduled(derived_stream) => { + let (source_type, derived_stream_str) = ( + "scheduled", + json::to_string(&derived_stream) + .expect("Serializing pipeline DerivedStream error"), + ); + sqlx::query( + r#" +UPDATE pipeline + SET version = ?, enabled = ?, name = ?, description = ?, org = ?, source_type = ?, derived_stream = ?, nodes = ?, edges = ? + WHERE id = ?; + "#, + ) + .bind(pipeline.version) + .bind(pipeline.enabled) + .bind(&pipeline.name) + .bind(&pipeline.description) + .bind(&pipeline.org) + .bind(source_type) + .bind(derived_stream_str) + .bind(json::to_string(&pipeline.nodes).expect("Serializing pipeline nodes error")) + .bind(json::to_string(&pipeline.edges).expect("Serializing pipeline edges error")) + .bind(&pipeline.id) + .execute(&mut *tx) + .await + } + } { + if let Err(e) = tx.rollback().await { + log::error!("[MYSQL] rollback push pipeline error: {}", e); + } + return Err(e.into()); + } + + if let Err(e) = tx.commit().await { + log::error!("[MYSQL] commit push pipeline error: {}", e); + return Err(e.into()); + } + + Ok(()) + } + + async fn get_by_stream(&self, stream_params: &StreamParams) -> Result { + let pool = CLIENT.clone(); + let query = r#" +SELECT * FROM pipeline WHERE org = ? AND source_type = ? AND stream_org = ? AND stream_name = ? AND stream_type = ?; + "#; + let pipeline = sqlx::query_as::<_, Pipeline>(query) + .bind(stream_params.org_id.as_str()) + .bind("realtime") + .bind(stream_params.org_id.as_str()) + .bind(stream_params.stream_name.as_str()) + .bind(stream_params.stream_type.as_str()) + .fetch_one(&pool) + .await?; + Ok(pipeline) + } + + async fn get_by_id(&self, pipeline_id: &str) -> Result { + let pool = CLIENT.clone(); + let query = r#"SELECT * FROM pipeline WHERE id = ?;"#; + let pipeline = match sqlx::query_as::<_, Pipeline>(query) + .bind(pipeline_id) + .fetch_one(&pool) + .await + { + Ok(pipeline) => pipeline, + Err(e) => { + log::debug!("[MYSQL] get pipeline by id error: {}", e); + return Err(Error::from(DbError::KeyNotExists(pipeline_id.to_string()))); + } + }; + Ok(pipeline) + } + + async fn get_with_same_source_stream(&self, pipeline: &Pipeline) -> Result { + let pool = CLIENT.clone(); + let similar_pipeline = match &pipeline.source { + PipelineSource::Realtime(stream_params) => { + sqlx::query_as::<_, Pipeline>( + r#" +SELECT * FROM pipeline + WHERE source_type = ? AND stream_org = ? AND stream_name = ? AND stream_type = ?; + "#, + ) + .bind("realtime") + .bind(stream_params.org_id.as_str()) + .bind(stream_params.stream_name.as_str()) + .bind(stream_params.stream_type.as_str()) + .fetch_one(&pool) + .await? + } + PipelineSource::Scheduled(_) => { + // only checks for realtime pipelines + return Err(Error::from(DbError::KeyNotExists("".to_string()))); + } + }; + + Ok(similar_pipeline) + } + + async fn list(&self) -> Result> { + let pool = CLIENT.clone(); + let query = r#"SELECT * FROM pipeline ORDER BY id;"#; + match sqlx::query_as::<_, Pipeline>(query).fetch_all(&pool).await { + Ok(pipelines) => Ok(pipelines), + Err(e) => { + log::debug!("[MYSQL] list all pipelines error: {}", e); + Ok(vec![]) + } + } + } + + async fn list_by_org(&self, org: &str) -> Result> { + let pool = CLIENT.clone(); + let query = r#"SELECT * FROM pipeline WHERE org = ? ORDER BY id;"#; + match sqlx::query_as::<_, Pipeline>(query) + .bind(org) + .fetch_all(&pool) + .await + { + Ok(pipelines) => Ok(pipelines), + Err(e) => { + log::debug!("[MYSQL] list pipelines by org error: {}", e); + Ok(vec![]) + } + } + } + + async fn list_streams_with_pipeline(&self, org: &str) -> Result> { + let pool = CLIENT.clone(); + let query = r#" +SELECT * FROM pipeline WHERE org = ? AND source_type = ? ORDER BY id; + "#; + match sqlx::query_as::<_, Pipeline>(query) + .bind(org) + .bind("realtime") + .fetch_all(&pool) + .await + { + Ok(pipelines) => Ok(pipelines), + Err(e) => { + log::debug!("[MYSQL] list streams with pipelines error: {}", e); + Ok(vec![]) + } + } + } + + async fn delete(&self, pipeline_id: &str) -> Result { + let pool = CLIENT.clone(); + let pipeline = sqlx::query_as::<_, Pipeline>("SELECT * FROM pipeline WHERE id = ?;") + .bind(pipeline_id) + .fetch_one(&pool) + .await?; + + sqlx::query(r#"DELETE FROM pipeline WHERE id = ?;"#) + .bind(pipeline_id) + .execute(&pool) + .await?; + + Ok(pipeline) + } +} diff --git a/src/infra/src/pipeline/postgres.rs b/src/infra/src/pipeline/postgres.rs new file mode 100644 index 0000000000..7bb865984e --- /dev/null +++ b/src/infra/src/pipeline/postgres.rs @@ -0,0 +1,375 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use async_trait::async_trait; +use config::{ + meta::{ + pipeline::{components::PipelineSource, Pipeline}, + stream::StreamParams, + }, + utils::json, +}; + +use crate::{ + db::postgres::CLIENT, + errors::{DbError, Error, Result}, +}; + +pub struct PostgresPipelineTable {} + +impl PostgresPipelineTable { + pub fn new() -> Self { + Self {} + } +} + +impl Default for PostgresPipelineTable { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl super::PipelineTable for PostgresPipelineTable { + async fn create_table(&self) -> Result<()> { + let pool = CLIENT.clone(); + sqlx::query( + r#" +CREATE TABLE IF NOT EXISTS pipeline +( + id VARCHAR(256) PRIMARY KEY, + version INT not null, + enabled BOOLEAN default true not null, + name VARCHAR(256) not null, + description TEXT, + org VARCHAR(100) not null, + source_type VARCHAR(50) not null, + stream_org VARCHAR(100), + stream_name VARCHAR(256), + stream_type VARCHAR(50), + derived_stream TEXT, + nodes TEXT, + edges TEXT, + created_at TIMESTAMP default CURRENT_TIMESTAMP +); + "#, + ) + .execute(&pool) + .await?; + Ok(()) + } + + async fn create_table_index(&self) -> Result<()> { + let pool = CLIENT.clone(); + + let queries = vec![ + "CREATE INDEX IF NOT EXISTS pipeline_org_idx ON pipeline (org);", + "CREATE INDEX IF NOT EXISTS pipeline_org_src_type_stream_params_idx ON pipeline (org, source_type, stream_org, stream_name, stream_type);", + ]; + + for query in queries { + if let Err(e) = sqlx::query(query).execute(&pool).await { + log::error!( + "[POSTGRES] create table index for pipeline table error: {}", + e + ); + return Err(e.into()); + } + } + Ok(()) + } + + async fn drop_table(&self) -> Result<()> { + let pool = CLIENT.clone(); + sqlx::query("DROP TABLE IF EXISTS pipeline;") + .execute(&pool) + .await?; + Ok(()) + } + + async fn put(&self, pipeline: &Pipeline) -> Result<()> { + let pool = CLIENT.clone(); + let mut tx = pool.begin().await?; + + if let Err(e) = match &pipeline.source { + PipelineSource::Realtime(stream_params) => { + let (source_type, stream_org, stream_name, stream_type): (&str, &str, &str, &str) = ( + "realtime", + stream_params.org_id.as_str(), + stream_params.stream_name.as_str(), + stream_params.stream_type.as_str(), + ); + sqlx::query( + r#" +INSERT INTO pipeline (id, version, enabled, name, description, org, source_type, stream_org, stream_name, stream_type, nodes, edges) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + ON CONFLICT DO NOTHING; + "#, + ) + .bind(&pipeline.id) + .bind(pipeline.version) + .bind(pipeline.enabled) + .bind(&pipeline.name) + .bind(&pipeline.description) + .bind(&pipeline.org) + .bind(source_type) + .bind(stream_org) + .bind(stream_name) + .bind(stream_type) + .bind(json::to_string(&pipeline.nodes).expect("Serializing pipeline nodes error")) + .bind(json::to_string(&pipeline.edges).expect("Serializing pipeline edges error")) + .execute(&mut *tx) + .await + } + PipelineSource::Scheduled(derived_stream) => { + let (source_type, derived_stream_str) = ( + "scheduled", + json::to_string(&derived_stream) + .expect("Serializing pipeline DerivedStream error"), + ); + + sqlx::query( + r#" +INSERT INTO pipeline (id, version, enabled, name, description, org, source_type, derived_stream, nodes, edges) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT DO NOTHING; + "#, + ) + .bind(&pipeline.id) + .bind(pipeline.version) + .bind(pipeline.enabled) + .bind(&pipeline.name) + .bind(&pipeline.description) + .bind(&pipeline.org) + .bind(source_type) + .bind(derived_stream_str) + .bind(json::to_string(&pipeline.nodes).expect("Serializing pipeline nodes error")) + .bind(json::to_string(&pipeline.edges).expect("Serializing pipeline edges error")) + .execute(&mut *tx) + .await + } + } { + if let Err(e) = tx.rollback().await { + log::error!("[POSTGRES] rollback push pipeline error: {}", e); + } + return Err(e.into()); + } + + if let Err(e) = tx.commit().await { + log::error!("[POSTGRES] commit push pipeline error: {}", e); + return Err(e.into()); + } + Ok(()) + } + + async fn update(&self, pipeline: &Pipeline) -> Result<()> { + let pool = CLIENT.clone(); + let mut tx = pool.begin().await?; + + if let Err(e) = match &pipeline.source { + PipelineSource::Realtime(stream_params) => { + let (source_type, stream_org, stream_name, stream_type): (&str, &str, &str, &str) = ( + "realtime", + stream_params.org_id.as_str(), + stream_params.stream_name.as_str(), + stream_params.stream_type.as_str(), + ); + sqlx::query( + r#" +UPDATE pipeline + SET version = $1, enabled = $2, name = $3, description = $4, org = $5, source_type = $6, stream_org = $7, stream_name = $8, stream_type = $9, nodes = $10, edges = $11 + WHERE id = $12; + "#, + ) + .bind(pipeline.version) + .bind(pipeline.enabled) + .bind(&pipeline.name) + .bind(&pipeline.description) + .bind(&pipeline.org) + .bind(source_type) + .bind(stream_org) + .bind(stream_name) + .bind(stream_type) + .bind(json::to_string(&pipeline.nodes).expect("Serializing pipeline nodes error")) + .bind(json::to_string(&pipeline.edges).expect("Serializing pipeline edges error")) + .bind(&pipeline.id) + .execute(&mut *tx) + .await + } + PipelineSource::Scheduled(derived_stream) => { + let (source_type, derived_stream_str) = ( + "scheduled", + json::to_string(&derived_stream) + .expect("Serializing pipeline DerivedStream error"), + ); + + sqlx::query( + r#" +UPDATE pipeline + SET version = $1, enabled = $2, name = $3, description = $4, org = $5, source_type = $6, derived_stream = $7, nodes = $8, edges = $9 + WHERE id = $10; + "#, + ) + .bind(pipeline.version) + .bind(pipeline.enabled) + .bind(&pipeline.name) + .bind(&pipeline.description) + .bind(&pipeline.org) + .bind(source_type) + .bind(derived_stream_str) + .bind(json::to_string(&pipeline.nodes).expect("Serializing pipeline nodes error")) + .bind(json::to_string(&pipeline.edges).expect("Serializing pipeline edges error")) + .bind(&pipeline.id) + .execute(&mut *tx) + .await + } + } { + if let Err(e) = tx.rollback().await { + log::error!("[POSTGRES] rollback push pipeline error: {}", e); + } + return Err(e.into()); + } + + if let Err(e) = tx.commit().await { + log::error!("[POSTGRES] commit push pipeline error: {}", e); + return Err(e.into()); + } + Ok(()) + } + + async fn get_by_stream(&self, stream_params: &StreamParams) -> Result { + let pool = CLIENT.clone(); + let query = r#" +SELECT * FROM pipeline WHERE org = $1 AND source_type = $2 AND stream_org = $3 AND stream_name = $4 AND stream_type = $5; + "#; + let pipeline = sqlx::query_as::<_, Pipeline>(query) + .bind(stream_params.org_id.as_str()) + .bind("realtime") + .bind(stream_params.org_id.as_str()) + .bind(stream_params.stream_name.as_str()) + .bind(stream_params.stream_type.as_str()) + .fetch_one(&pool) + .await?; + Ok(pipeline) + } + + async fn get_by_id(&self, pipeline_id: &str) -> Result { + let pool = CLIENT.clone(); + let query = "SELECT * FROM pipeline WHERE id = $1;"; + let pipeline = match sqlx::query_as::<_, Pipeline>(query) + .bind(pipeline_id) + .fetch_one(&pool) + .await + { + Ok(pipeline) => pipeline, + Err(e) => { + log::error!("[POSTGRES] get pipeline by id error: {}", e); + return Err(Error::from(DbError::KeyNotExists(pipeline_id.to_string()))); + } + }; + Ok(pipeline) + } + + async fn get_with_same_source_stream(&self, pipeline: &Pipeline) -> Result { + let pool = CLIENT.clone(); + let similar_pipeline = match &pipeline.source { + PipelineSource::Realtime(stream_params) => { + sqlx::query_as::<_, Pipeline>( + r#" +SELECT * FROM pipeline + WHERE source_type = $1 AND stream_org = $2 AND stream_name = $3 AND stream_type = $4; + "#, + ) + .bind("realtime") + .bind(stream_params.org_id.as_str()) + .bind(stream_params.stream_name.as_str()) + .bind(stream_params.stream_type.as_str()) + .fetch_one(&pool) + .await? + } + PipelineSource::Scheduled(_) => { + // only checks for realtime pipelines + return Err(Error::from(DbError::KeyNotExists( + "No matching pipeline found for scheduled source".to_string(), + ))); + } + }; + + Ok(similar_pipeline) + } + + async fn list(&self) -> Result> { + let pool = CLIENT.clone(); + let query = "SELECT * FROM pipeline ORDER BY id;"; + match sqlx::query_as::<_, Pipeline>(query).fetch_all(&pool).await { + Ok(pipelines) => Ok(pipelines), + Err(e) => { + log::debug!("[POSTGRES] list all pipelines error: {}", e); + Ok(vec![]) // Return empty vector instead of error + } + } + } + + async fn list_by_org(&self, org: &str) -> Result> { + let pool = CLIENT.clone(); + let query = "SELECT * FROM pipeline WHERE org = $1 ORDER BY id;"; + match sqlx::query_as::<_, Pipeline>(query) + .bind(org) + .fetch_all(&pool) + .await + { + Ok(pipelines) => Ok(pipelines), + Err(e) => { + log::debug!("[POSTGRES] list pipelines by org error: {}", e); + Ok(vec![]) + } + } + } + + async fn list_streams_with_pipeline(&self, org: &str) -> Result> { + let pool = CLIENT.clone(); + let query = r#" +SELECT * FROM pipeline WHERE org = $1 AND source_type = $2 ORDER BY id; + "#; + match sqlx::query_as::<_, Pipeline>(query) + .bind(org) + .bind("realtime") + .fetch_all(&pool) + .await + { + Ok(pipelines) => Ok(pipelines), + Err(e) => { + log::debug!("[POSTGRES] list streams with pipelines error: {}", e); + Ok(vec![]) + } + } + } + + async fn delete(&self, pipeline_id: &str) -> Result { + let pool = CLIENT.clone(); + + let pipeline = sqlx::query_as::<_, Pipeline>("SELECT * FROM pipeline WHERE id = $1;") + .bind(pipeline_id) + .fetch_one(&pool) + .await?; + + sqlx::query(r#"DELETE FROM pipeline WHERE id = $1;"#) + .bind(pipeline_id) + .execute(&pool) + .await?; + + Ok(pipeline) + } +} diff --git a/src/infra/src/pipeline/sqlite.rs b/src/infra/src/pipeline/sqlite.rs new file mode 100644 index 0000000000..bc37bfa017 --- /dev/null +++ b/src/infra/src/pipeline/sqlite.rs @@ -0,0 +1,387 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use async_trait::async_trait; +use config::{ + meta::{ + pipeline::{components::PipelineSource, Pipeline}, + stream::StreamParams, + }, + utils::json, +}; + +use crate::{ + db::sqlite::{CLIENT_RO, CLIENT_RW}, + errors::{DbError, Error, Result}, +}; + +pub struct SqlitePipelineTable {} + +impl SqlitePipelineTable { + pub fn new() -> Self { + Self {} + } +} + +impl Default for SqlitePipelineTable { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl super::PipelineTable for SqlitePipelineTable { + async fn create_table(&self) -> Result<()> { + let client = CLIENT_RW.clone(); + let client = client.lock().await; + sqlx::query( + r#" +CREATE TABLE IF NOT EXISTS pipeline +( + id VARCHAR(256) not null primary key, + version INT not null, + enabled BOOLEAN default true not null, + name VARCHAR(256) not null, + description TEXT, + org VARCHAR(100) not null, + source_type VARCHAR(50) not null, + stream_org VARCHAR(100), + stream_name VARCHAR(256), + stream_type VARCHAR(50), + derived_stream TEXT, + nodes TEXT, + edges TEXT, + created_at TIMESTAMP default CURRENT_TIMESTAMP +); + "#, + ) + .execute(&*client) + .await?; + Ok(()) + } + + async fn create_table_index(&self) -> Result<()> { + let client = CLIENT_RW.clone(); + let client = client.lock().await; + let queries = vec![ + "CREATE INDEX IF NOT EXISTS pipeline_org_idx ON pipeline (org);", + "CREATE INDEX IF NOT EXISTS pipeline_org_src_type_stream_params_idx ON pipeline (org, source_type, stream_org, stream_name, stream_type);", + ]; + + for query in queries { + sqlx::query(query).execute(&*client).await?; + } + Ok(()) + } + + async fn drop_table(&self) -> Result<()> { + let client = CLIENT_RW.clone(); + let client = client.lock().await; + sqlx::query("DROP TABLE IF EXISTS pipeline;") + .execute(&*client) + .await?; + Ok(()) + } + + async fn put(&self, pipeline: &Pipeline) -> Result<()> { + let client = CLIENT_RW.clone(); + let client = client.lock().await; + let mut tx = client.begin().await?; + + if let Err(e) = match &pipeline.source { + PipelineSource::Realtime(stream_params) => { + let (source_type, stream_org, stream_name, stream_type): (&str, &str, &str, &str) = ( + "realtime", + stream_params.org_id.as_str(), + stream_params.stream_name.as_str(), + stream_params.stream_type.as_str(), + ); + sqlx::query( + r#" +INSERT INTO pipeline (id, version, enabled, name, description, org, source_type, stream_org, stream_name, stream_type, nodes, edges) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + ON CONFLICT DO NOTHING; + "#, + ) + .bind(&pipeline.id) + .bind(pipeline.version) + .bind(pipeline.enabled) + .bind(&pipeline.name) + .bind(&pipeline.description) + .bind(&pipeline.org) + .bind(source_type) + .bind(stream_org) + .bind(stream_name) + .bind(stream_type) + .bind(json::to_string(&pipeline.nodes).expect("Serializing pipeline nodes error")) + .bind(json::to_string(&pipeline.edges).expect("Serializing pipeline edges error")) + .execute(&mut *tx) + .await + } + PipelineSource::Scheduled(derived_stream) => { + let (source_type, derived_stream_str) = ( + "scheduled", + json::to_string(&derived_stream) + .expect("Serializing pipeline DerivedStream error"), + ); + sqlx::query( + r#" +INSERT INTO pipeline (id, version, enabled, name, description, org, source_type, derived_stream, nodes, edges) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT DO NOTHING; + "#, + ) + .bind(&pipeline.id) + .bind(pipeline.version) + .bind(pipeline.enabled) + .bind(&pipeline.name) + .bind(&pipeline.description) + .bind(&pipeline.org) + .bind(source_type) + .bind(derived_stream_str) + .bind(json::to_string(&pipeline.nodes).expect("Serializing pipeline nodes error")) + .bind(json::to_string(&pipeline.edges).expect("Serializing pipeline edges error")) + .execute(&mut *tx) + .await + } + } { + if let Err(e) = tx.rollback().await { + log::error!("[SQLITE] rollback push pipeline error: {}", e); + } + return Err(e.into()); + } + + if let Err(e) = tx.commit().await { + log::error!("[SQLITE] commit push pipeline error: {}", e); + return Err(e.into()); + } + + // release lock + drop(client); + Ok(()) + } + + async fn update(&self, pipeline: &Pipeline) -> Result<()> { + let client = CLIENT_RW.clone(); + let client = client.lock().await; + let mut tx = client.begin().await?; + + if let Err(e) = match &pipeline.source { + PipelineSource::Realtime(stream_params) => { + let (source_type, stream_org, stream_name, stream_type): (&str, &str, &str, &str) = ( + "realtime", + stream_params.org_id.as_str(), + stream_params.stream_name.as_str(), + stream_params.stream_type.as_str(), + ); + sqlx::query( + r#" +UPDATE pipeline + SET version = $1, enabled = $2, name = $3, description = $4, org = $5, source_type = $6, stream_org = $7, stream_name = $8, stream_type = $9, nodes = $10, edges = $11 + WHERE id = $12; + "#, + ) + .bind(pipeline.version) + .bind(pipeline.enabled) + .bind(&pipeline.name) + .bind(&pipeline.description) + .bind(&pipeline.org) + .bind(source_type) + .bind(stream_org) + .bind(stream_name) + .bind(stream_type) + .bind(json::to_string(&pipeline.nodes).expect("Serializing pipeline nodes error")) + .bind(json::to_string(&pipeline.edges).expect("Serializing pipeline edges error")) + .bind(&pipeline.id) + .execute(&mut *tx) + .await + } + PipelineSource::Scheduled(derived_stream) => { + let (source_type, derived_stream_str) = ( + "scheduled", + json::to_string(&derived_stream) + .expect("Serializing pipeline DerivedStream error"), + ); + sqlx::query( + r#" +UPDATE pipeline + SET version = $1, enabled = $2, name = $3, description = $4, org = $5, source_type = $6, derived_stream = $7, nodes = $8, edges = $9 + WHERE id = $10; + "#, + ) + .bind(pipeline.version) + .bind(pipeline.enabled) + .bind(&pipeline.name) + .bind(&pipeline.description) + .bind(&pipeline.org) + .bind(source_type) + .bind(derived_stream_str) + .bind(json::to_string(&pipeline.nodes).expect("Serializing pipeline nodes error")) + .bind(json::to_string(&pipeline.edges).expect("Serializing pipeline edges error")) + .bind(&pipeline.id) + .execute(&mut *tx) + .await + } + } { + if let Err(e) = tx.rollback().await { + log::error!("[SQLITE] rollback push pipeline error: {}", e); + } + return Err(e.into()); + } + + if let Err(e) = tx.commit().await { + log::error!("[SQLITE] commit push pipeline error: {}", e); + return Err(e.into()); + } + + // release lock + drop(client); + Ok(()) + } + + async fn get_by_stream(&self, stream_params: &StreamParams) -> Result { + let pool = CLIENT_RO.clone(); + let query = r#" +SELECT * FROM pipeline WHERE org = $1 AND source_type = $2 AND stream_org = $3 AND stream_name = $4 AND stream_type = $5; + "#; + let pipeline = sqlx::query_as::<_, Pipeline>(query) + .bind(stream_params.org_id.as_str()) + .bind("realtime") + .bind(stream_params.org_id.as_str()) + .bind(stream_params.stream_name.as_str()) + .bind(stream_params.stream_type.as_str()) + .fetch_one(&pool) + .await?; + Ok(pipeline) + } + + async fn get_by_id(&self, pipeline_id: &str) -> Result { + let pool = CLIENT_RO.clone(); + let query = "SELECT * FROM pipeline WHERE id = $1;"; + let pipeline = match sqlx::query_as::<_, Pipeline>(query) + .bind(pipeline_id) + .fetch_one(&pool) + .await + { + Ok(pipeline) => pipeline, + Err(e) => { + log::error!("[SQLITE] get pipeline by id error: {}", e); + return Err(Error::from(DbError::KeyNotExists(pipeline_id.to_string()))); + } + }; + Ok(pipeline) + } + + async fn get_with_same_source_stream(&self, pipeline: &Pipeline) -> Result { + let pool = CLIENT_RO.clone(); + let similar_pipeline = match &pipeline.source { + PipelineSource::Realtime(stream_params) => { + sqlx::query_as::<_, Pipeline>( + r#" +SELECT * FROM pipeline + WHERE source_type = $1 AND stream_org = $2 AND stream_name = $3 AND stream_type = $4; + "#, + ) + .bind("realtime") + .bind(stream_params.org_id.as_str()) + .bind(stream_params.stream_name.as_str()) + .bind(stream_params.stream_type.as_str()) + .fetch_one(&pool) + .await? + } + PipelineSource::Scheduled(_) => { + // only checks for realtime pipelines + return Err(Error::from(DbError::KeyNotExists("".to_string()))); + } + }; + + Ok(similar_pipeline) + } + + async fn list(&self) -> Result> { + let client = CLIENT_RO.clone(); + let query = "SELECT * FROM pipeline ORDER BY id;"; + match sqlx::query_as::<_, Pipeline>(query) + .fetch_all(&client) + .await + { + Ok(pipelines) => Ok(pipelines), + Err(e) => { + log::debug!("[SQLITE] list all pipelines error: {}", e); + Ok(vec![]) + } + } + } + + async fn list_by_org(&self, org: &str) -> Result> { + let client = CLIENT_RO.clone(); + let query = "SELECT * FROM pipeline WHERE org = $1 ORDER BY id;"; + match sqlx::query_as::<_, Pipeline>(query) + .bind(org) + .fetch_all(&client) + .await + { + Ok(pipelines) => Ok(pipelines), + Err(e) => { + log::debug!("[SQLITE] list pipeline by org error: {}", e); + Ok(vec![]) + } + } + } + + async fn list_streams_with_pipeline(&self, org: &str) -> Result> { + let client = CLIENT_RO.clone(); + let query = r#" +SELECT * FROM pipeline WHERE org = $1 AND source_type = $2 ORDER BY id; + "#; + match sqlx::query_as::<_, Pipeline>(query) + .bind(org) + .bind("realtime") + .fetch_all(&client) + .await + { + Ok(pipelines) => Ok(pipelines), + Err(e) => { + log::debug!("[SQLITE] list streams with pipelines error: {}", e); + Ok(vec![]) + } + } + } + + async fn delete(&self, pipeline_id: &str) -> Result { + let client = CLIENT_RW.clone(); + let client = client.lock().await; + let mut tx = client.begin().await?; + + let pipeline = sqlx::query_as::<_, Pipeline>("SELECT * FROM pipeline WHERE id = $1;") + .bind(pipeline_id) + .fetch_one(&mut *tx) + .await?; + + sqlx::query(r#"DELETE FROM pipeline WHERE id = $1;"#) + .bind(pipeline_id) + .execute(&mut *tx) + .await?; + + if let Err(e) = tx.commit().await { + log::error!("[SQLITE] commit delete pipeline error: {}", e); + return Err(e.into()); + } + + // release lock + drop(client); + Ok(pipeline) + } +} diff --git a/src/job/mod.rs b/src/job/mod.rs index 63bccff7d2..6c31d1fd1a 100644 --- a/src/job/mod.rs +++ b/src/job/mod.rs @@ -122,9 +122,6 @@ pub async fn init() -> Result<(), anyhow::Error> { tokio::task::spawn(async move { db::organization::watch().await }); #[cfg(feature = "enterprise")] tokio::task::spawn(async move { db::ofga::watch().await }); - if LOCAL_NODE.is_ingester() { - tokio::task::spawn(async move { db::pipelines::watch().await }); - } #[cfg(feature = "enterprise")] if !LOCAL_NODE.is_compactor() || LOCAL_NODE.is_single_node() { @@ -168,9 +165,9 @@ pub async fn init() -> Result<(), anyhow::Error> { db::syslog::cache_syslog_settings() .await .expect("syslog settings cache failed"); - if LOCAL_NODE.is_ingester() { - db::pipelines::cache().await.expect("syslog cache failed"); - } + + // cache pipeline + db::pipeline::cache().await.expect("Pipeline cache failed"); // cache file list if !cfg.common.meta_store_external { diff --git a/src/service/alerts/alert.rs b/src/service/alerts/alert.rs index 336c3c37e1..2090e5935d 100644 --- a/src/service/alerts/alert.rs +++ b/src/service/alerts/alert.rs @@ -19,10 +19,18 @@ use std::{ }; use actix_web::http; +use async_trait::async_trait; use chrono::{Duration, Local, TimeZone, Timelike, Utc}; use config::{ get_config, - meta::stream::{StreamParams, StreamType}, + meta::{ + alerts::{ + alert::{Alert, AlertListFilter}, + destinations::{DestinationType, DestinationWithTemplate, HTTPType}, + FrequencyType, Operator, QueryType, + }, + stream::StreamType, + }, utils::{ base64, json::{Map, Value}, @@ -34,18 +42,11 @@ use lettre::{message::SinglePart, AsyncTransport, Message}; use crate::{ common::{ - meta::{ - alerts::{ - alert::{Alert, AlertListFilter}, - destinations::{DestinationType, DestinationWithTemplate, HTTPType}, - FrequencyType, Operator, QueryType, - }, - authz::Authz, - }, + meta::authz::Authz, utils::auth::{is_ofga_unsupported, remove_ownership, set_ownership}, }, service::{ - alerts::{build_sql, destinations}, + alerts::{build_sql, destinations, QueryConditionExt}, db, search::sql::RE_ONLY_SELECT, short_url, @@ -343,10 +344,29 @@ pub async fn trigger( .map_err(|e| (http::StatusCode::INTERNAL_SERVER_ERROR, e)) } -impl Alert { +#[async_trait] +pub trait AlertExt: Sync + Send + 'static { /// Returns the evaluated row data and the end time of the search timerange, /// for realtime this is 0. `start_time` is the start time of the search timerange. - pub async fn evaluate( + async fn evaluate( + &self, + row: Option<&Map>, + start_time: Option, + ) -> Result<(Option>>, i64), anyhow::Error>; + + /// Returns a tuple containing a boolean - if all the send notification jobs succeeded + /// and the error message if any + async fn send_notification( + &self, + rows: &[Map], + rows_end_time: i64, + start_time: Option, + ) -> Result<(String, String), anyhow::Error>; +} + +#[async_trait] +impl AlertExt for Alert { + async fn evaluate( &self, row: Option<&Map>, start_time: Option, @@ -356,7 +376,9 @@ impl Alert { } else { self.query_condition .evaluate_scheduled( - &self.get_stream_params(), + &self.org_id, + Some(&self.stream_name), + self.stream_type, &self.trigger_condition, start_time, ) @@ -364,9 +386,7 @@ impl Alert { } } - /// Returns a tuple containing a boolean - if all the send notification jobs succeeded - /// and the error message if any - pub async fn send_notification( + async fn send_notification( &self, rows: &[Map], rows_end_time: i64, @@ -406,14 +426,6 @@ impl Alert { Ok((success_message, err_message)) } } - - pub fn get_stream_params(&self) -> StreamParams { - StreamParams { - org_id: self.org_id.clone().into(), - stream_name: self.stream_name.clone().into(), - stream_type: self.stream_type, - } - } } pub async fn send_notification( @@ -811,7 +823,9 @@ async fn process_dest_template( QueryType::Custom => { if let Some(conditions) = &alert.query_condition.conditions { if let Ok(v) = build_sql( - &alert.get_stream_params(), + &alert.org_id, + &alert.stream_name, + alert.stream_type, &alert.query_condition, conditions, ) diff --git a/src/service/alerts/derived_streams.rs b/src/service/alerts/derived_streams.rs index 9c46fb8e88..c6a5108d7b 100644 --- a/src/service/alerts/derived_streams.rs +++ b/src/service/alerts/derived_streams.rs @@ -15,46 +15,27 @@ use std::str::FromStr; +use async_trait::async_trait; use chrono::Utc; use config::{ get_config, + meta::{ + alerts::{FrequencyType, QueryType}, + pipeline::components::DerivedStream, + }, utils::json::{Map, Value}, }; use cron::Schedule; -use hashbrown::HashMap; -use crate::{ - common::{ - meta::{ - // authz::Authz, - alerts::{derived_streams::DerivedStreamMeta, FrequencyType, QueryType}, - }, - // utils::auth::{remove_ownership, set_ownership}, - }, - service::db, -}; +use crate::service::{alerts::QueryConditionExt, db}; pub async fn save( - mut derived_stream: DerivedStreamMeta, + mut derived_stream: DerivedStream, pipeline_name: &str, + pipeline_id: &str, ) -> Result<(), anyhow::Error> { - derived_stream.name = derived_stream.name.trim().to_string(); // 1. Start validate DerivedStream - if !derived_stream.is_valid() { - return Err(anyhow::anyhow!( - "DerivedStream Name, destination, and Trigger period required" - )); - } - - // query_type for realtime - if derived_stream.is_real_time && derived_stream.query_condition.query_type != QueryType::Custom - { - return Err(anyhow::anyhow!( - "Realtime DerivedStream should use Custom query type" - )); - } - - // other checks for query type + // checks for query type match derived_stream.query_condition.query_type { QueryType::SQL => { if derived_stream @@ -82,22 +63,7 @@ pub async fn save( } } _ => {} - } - - // check source stream schema - let schema = infra::schema::get( - &derived_stream.source.org_id, - &derived_stream.source.stream_name, - derived_stream.source.stream_type, - ) - .await?; - if schema.fields().is_empty() { - return Err(anyhow::anyhow!( - "Source Stream {}/{} schema is empty.", - derived_stream.source.org_id, - derived_stream.source.stream_name, - )); - } + }; // End input validation // 2. update the frequency @@ -110,17 +76,8 @@ pub async fn save( std::cmp::max(1, get_config().limit.derived_stream_schedule_interval / 60); } - // 3. clean up DerivedStream context attributes - if let Some(attrs) = &derived_stream.context_attributes { - let mut new_attrs = HashMap::with_capacity(attrs.len()); - for (key, val) in attrs.iter() { - new_attrs.insert(key.trim().to_string(), val.to_string()); - } - derived_stream.context_attributes = Some(new_attrs); - } - // test derived_stream - if let Err(e) = &derived_stream.evaluate(None, None).await { + if let Err(e) = &derived_stream.evaluate(None).await { return Err(anyhow::anyhow!( "DerivedStream not saved due to failed test run caused by {}", e.to_string() @@ -130,11 +87,11 @@ pub async fn save( // Save the trigger to db let next_run_at = Utc::now().timestamp_micros(); let trigger = db::scheduler::Trigger { - org: derived_stream.source.org_id.to_string(), + org: derived_stream.org_id.to_string(), module: db::scheduler::TriggerModule::DerivedStream, - module_key: derived_stream.get_scheduler_module_key(pipeline_name), + module_key: derived_stream.get_scheduler_module_key(pipeline_name, pipeline_id), next_run_at, - is_realtime: derived_stream.is_real_time, + is_realtime: false, is_silenced: false, ..Default::default() }; @@ -150,45 +107,49 @@ pub async fn save( } pub async fn delete( - derived_stream: DerivedStreamMeta, + derived_stream: DerivedStream, pipeline_name: &str, + pipeline_id: &str, ) -> Result<(), anyhow::Error> { db::scheduler::delete( - &derived_stream.source.org_id, + &derived_stream.org_id, db::scheduler::TriggerModule::DerivedStream, - &derived_stream.get_scheduler_module_key(pipeline_name), + &derived_stream.get_scheduler_module_key(pipeline_name, pipeline_id), ) .await .map_err(|e| anyhow::anyhow!("Error deleting derived stream trigger: {e}")) } -impl DerivedStreamMeta { - pub fn is_valid(&self) -> bool { - !self.name.is_empty() - && !self.is_real_time // TODO(taiming): support realtime DerivedStream - && self.source.is_valid() - && self.destination.is_valid() - && self.trigger_condition.period != 0 - } +#[async_trait] +pub trait DerivedStreamExt: Sync + Send + 'static { + fn get_scheduler_module_key(&self, pipeline_name: &str, pipeline_id: &str) -> String; + async fn evaluate( + &self, + start_time: Option, + ) -> Result<(Option>>, i64), anyhow::Error>; +} - pub fn get_scheduler_module_key(&self, pipeline_name: &str) -> String { +#[async_trait] +impl DerivedStreamExt for DerivedStream { + fn get_scheduler_module_key(&self, pipeline_name: &str, pipeline_id: &str) -> String { format!( "{}/{}/{}/{}", - self.source.stream_type, self.source.stream_name, pipeline_name, self.name + self.stream_type, self.org_id, pipeline_name, pipeline_id ) } - pub async fn evaluate( + async fn evaluate( &self, - row: Option<&Map>, start_time: Option, ) -> Result<(Option>>, i64), anyhow::Error> { - if self.is_real_time { - self.query_condition.evaluate_realtime(row).await - } else { - self.query_condition - .evaluate_scheduled(&self.source, &self.trigger_condition, start_time) - .await - } + self.query_condition + .evaluate_scheduled( + &self.org_id, + None, + self.stream_type, + &self.trigger_condition, + start_time, + ) + .await } } diff --git a/src/service/alerts/destinations.rs b/src/service/alerts/destinations.rs index f3584b7273..76c141ba60 100644 --- a/src/service/alerts/destinations.rs +++ b/src/service/alerts/destinations.rs @@ -14,14 +14,12 @@ // along with this program. If not, see . use actix_web::http; +use config::meta::alerts::destinations::{Destination, DestinationType, DestinationWithTemplate}; use crate::{ common::{ infra::config::STREAM_ALERTS, - meta::{ - alerts::destinations::{Destination, DestinationType, DestinationWithTemplate}, - authz::Authz, - }, + meta::authz::Authz, utils::auth::{is_ofga_unsupported, remove_ownership, set_ownership}, }, service::db::{self, user}, diff --git a/src/service/alerts/mod.rs b/src/service/alerts/mod.rs index 4087922ade..c5472c4a02 100644 --- a/src/service/alerts/mod.rs +++ b/src/service/alerts/mod.rs @@ -15,12 +15,14 @@ use alert::to_float; use arrow_schema::DataType; +use async_trait::async_trait; use chrono::{Duration, Utc}; use config::{ get_config, ider, meta::{ + alerts::{AggFunction, Condition, Operator, QueryCondition, QueryType, TriggerCondition}, search::{SearchEventType, SqlQuery}, - stream::StreamParams, + stream::StreamType, }, utils::{ base64, @@ -29,12 +31,7 @@ use config::{ }; use super::promql; -use crate::{ - common::meta::alerts::{ - AggFunction, Condition, Operator, QueryCondition, QueryType, TriggerCondition, - }, - service::search as SearchService, -}; +use crate::service::search as SearchService; pub mod alert; pub mod derived_streams; @@ -42,8 +39,26 @@ pub mod destinations; pub mod scheduler; pub mod templates; -impl QueryCondition { - pub async fn evaluate_realtime( +#[async_trait] +pub trait QueryConditionExt: Sync + Send + 'static { + async fn evaluate_realtime( + &self, + row: Option<&Map>, + ) -> Result<(Option>>, i64), anyhow::Error>; + + async fn evaluate_scheduled( + &self, + org_id: &str, + stream_name: Option<&str>, + stream_type: StreamType, + trigger_condition: &TriggerCondition, + start_time: Option, + ) -> Result<(Option>>, i64), anyhow::Error>; +} + +#[async_trait] +impl QueryConditionExt for QueryCondition { + async fn evaluate_realtime( &self, row: Option<&Map>, ) -> Result<(Option>>, i64), anyhow::Error> { @@ -69,19 +84,23 @@ impl QueryCondition { Ok((Some(vec![row.to_owned()]), now)) } - pub async fn evaluate_scheduled( + async fn evaluate_scheduled( &self, - stream_param: &StreamParams, + org_id: &str, + stream_name: Option<&str>, + stream_type: StreamType, trigger_condition: &TriggerCondition, start_time: Option, ) -> Result<(Option>>, i64), anyhow::Error> { let now = Utc::now().timestamp_micros(); let sql = match self.query_type { QueryType::Custom => { - let Some(v) = self.conditions.as_ref() else { + let (Some(stream_name), Some(v)) = (stream_name, self.conditions.as_ref()) else { + // CustomQuery type needs to provide source StreamName. + // CustomQuery is only used by Alerts' triggers. return Ok((None, now)); }; - build_sql(stream_param, self, v).await? + build_sql(org_id, stream_name, stream_type, self, v).await? } QueryType::SQL => { let Some(v) = self.sql.as_ref() else { @@ -127,7 +146,7 @@ impl QueryCondition { (end - start) / promql::MAX_DATA_POINTS, ), }; - let resp = match promql::search::search(&stream_param.org_id, &req, 0, "").await { + let resp = match promql::search::search(org_id, &req, 0, "").await { Ok(v) => v, Err(_) => { return Ok((None, now)); @@ -275,14 +294,7 @@ impl QueryCondition { per_query_response: false, // Will return results in single array }; - SearchService::search_multi( - &trace_id, - &stream_param.org_id, - stream_param.stream_type, - None, - &req, - ) - .await + SearchService::search_multi(&trace_id, org_id, stream_type, None, &req).await } else { // fire the query let req = config::meta::search::Request { @@ -320,14 +332,7 @@ impl QueryCondition { * scheduled & inform FE */ index_type: "".to_string(), }; - SearchService::search( - &trace_id, - &stream_param.org_id, - stream_param.stream_type, - None, - &req, - ) - .await + SearchService::search(&trace_id, org_id, stream_type, None, &req).await }; // Resp hits can be of two types - @@ -406,8 +411,14 @@ impl QueryCondition { } } -impl Condition { - pub async fn evaluate(&self, row: &Map) -> bool { +#[async_trait] +pub trait ConditionExt: Sync + Send + 'static { + async fn evaluate(&self, row: &Map) -> bool; +} + +#[async_trait] +impl ConditionExt for Condition { + async fn evaluate(&self, row: &Map) -> bool { let val = match row.get(&self.column) { Some(val) => val, None => { @@ -473,16 +484,13 @@ impl Condition { } async fn build_sql( - stream_params: &StreamParams, + org_id: &str, + stream_name: &str, + stream_type: StreamType, query_condition: &QueryCondition, conditions: &[Condition], ) -> Result { - let schema = infra::schema::get( - &stream_params.org_id, - &stream_params.stream_name, - stream_params.stream_type, - ) - .await?; + let schema = infra::schema::get(org_id, stream_name, stream_type).await?; let mut wheres = Vec::with_capacity(conditions.len()); for cond in conditions.iter() { let data_type = match schema.field_with_name(&cond.column) { @@ -491,7 +499,7 @@ async fn build_sql( return Err(anyhow::anyhow!( "Column {} not found on stream {}", &cond.column, - &stream_params.stream_name + stream_name )); } }; @@ -504,10 +512,7 @@ async fn build_sql( String::new() }; if query_condition.aggregation.is_none() { - return Ok(format!( - "SELECT * FROM \"{}\" {}", - stream_params.stream_name, where_sql - )); + return Ok(format!("SELECT * FROM \"{}\" {}", stream_name, where_sql)); } // handle aggregation @@ -520,7 +525,7 @@ async fn build_sql( return Err(anyhow::anyhow!( "Aggregation column {} not found on stream {}", &agg.having.column, - &stream_params.stream_name + &stream_name )); } }; @@ -550,7 +555,7 @@ async fn build_sql( func_expr, cfg.common.column_timestamp, cfg.common.column_timestamp, - stream_params.stream_name, + stream_name, where_sql, group.join(", "), having_expr @@ -563,7 +568,7 @@ async fn build_sql( func_expr, cfg.common.column_timestamp, cfg.common.column_timestamp, - stream_params.stream_name, + stream_name, where_sql, having_expr ); diff --git a/src/service/alerts/scheduler.rs b/src/service/alerts/scheduler.rs index dc5020ff2b..8cf345d295 100644 --- a/src/service/alerts/scheduler.rs +++ b/src/service/alerts/scheduler.rs @@ -13,13 +13,14 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{str::FromStr, time::Instant}; +use std::{collections::HashMap, str::FromStr, time::Instant}; use chrono::{Duration, FixedOffset, Utc}; use config::{ get_config, meta::{ - stream::StreamType, + alerts::FrequencyType, + stream::{StreamParams, StreamType}, usage::{TriggerData, TriggerDataStatus, TriggerDataType}, }, utils::{ @@ -34,11 +35,15 @@ use infra::scheduler::get_scheduler_max_retries; use proto::cluster_rpc; use crate::{ - common::meta::{alerts::FrequencyType, dashboards::reports::ReportFrequencyType}, + common::meta::dashboards::reports::ReportFrequencyType, service::{ - alerts::alert::{get_alert_start_end_time, get_row_column_map}, + alerts::{ + alert::{get_alert_start_end_time, get_row_column_map, AlertExt}, + derived_streams::DerivedStreamExt, + }, db::{self, scheduler::ScheduledTriggerData}, ingestion::ingestion_service, + pipeline::batch_execution::ExecutablePipeline, usage::publish_triggers_usage, }, }; @@ -774,64 +779,14 @@ async fn handle_derived_stream_triggers( ); let (_, max_retries) = get_scheduler_max_retries(); - // module_key format: stream_type/stream_name/pipeline_name/derived_stream_name + // module_key format: stream_type/org_id/pipeline_name/pipeline_id let columns = trigger.module_key.split('/').collect::>(); assert_eq!(columns.len(), 4); - let org_id = &trigger.org; let stream_type: StreamType = columns[0].into(); - let stream_name = columns[1]; + let org_id = columns[1]; let pipeline_name = columns[2]; - let name = columns[3]; + let pipeline_id = columns[3]; - let is_real_time = trigger.is_realtime; - let is_silenced = trigger.is_silenced; - if is_real_time && is_silenced { - log::debug!( - "Realtime derived_stream needs to wake up, {}/{}", - org_id, - trigger.module_key - ); - let new_trigger = db::scheduler::Trigger { - next_run_at: Utc::now().timestamp_micros(), - is_silenced: false, - status: db::scheduler::TriggerStatus::Waiting, - ..trigger.clone() - }; - db::scheduler::update_trigger(new_trigger).await?; - return Ok(()); - } - - let Ok(pipeline) = db::pipelines::get(org_id, stream_type, stream_name, pipeline_name).await - else { - return Err(anyhow::anyhow!( - "Pipeline associated with trigger not found: {}/{}/{}/{}", - org_id, - stream_name, - stream_type, - pipeline_name - )); - }; - - let Some(derived_stream) = pipeline - .derived_streams - .and_then(|ds| ds.into_iter().find(|ds| ds.name == name)) - else { - return Err(anyhow::anyhow!( - "DerivedStream associated with the trigger not found in pipeline: {}/{}/{}/{}", - org_id, - stream_name, - stream_type, - name, - )); - }; - let trigger_data: Option = json::from_str(&trigger.data).ok(); - let start_time = if let Some(trigger_data) = trigger_data { - trigger_data - .period_end_time - .map(|period_end_time| period_end_time + 1) - } else { - None - }; let mut new_trigger = db::scheduler::Trigger { next_run_at: Utc::now().timestamp_micros(), is_silenced: false, @@ -840,6 +795,53 @@ async fn handle_derived_stream_triggers( ..trigger.clone() }; + let Ok(pipeline) = db::pipeline::get_by_id(pipeline_id).await else { + log::warn!( + "Pipeline associated with trigger not found: {}/{}/{}/{}. Deleting this trigger", + org_id, + stream_type, + pipeline_name, + pipeline_id + ); + db::scheduler::delete(&trigger.org, trigger.module, &trigger.module_key).await?; + return Err(anyhow::anyhow!( + "Pipeline associated with trigger not found: {}/{}/{}/{}", + org_id, + stream_type, + pipeline_name, + pipeline_id + )); + }; + + if !pipeline.enabled { + // remove the trigger from scheduler. Trigger will be added back when the pipeline is + // enabled again + log::info!("Pipeline associated with trigger not enabled. Removing trigger from Scheduler"); + db::scheduler::delete(&trigger.org, trigger.module, &trigger.module_key).await?; + return Ok(()); + } + + let Some(derived_stream) = pipeline.get_derived_stream() else { + return Err(anyhow::anyhow!( + "DerivedStream associated with the trigger not found in pipeline: {}/{}/{}", + org_id, + pipeline_name, + pipeline_id, + )); + }; + let start_time = if trigger.data.is_empty() { + None + } else { + let trigger_data: Option = json::from_str(&trigger.data).ok(); + if let Some(trigger_data) = trigger_data { + trigger_data + .period_end_time + .map(|period_end_time| period_end_time + 1) + } else { + None + } + }; + if trigger.retries >= max_retries { // It has been tried the maximum time, just update the // next_run_at to the next expected trigger time @@ -870,7 +872,7 @@ async fn handle_derived_stream_triggers( } // evaluate trigger and configure trigger next run time - let (ret, end_time) = derived_stream.evaluate(None, start_time).await?; + let (ret, end_time) = derived_stream.evaluate(start_time).await?; if ret.is_some() { log::info!( "DerivedStream conditions satisfied, org: {}, module_key: {}", @@ -963,36 +965,87 @@ async fn handle_derived_stream_triggers( db::scheduler::update_trigger(new_trigger).await?; trigger_data_stream.status = TriggerDataStatus::ConditionNotSatisfied; } else { - // Ingest result into destination stream - let (org_id, stream_name, stream_type): (String, String, i32) = { - ( - derived_stream.destination.org_id.into(), - derived_stream.destination.stream_name.into(), - cluster_rpc::StreamType::from(derived_stream.destination.stream_type).into(), - ) - }; - let req = cluster_rpc::IngestionRequest { - org_id: org_id.clone(), - stream_name: stream_name.clone(), - stream_type, - data: Some(cluster_rpc::IngestionData::from(local_val)), - ingestion_type: Some(cluster_rpc::IngestionType::Json.into()), /* TODO(taiming): finalize IngestionType for derived_stream */ - }; - match ingestion_service::ingest(&org_id, req).await { - Ok(resp) if resp.status_code == 200 => { - log::info!( - "DerivedStream result ingested to destination {org_id}/{stream_name}/{stream_type}", + // pass search results to pipeline to get modified results before ingesting + let mut json_data_by_stream: HashMap> = HashMap::new(); + let mut ingestion_error_msg = None; + + match ExecutablePipeline::new(&pipeline).await { + Err(e) => { + let err_msg = format!( + "Pipeline: {}/{} associated with the DerivedStream failed to initialize ExecutablePipeline. Caused by: {}", + org_id, pipeline_name, e ); - db::scheduler::update_trigger(new_trigger).await?; + log::error!("{err_msg}"); + ingestion_error_msg = Some(err_msg); } - error => { - let err = error.map_or_else(|e| e.to_string(), |resp| resp.message); - log::error!( - "Error in ingesting DerivedStream result to destination {:?}, org: {}, module_key: {}", - err, - new_trigger.org, - new_trigger.module_key - ); + Ok(exec_pl) => match exec_pl.process_batch(org_id, local_val).await { + Err(e) => { + let err_msg = format!( + "DerivedStream query results failed to pass through the associated pipeline: {}/{}. Caused by: {}", + org_id, pipeline_name, e + ); + log::error!("{err_msg}"); + ingestion_error_msg = Some(err_msg); + } + Ok(pl_results) => { + for (stream_params, stream_pl_results) in pl_results { + if matches!( + stream_params.stream_type, + StreamType::Logs | StreamType::EnrichmentTables + ) { + let (_, results): (Vec<_>, Vec<_>) = + stream_pl_results.into_iter().unzip(); + json_data_by_stream + .entry(stream_params) + .or_default() + .extend(results); + } + } + } + }, + }; + + // Ingest result into destination stream + if ingestion_error_msg.is_none() { + for (dest_stream, records) in json_data_by_stream { + let (org_id, stream_name, stream_type): (String, String, i32) = { + ( + dest_stream.org_id.into(), + dest_stream.stream_name.into(), + cluster_rpc::StreamType::from(dest_stream.stream_type).into(), + ) + }; + let req = cluster_rpc::IngestionRequest { + org_id: org_id.clone(), + stream_name: stream_name.clone(), + stream_type, + data: Some(cluster_rpc::IngestionData::from(records)), + ingestion_type: Some(cluster_rpc::IngestionType::Json.into()), + }; + match ingestion_service::ingest(&org_id, req).await { + Ok(resp) if resp.status_code == 200 => { + log::info!( + "DerivedStream result ingested to destination {org_id}/{stream_name}/{stream_type}", + ); + } + error => { + let err = error.map_or_else(|e| e.to_string(), |resp| resp.message); + log::error!( + "Error in ingesting DerivedStream result to destination {:?}, org: {}, module_key: {}", + err, + new_trigger.org, + new_trigger.module_key + ); + ingestion_error_msg = Some(err); + break; + } + }; + } + } + + match ingestion_error_msg { + None => db::scheduler::update_trigger(new_trigger).await?, + Some(err) => { if trigger.retries + 1 >= max_retries { // It has been tried the maximum time, just update the // next_run_at to the next expected trigger time diff --git a/src/service/alerts/templates.rs b/src/service/alerts/templates.rs index c604a07d74..26dfa55b5b 100644 --- a/src/service/alerts/templates.rs +++ b/src/service/alerts/templates.rs @@ -14,11 +14,12 @@ // along with this program. If not, see . use actix_web::http; +use config::meta::alerts::templates::Template; use crate::{ common::{ infra::config::ALERTS_DESTINATIONS, - meta::{alerts::templates::Template, authz::Authz}, + meta::authz::Authz, utils::auth::{is_ofga_unsupported, remove_ownership, set_ownership}, }, service::db, diff --git a/src/service/db/alerts/alert.rs b/src/service/db/alerts/alert.rs index fe62351343..04c1de459b 100644 --- a/src/service/db/alerts/alert.rs +++ b/src/service/db/alerts/alert.rs @@ -15,13 +15,13 @@ use std::sync::Arc; -use config::{meta::stream::StreamType, utils::json}; - -use crate::{ - common::{infra::config::STREAM_ALERTS, meta::alerts::alert::Alert}, - service::db, +use config::{ + meta::{alerts::alert::Alert, stream::StreamType}, + utils::json, }; +use crate::{common::infra::config::STREAM_ALERTS, service::db}; + pub async fn get( org_id: &str, stream_type: StreamType, diff --git a/src/service/db/alerts/destinations.rs b/src/service/db/alerts/destinations.rs index 7dda9684ae..0ab286efba 100644 --- a/src/service/db/alerts/destinations.rs +++ b/src/service/db/alerts/destinations.rs @@ -15,13 +15,10 @@ use std::sync::Arc; -use config::utils::json; +use config::{meta::alerts::destinations::Destination, utils::json}; use itertools::Itertools; -use crate::{ - common::{infra::config::ALERTS_DESTINATIONS, meta::alerts::destinations::Destination}, - service::db, -}; +use crate::{common::infra::config::ALERTS_DESTINATIONS, service::db}; pub async fn get(org_id: &str, name: &str) -> Result { let map_key = format!("{org_id}/{name}"); diff --git a/src/service/db/alerts/templates.rs b/src/service/db/alerts/templates.rs index 3075ee53ae..6d66858839 100644 --- a/src/service/db/alerts/templates.rs +++ b/src/service/db/alerts/templates.rs @@ -15,14 +15,11 @@ use std::sync::Arc; -use config::utils::json; +use config::{meta::alerts::templates::Template, utils::json}; use itertools::Itertools; use crate::{ - common::{ - infra::config::ALERTS_TEMPLATES, - meta::{alerts::templates::Template, organization::DEFAULT_ORG}, - }, + common::{infra::config::ALERTS_TEMPLATES, meta::organization::DEFAULT_ORG}, service::db, }; diff --git a/src/service/db/functions.rs b/src/service/db/functions.rs index 5381f90fcd..72de2a0c85 100644 --- a/src/service/db/functions.rs +++ b/src/service/db/functions.rs @@ -15,15 +15,9 @@ use std::sync::Arc; -use config::utils::json; +use config::{meta::function::Transform, utils::json}; -use crate::{ - common::{ - infra::config::{QUERY_FUNCTIONS, STREAM_FUNCTIONS}, - meta::functions::{StreamFunctionsList, Transform}, - }, - service::db, -}; +use crate::{common::infra::config::QUERY_FUNCTIONS, service::db}; pub async fn set(org_id: &str, name: &str, js_func: &Transform) -> Result<(), anyhow::Error> { let key = format!("/function/{org_id}/{name}"); @@ -87,7 +81,6 @@ pub async fn watch() -> Result<(), anyhow::Error> { match ev { db::Event::Put(ev) => { let item_key = ev.key.strip_prefix(key).unwrap(); - let org_id = &item_key[0..item_key.find('/').unwrap()]; let item_value: Transform = if config::get_config().common.meta_store_external { match db::get(&ev.key).await { Ok(val) => match json::from_slice(&val) { @@ -105,29 +98,7 @@ pub async fn watch() -> Result<(), anyhow::Error> { } else { json::from_slice(&ev.value.unwrap()).unwrap() }; - if item_value.streams.is_some() { - for stream_fn in item_value.to_stream_transform() { - let mut group = STREAM_FUNCTIONS - .entry(format!( - "{}/{}/{}", - org_id, stream_fn.stream_type, stream_fn.stream - )) - .or_insert_with(|| StreamFunctionsList { list: vec![] }); - if group.list.contains(&stream_fn) { - let stream_name = - group.list.iter().position(|x| x.eq(&stream_fn)).unwrap(); - if stream_fn.is_removed { - group.list.remove(stream_name); - } else { - let _ = std::mem::replace(&mut group.list[stream_name], stream_fn); - } - } else if !stream_fn.is_removed { - group.list.push(stream_fn); - } - } - } else { - QUERY_FUNCTIONS.insert(item_key.to_owned(), item_value); - } + QUERY_FUNCTIONS.insert(item_key.to_owned(), item_value); } db::Event::Delete(ev) => { let item_key = ev.key.strip_prefix(key).unwrap(); @@ -145,25 +116,7 @@ pub async fn cache() -> Result<(), anyhow::Error> { for (item_key, item_value) in ret { let item_key = item_key.strip_prefix(key).unwrap(); let json_val: Transform = json::from_slice(&item_value).unwrap(); - let org_id = &item_key[0..item_key.find('/').unwrap()]; - if json_val.streams.is_some() { - for stream_fn in json_val.to_stream_transform() { - let mut group = STREAM_FUNCTIONS - .entry(format!( - "{}/{}/{}", - org_id, stream_fn.stream_type, stream_fn.stream - )) - .or_insert_with(|| StreamFunctionsList { list: vec![] }); - if !stream_fn.is_removed { - group.list.push(stream_fn); - } - } - let mut func = json_val.clone(); - func.streams = None; - QUERY_FUNCTIONS.insert(item_key.to_string(), func); - } else { - QUERY_FUNCTIONS.insert(item_key.to_string(), json_val); - } + QUERY_FUNCTIONS.insert(item_key.to_string(), json_val); } log::info!("Functions Cached"); Ok(()) diff --git a/src/service/db/mod.rs b/src/service/db/mod.rs index 5a08709630..2bcad6c0e0 100644 --- a/src/service/db/mod.rs +++ b/src/service/db/mod.rs @@ -34,7 +34,7 @@ pub mod metrics; #[cfg(feature = "enterprise")] pub mod ofga; pub mod organization; -pub mod pipelines; +pub mod pipeline; pub mod saved_view; pub mod scheduler; pub mod schema; diff --git a/src/service/db/pipeline.rs b/src/service/db/pipeline.rs new file mode 100644 index 0000000000..366efc5d48 --- /dev/null +++ b/src/service/db/pipeline.rs @@ -0,0 +1,231 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// use std::sync::Arc; + +use anyhow::Result; +use config::meta::{ + pipeline::{components::PipelineSource, Pipeline}, + stream::StreamParams, +}; +use infra::pipeline::{self as infra_pipeline}; + +use crate::{ + common::infra::config::STREAM_EXECUTABLE_PIPELINES, + service::pipeline::batch_execution::ExecutablePipeline, +}; + +/// Stores a new pipeline to database. +/// +/// Pipeline validation should be handled by the caller. +pub async fn set(pipeline: &Pipeline) -> Result<()> { + if let Err(e) = infra_pipeline::put(pipeline).await { + log::error!("Error saving pipeline: {}", e); + return Err(anyhow::anyhow!("Error saving pipeline: {}", e)); + } + + // save to cache if realtime pipeline + if let PipelineSource::Realtime(stream_params) = &pipeline.source { + if pipeline.enabled { + update_cache(stream_params, pipeline, PipelineTableEvent::Add).await; + } + } + + Ok(()) +} + +/// Updates a pipeline entry with the sane values. +/// +/// Pipeline validation should be handled by the caller. +pub async fn update(pipeline: &Pipeline) -> Result<()> { + if let Err(e) = infra_pipeline::update(pipeline).await { + log::error!("Error updating pipeline: {}", e); + return Err(anyhow::anyhow!("Error updating pipeline: {}", e)); + } + + // save to cache if realtime pipeline + if let PipelineSource::Realtime(stream_params) = &pipeline.source { + let db_event = if pipeline.enabled { + PipelineTableEvent::Add + } else { + PipelineTableEvent::Remove + }; + update_cache(stream_params, pipeline, db_event).await; + } + + Ok(()) +} + +/// Returns all streams with existing pipelines. +pub async fn list_streams_with_pipeline(org: &str) -> Result> { + infra_pipeline::list_streams_with_pipeline(org) + .await + .map_err(|e| { + log::error!("Error getting streams with pipeline for org({org}): {}", e); + anyhow::anyhow!("Error getting streams with pipeline for org({org}): {}", e) + }) +} + +/// Transform the initialized and enabled pipeline into ExecutablePipeline struct that's ready for +/// batch processing records. +/// +/// Used for pipeline execution. +pub async fn get_executable_pipeline(stream_params: &StreamParams) -> Option { + if let Some(exec_pl) = STREAM_EXECUTABLE_PIPELINES.read().await.get(stream_params) { + return Some(exec_pl.clone()); + } + let pipeline = infra_pipeline::get_by_stream(stream_params).await.ok(); + match pipeline { + Some(pl) if pl.enabled => match ExecutablePipeline::new(&pl).await { + Ok(exec_pl) => { + let mut stream_exec_pl_cache = STREAM_EXECUTABLE_PIPELINES.write().await; + stream_exec_pl_cache.insert(stream_params.to_owned(), exec_pl.clone()); + drop(stream_exec_pl_cache); + Some(exec_pl) + } + Err(e) => { + log::error!( + "[Pipeline]: failed to initialize ExecutablePipeline from Pipeline read from database, {}", + e + ); + None + } + }, + _ => None, + } +} + +/// Returns the pipeline by id. +/// +/// Used to get the pipeline associated with the ID when scheduled job is ran. +pub async fn get_by_id(pipeline_id: &str) -> Result { + infra_pipeline::get_by_id(pipeline_id).await.map_err(|e| { + log::error!("Error getting pipeline with ID({pipeline_id}): {}", e); + anyhow::anyhow!("Error getting pipeline with ID({pipeline_id}): {}", e) + }) +} + +/// Finds the pipeline with the same source +/// +/// Used to validate if a duplicate pipeline exists. +pub async fn get_with_same_source_stream(pipeline: &Pipeline) -> Result { + infra_pipeline::get_with_same_source_stream(pipeline) + .await + .map_err(|_| anyhow::anyhow!("No pipeline with the same source found")) +} + +/// Lists all pipelines across all orgs. +pub async fn list() -> Result> { + infra_pipeline::list().await.map_err(|e| { + log::debug!("Error listing pipelines for all orgs: {}", e); + anyhow::anyhow!("Error listing pipelines for all orgs: {}", e) + }) +} + +/// Lists all pipelines for a given organization. +pub async fn list_by_org(org: &str) -> Result> { + infra_pipeline::list_by_org(org).await.map_err(|e| { + log::debug!("Error listing pipelines for org({org}): {}", e); + anyhow::anyhow!("Error listing pipelines for org({org}): {}", e) + }) +} + +/// Deletes a pipeline by ID. +pub async fn delete(pipeline_id: &str) -> Result<()> { + match infra_pipeline::delete(pipeline_id).await { + Err(e) => { + log::error!("Error deleting pipeline with ID({pipeline_id}): {}", e); + return Err(anyhow::anyhow!( + "Error deleting pipeline with ID({pipeline_id}): {}", + e + )); + } + Ok(pipeline) => { + // remove from cache if realtime pipeline + if let PipelineSource::Realtime(stream_params) = &pipeline.source { + update_cache(stream_params, &pipeline, PipelineTableEvent::Remove).await; + } + } + } + + Ok(()) +} + +/// Preload all enabled pipelines into the cache at startup. +pub async fn cache() -> Result<(), anyhow::Error> { + let pipelines = list().await?; + let mut stream_exec_pl = STREAM_EXECUTABLE_PIPELINES.write().await; + for pipeline in pipelines.into_iter() { + if pipeline.enabled { + if let PipelineSource::Realtime(stream_params) = &pipeline.source { + match ExecutablePipeline::new(&pipeline).await { + Err(e) => { + log::error!( + "[Pipeline] error initializing ExecutablePipeline from pipeline {}/{}. {}. Not cached", + pipeline.org, + pipeline.id, + e + ) + } + Ok(exec_pl) => { + stream_exec_pl.insert(stream_params.clone(), exec_pl); + } + }; + } + } + } + log::info!("[Pipeline] Cached with len: {}", stream_exec_pl.len()); + Ok(()) +} + +/// Update STREAM_PIPELINES cache for realtime pipelines +async fn update_cache( + stream_params: &StreamParams, + pipeline: &Pipeline, + event: PipelineTableEvent, +) { + match event { + PipelineTableEvent::Remove => { + log::info!("[Pipeline]: pipeline {} removed from cache.", &pipeline.id); + STREAM_EXECUTABLE_PIPELINES + .write() + .await + .remove(stream_params); + } + PipelineTableEvent::Add => { + let mut stream_pl_exec = STREAM_EXECUTABLE_PIPELINES.write().await; + match ExecutablePipeline::new(pipeline).await { + Err(e) => { + log::error!( + "[Pipeline] {}/{}/{}: Error initializing pipeline into ExecutablePipeline when updating cache: {}", + pipeline.org, + pipeline.name, + pipeline.id, + e + ); + } + Ok(exec_pl) => { + stream_pl_exec.insert(stream_params.clone(), exec_pl); + } + }; + log::info!("[Pipeline]: pipeline {} added to cache.", &pipeline.id); + } + } +} + +enum PipelineTableEvent { + Add, + Remove, +} diff --git a/src/service/db/pipelines.rs b/src/service/db/pipelines.rs deleted file mode 100644 index e98d52ad17..0000000000 --- a/src/service/db/pipelines.rs +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2024 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -use std::sync::Arc; - -use config::{meta::stream::StreamType, utils::json}; - -use crate::{ - common::{infra::config::STREAM_PIPELINES, meta::pipelines::PipeLine}, - service::db, -}; - -pub async fn set(org_id: &str, pipeline: &PipeLine) -> Result<(), anyhow::Error> { - let key = format!( - "/pipeline/{org_id}/{}/{}/{}", - pipeline.stream_type, pipeline.stream_name, pipeline.name - ); - match db::put( - &key, - json::to_vec(pipeline).unwrap().into(), - db::NEED_WATCH, - None, - ) - .await - { - Ok(_) => {} - Err(e) => { - log::error!("Error saving pipeline: {}", e); - return Err(anyhow::anyhow!("Error saving pipeline: {}", e)); - } - } - - Ok(()) -} - -pub async fn get( - org_id: &str, - stream_type: StreamType, - stream_name: &str, - name: &str, -) -> Result { - let val = db::get(&format!( - "/pipeline/{org_id}/{stream_type}/{stream_name}/{name}" - )) - .await?; - Ok(json::from_slice(&val).unwrap()) -} - -pub async fn delete( - org_id: &str, - stream_type: StreamType, - stream_name: &str, - name: &str, -) -> Result<(), anyhow::Error> { - let key = format!("/pipeline/{org_id}/{stream_type}/{stream_name}/{name}"); - match db::delete(&key, false, db::NEED_WATCH, None).await { - Ok(_) => {} - Err(e) => { - log::error!("Error deleting pipeline: {}", e); - return Err(anyhow::anyhow!("Error deleting pipeline: {}", e)); - } - } - Ok(()) -} - -pub async fn list(org_id: &str) -> Result, anyhow::Error> { - Ok(db::list(&format!("/pipeline/{org_id}/")) - .await? - .values() - .map(|val| json::from_slice(val).unwrap()) - .collect()) -} - -pub async fn watch() -> Result<(), anyhow::Error> { - let key = "/pipeline/"; - let cluster_coordinator = db::get_coordinator().await; - let mut events = cluster_coordinator.watch(key).await?; - let events = Arc::get_mut(&mut events).unwrap(); - log::info!("Start watching pipeline"); - loop { - let ev = match events.recv().await { - Some(ev) => ev, - None => { - log::error!("watch_pipelines: event channel closed"); - break; - } - }; - match ev { - db::Event::Put(ev) => { - let item_key = ev.key.strip_prefix(key).unwrap(); - let org_id = &item_key[0..item_key.find('/').unwrap()]; - let item_value: PipeLine = if config::get_config().common.meta_store_external { - match db::get(&ev.key).await { - Ok(val) => match json::from_slice(&val) { - Ok(val) => val, - Err(e) => { - log::error!("Error getting value: {}", e); - continue; - } - }, - Err(e) => { - log::error!("Error getting value: {}", e); - continue; - } - } - } else { - json::from_slice(&ev.value.unwrap()).unwrap() - }; - let key = format!( - "{org_id}/{}/{}", - item_value.stream_type, item_value.stream_name - ); - STREAM_PIPELINES.insert(key.to_owned(), item_value); - } - db::Event::Delete(ev) => { - let item_key = ev.key.strip_prefix(key).unwrap(); - let removal_key = match item_key.rfind('/') { - Some(index) => &item_key[0..index], - None => item_key, - }; - - STREAM_PIPELINES.remove(removal_key); - } - db::Event::Empty => {} - } - } - Ok(()) -} - -pub async fn cache() -> Result<(), anyhow::Error> { - let key = "/pipeline/"; - let ret = db::list(key).await?; - for (item_key, item_value) in ret { - let item_key = item_key.strip_prefix(key).unwrap(); - let json_val: PipeLine = json::from_slice(&item_value).unwrap(); - let org_id = &item_key[0..item_key.find('/').unwrap()]; - let key = format!("{org_id}/{}/{}", json_val.stream_type, json_val.stream_name); - - STREAM_PIPELINES.insert(key.to_string(), json_val); - } - - log::info!("Pipelines Cached"); - Ok(()) -} - -pub async fn reset() -> Result<(), anyhow::Error> { - let key = "/pipeline/"; - db::delete(key, true, db::NO_NEED_WATCH, None).await?; - Ok(()) -} diff --git a/src/service/functions.rs b/src/service/functions.rs index f2973eb844..1a8d3325a4 100644 --- a/src/service/functions.rs +++ b/src/service/functions.rs @@ -19,18 +19,11 @@ use actix_web::{ http::{self, StatusCode}, HttpResponse, }; -use config::meta::stream::StreamType; +use config::meta::function::{FunctionList, Transform}; use crate::{ common::{ - infra::config::STREAM_FUNCTIONS, - meta::{ - authz::Authz, - functions::{ - FunctionList, StreamFunctionsList, StreamOrder, StreamTransform, Transform, - }, - http::HttpResponse as MetaHttpResponse, - }, + meta::{authz::Authz, http::HttpResponse as MetaHttpResponse}, utils::auth::{remove_ownership, set_ownership}, }, service::{db, ingestion::compile_vrl_function}, @@ -38,8 +31,6 @@ use crate::{ const FN_SUCCESS: &str = "Function saved successfully"; const FN_NOT_FOUND: &str = "Function not found"; -const FN_ADDED: &str = "Function applied to stream"; -const FN_REMOVED: &str = "Function removed from stream"; const FN_DELETED: &str = "Function deleted"; const FN_ALREADY_EXIST: &str = "Function already exist"; const FN_IN_USE: &str = @@ -101,10 +92,6 @@ pub async fn update_function( return Ok(HttpResponse::Ok().json(func)); } - // UI mostly like in 1st version won't send streams, so we need to add them back - // from existing function - func.streams = existing_fn.streams; - if !func.function.ends_with('.') { func.function = format!("{} \n .", func.function); } @@ -206,118 +193,6 @@ pub async fn delete_function(org_id: String, fn_name: String) -> Result Result { - if let Some(val) = STREAM_FUNCTIONS.get(&format!("{}/{}/{}", org_id, stream_type, stream_name)) - { - Ok(HttpResponse::Ok().json(val.value())) - } else { - Ok(HttpResponse::Ok().json(StreamFunctionsList { list: vec![] })) - } -} - -pub async fn delete_stream_function( - org_id: &str, - stream_type: StreamType, - stream_name: &str, - fn_name: &str, -) -> Result { - let mut existing_fn = match check_existing_fn(org_id, fn_name).await { - Some(function) => function, - None => { - return Ok(HttpResponse::NotFound().json(MetaHttpResponse::error( - StatusCode::NOT_FOUND.into(), - FN_NOT_FOUND.to_string(), - ))); - } - }; - - if let Some(ref mut val) = existing_fn.streams { - for stream in val.iter_mut() { - if stream.stream == stream_name && stream.stream_type == stream_type { - stream.is_removed = true; - stream.order = 0; - break; - } - } - if let Err(error) = db::functions::set(org_id, fn_name, &existing_fn).await { - Ok( - HttpResponse::InternalServerError().json(MetaHttpResponse::message( - http::StatusCode::INTERNAL_SERVER_ERROR.into(), - error.to_string(), - )), - ) - } else { - // can't be removed from watcher of function as stream name & type won't be - // available , hence being removed here - // let key = format!("{}/{}/{}", org_id, stream_type, stream_name); - // remove_stream_fn_from_cache(&key, fn_name); - Ok(HttpResponse::Ok().json(MetaHttpResponse::message( - http::StatusCode::OK.into(), - FN_REMOVED.to_string(), - ))) - } - } else { - Ok(HttpResponse::NotFound().json(MetaHttpResponse::error( - StatusCode::NOT_FOUND.into(), - FN_NOT_FOUND.to_string(), - ))) - } -} - -pub async fn add_function_to_stream( - org_id: &str, - stream_type: StreamType, - stream_name: &str, - fn_name: &str, - mut stream_order: StreamOrder, -) -> Result { - let mut existing_fn = match check_existing_fn(org_id, fn_name).await { - Some(function) => function, - None => { - return Ok(HttpResponse::NotFound().json(MetaHttpResponse::error( - StatusCode::NOT_FOUND.into(), - FN_NOT_FOUND.to_string(), - ))); - } - }; - - stream_order.stream = stream_name.to_owned(); - stream_order.stream_type = stream_type; - - if let Some(mut val) = existing_fn.streams { - if let Some(existing) = val.iter_mut().find(|x| { - x.stream == stream_order.stream && x.stream_type.eq(&stream_order.stream_type) - }) { - existing.is_removed = false; - existing.order = stream_order.order; - existing.apply_before_flattening = stream_order.apply_before_flattening; - } else { - val.push(stream_order); - } - existing_fn.streams = Some(val); - } else { - existing_fn.streams = Some(vec![stream_order]); - } - - if let Err(error) = db::functions::set(org_id, fn_name, &existing_fn).await { - Ok( - HttpResponse::InternalServerError().json(MetaHttpResponse::message( - http::StatusCode::INTERNAL_SERVER_ERROR.into(), - error.to_string(), - )), - ) - } else { - Ok(HttpResponse::Ok().json(MetaHttpResponse::message( - http::StatusCode::OK.into(), - FN_ADDED.to_string(), - ))) - } -} - fn extract_num_args(func: &mut Transform) { if func.trans_type.unwrap() == 1 { let src: String = func.function.to_owned(); @@ -342,24 +217,10 @@ async fn check_existing_fn(org_id: &str, fn_name: &str) -> Option { } } -fn _remove_stream_fn_from_cache(key: &str, fn_name: &str) { - if let Some(val) = STREAM_FUNCTIONS.clone().get(key) { - if val.list.len() > 1 { - let final_list = val - .clone() - .list - .into_iter() - .filter(|x| x.transform.name != fn_name) - .collect::>(); - STREAM_FUNCTIONS.insert(key.to_string(), StreamFunctionsList { list: final_list }); - } else { - STREAM_FUNCTIONS.remove(key); - } - } -} - #[cfg(test)] mod tests { + use config::meta::{function::StreamOrder, stream::StreamType}; + use super::*; #[tokio::test] diff --git a/src/service/ingestion/mod.rs b/src/service/ingestion/mod.rs index 271f214635..63619e8c8c 100644 --- a/src/service/ingestion/mod.rs +++ b/src/service/ingestion/mod.rs @@ -25,9 +25,10 @@ use config::{ get_config, ider::SnowflakeIdGenerator, meta::{ + alerts::alert::Alert, + function::{VRLResultResolver, VRLRuntimeConfig}, stream::{ - PartitionTimeLevel, PartitioningDetails, Routing, StreamParams, StreamPartition, - StreamType, + PartitionTimeLevel, PartitioningDetails, StreamParams, StreamPartition, StreamType, }, usage::{RequestStats, TriggerData, TriggerDataStatus, TriggerDataType}, }, @@ -38,27 +39,21 @@ use futures::future::try_join_all; use infra::schema::STREAM_RECORD_ID_GENERATOR; use proto::cluster_rpc::IngestionType; use tokio::sync::Semaphore; -use vector_enrichment::TableRegistry; use vrl::{ compiler::{runtime::Runtime, CompilationResult, TargetValueRef}, prelude::state, }; -use super::usage::publish_triggers_usage; +use super::{ + db::pipeline, pipeline::batch_execution::ExecutablePipeline, usage::publish_triggers_usage, +}; use crate::{ common::{ - infra::config::{ - REALTIME_ALERT_TRIGGERS, STREAM_ALERTS, STREAM_FUNCTIONS, STREAM_PIPELINES, - }, - meta::{ - alerts::alert::Alert, - functions::{StreamTransform, VRLResultResolver, VRLRuntimeConfig}, - ingestion::IngestionRequest, - stream::SchemaRecords, - }, + infra::config::{REALTIME_ALERT_TRIGGERS, STREAM_ALERTS}, + meta::{ingestion::IngestionRequest, stream::SchemaRecords}, utils::functions::get_vrl_compiler_config, }, - service::db, + service::{alerts::alert::AlertExt, db}, }; pub mod grpc; @@ -101,13 +96,13 @@ pub fn compile_vrl_function(func: &str, org_id: &str) -> Result Value { let mut metadata = vrl::value::Value::from(BTreeMap::new()); let mut target = TargetValueRef { - value: &mut vrl::value::Value::from(row), + value: &mut vrl::value::Value::from(&row), metadata: &mut metadata, secrets: &mut vrl::value::Secrets::new(), }; @@ -127,7 +122,7 @@ pub fn apply_vrl_fn( stream_name, err, ); - row.clone() + row } }, Err(err) => { @@ -137,43 +132,11 @@ pub fn apply_vrl_fn( stream_name, err, ); - row.clone() + row } } } -pub async fn get_stream_functions<'a>( - streams: &[StreamParams], - stream_before_functions_map: &mut HashMap>, - stream_after_functions_map: &mut HashMap>, - stream_vrl_map: &mut HashMap, -) { - for stream in streams { - let key = format!( - "{}/{}/{}", - stream.org_id, stream.stream_type, stream.stream_name - ); - if stream_after_functions_map.contains_key(&key) - || stream_before_functions_map.contains_key(&key) - { - // functions for this stream already fetched - continue; - } - // let mut _local_trans: Vec = vec![]; - // let local_stream_vrl_map; - let (before_local_trans, after_local_trans, local_stream_vrl_map) = - crate::service::ingestion::register_stream_functions( - &stream.org_id, - &stream.stream_type, - &stream.stream_name, - ); - stream_vrl_map.extend(local_stream_vrl_map); - - stream_before_functions_map.insert(key.clone(), before_local_trans); - stream_after_functions_map.insert(key, after_local_trans); - } -} - pub async fn get_stream_partition_keys( org_id: &str, stream_type: &StreamType, @@ -188,6 +151,15 @@ pub async fn get_stream_partition_keys( } } +pub async fn get_stream_executable_pipeline( + org_id: &str, + stream_name: &str, + stream_type: &StreamType, +) -> Option { + let stream_params = StreamParams::new(org_id, stream_name, *stream_type); + pipeline::get_executable_pipeline(&stream_params).await +} + pub async fn get_stream_alerts( streams: &[StreamParams], stream_alerts_map: &mut HashMap>, @@ -353,74 +325,6 @@ pub fn get_write_partition_key( time_key } -pub fn register_stream_functions( - org_id: &str, - stream_type: &StreamType, - stream_name: &str, -) -> ( - Vec, - Vec, - HashMap, -) { - let mut before_local_trans = vec![]; - let mut after_local_trans = vec![]; - let mut stream_vrl_map: HashMap = HashMap::new(); - let key = format!("{}/{}/{}", org_id, stream_type, stream_name); - - if let Some(transforms) = STREAM_FUNCTIONS.get(&key) { - (before_local_trans, after_local_trans) = (*transforms.list) - .iter() - .cloned() - .partition(|elem| elem.apply_before_flattening); - before_local_trans.sort_by(|a, b| a.order.cmp(&b.order)); - after_local_trans.sort_by(|a, b| a.order.cmp(&b.order)); - for trans in before_local_trans.iter().chain(after_local_trans.iter()) { - let func_key = format!("{}/{}", &stream_name, trans.transform.name); - if let Ok(vrl_runtime_config) = compile_vrl_function(&trans.transform.function, org_id) - { - let registry = vrl_runtime_config - .config - .get_custom::() - .unwrap(); - registry.finish_load(); - stream_vrl_map.insert( - func_key, - VRLResultResolver { - program: vrl_runtime_config.program, - fields: vrl_runtime_config.fields, - }, - ); - } - } - } - - (before_local_trans, after_local_trans, stream_vrl_map) -} - -pub fn apply_stream_functions( - local_trans: &[StreamTransform], - mut value: Value, - stream_vrl_map: &HashMap, - org_id: &str, - stream_name: &str, - runtime: &mut Runtime, -) -> Result { - for trans in local_trans { - let func_key = format!("{stream_name}/{}", trans.transform.name); - if stream_vrl_map.contains_key(&func_key) && !value.is_null() { - let vrl_runtime = stream_vrl_map.get(&func_key).unwrap(); - value = apply_vrl_fn( - runtime, - vrl_runtime, - &value, - org_id, - &[stream_name.to_string()], - ); - } - } - flatten::flatten_with_level(value, get_config().limit.ingest_flatten_level) -} - pub fn init_functions_runtime() -> Runtime { crate::common::utils::functions::init_vrl_runtime() } @@ -592,30 +496,6 @@ pub fn get_val_with_type_retained(val: &Value) -> Value { Value::Null => Value::Null, } } - -pub async fn get_stream_routing( - stream_params: StreamParams, - stream_routing_map: &mut HashMap>, -) { - if let Some(pipeline) = STREAM_PIPELINES.get(&format!( - "{}/{}/{}", - &stream_params.org_id, stream_params.stream_type, &stream_params.stream_name, - )) { - let Some(routing) = pipeline.routing.as_ref() else { - return; - }; - let res: Vec = routing - .iter() - .map(|(k, v)| Routing { - destination: k.to_string(), - routing: v.clone(), - }) - .collect(); - - stream_routing_map.insert(stream_params.stream_name.to_string(), res); - } -} - pub async fn get_uds_and_original_data_streams( streams: &[StreamParams], user_defined_schema_map: &mut HashMap>, diff --git a/src/service/logs/bulk.rs b/src/service/logs/bulk.rs index 794bf0e908..2a3b7dd35d 100644 --- a/src/service/logs/bulk.rs +++ b/src/service/logs/bulk.rs @@ -24,7 +24,7 @@ use chrono::{Duration, Utc}; use config::{ get_config, meta::{ - stream::{Routing, StreamParams, StreamType}, + stream::{StreamParams, StreamType}, usage::UsageType, }, metrics, @@ -33,18 +33,19 @@ use config::{ }; use crate::{ - common::meta::{ - functions::{StreamTransform, VRLResultResolver}, - ingestion::{BulkResponse, BulkResponseError, BulkResponseItem, IngestionStatus}, - }, + common::meta::ingestion::{BulkResponse, BulkResponseError, BulkResponseItem, IngestionStatus}, service::{ - format_stream_name, ingestion::check_ingestion_allowed, schema::get_upto_discard_error, + format_stream_name, + ingestion::check_ingestion_allowed, + pipeline::batch_execution::{ExecutablePipeline, ExecutablePipelineBulkInputs}, + schema::get_upto_discard_error, }, }; pub const TRANSFORM_FAILED: &str = "document_failed_transform"; pub const TS_PARSE_FAILED: &str = "timestamp_parsing_failed"; pub const SCHEMA_CONFORMANCE_FAILED: &str = "schema_conformance_failed"; +pub const PIPELINE_EXEC_FAILED: &str = "pipeline_execution_failed"; pub async fn ingest( thread_id: usize, @@ -69,18 +70,16 @@ pub async fn ingest( let min_ts = (Utc::now() - Duration::try_hours(cfg.limit.ingest_allowed_upto).unwrap()) .timestamp_micros(); - let mut runtime = crate::service::ingestion::init_functions_runtime(); - - let mut stream_vrl_map: HashMap = HashMap::new(); - let mut stream_before_functions_map: HashMap> = HashMap::new(); - let mut stream_after_functions_map: HashMap> = HashMap::new(); - let mut action = String::from(""); let mut stream_name = String::from(""); let mut doc_id = None; let mut blocked_stream_warnings: HashMap = HashMap::new(); - let mut stream_routing_map: HashMap> = HashMap::new(); + + let mut stream_executable_pipelines: HashMap> = + HashMap::new(); + let mut stream_pipeline_inputs: HashMap = HashMap::new(); + let mut user_defined_schema_map: HashMap> = HashMap::new(); let mut streams_need_original_set: HashSet = HashSet::new(); @@ -118,33 +117,27 @@ pub async fn ingest( continue; // skip } - // Start get routing keys - crate::service::ingestion::get_stream_routing( - StreamParams { - org_id: org_id.to_owned().into(), - stream_type: StreamType::Logs, - stream_name: stream_name.to_owned().into(), - }, - &mut stream_routing_map, - ) - .await; - let mut streams = vec![StreamParams { org_id: org_id.to_owned().into(), stream_type: StreamType::Logs, stream_name: stream_name.to_owned().into(), }]; - if let Some(routes) = stream_routing_map.get(&stream_name) { - for route in routes { - streams.push(StreamParams { - org_id: org_id.to_owned().into(), - stream_type: StreamType::Logs, - stream_name: route.destination.clone().into(), - }); + // Start retrieve associated pipeline and initialize ExecutablePipeline + if !stream_executable_pipelines.contains_key(&stream_name) { + let exec_pl_option = crate::service::ingestion::get_stream_executable_pipeline( + org_id, + &stream_name, + &StreamType::Logs, + ) + .await; + if let Some(exec_pl) = &exec_pl_option { + let pl_destinations = exec_pl.get_all_destination_streams(); + streams.extend(pl_destinations); } + stream_executable_pipelines.insert(stream_name.clone(), exec_pl_option); } - // End get stream keys + // End pipeline params construction crate::service::ingestion::get_uds_and_original_data_streams( &streams, @@ -154,217 +147,265 @@ pub async fn ingest( .await; next_line_is_data = true; - - // Start Register functions for stream - crate::service::ingestion::get_stream_functions( - &streams, - &mut stream_before_functions_map, - &mut stream_after_functions_map, - &mut stream_vrl_map, - ) - .await; - // End Register functions for index } else { next_line_is_data = false; - // store a copy of original data before it's being transformed and/or flattened, unless + // store a copy of original data before it's being transformed and/or flattened, when // 1. original data is not an object -> won't be flattened. - // 2. no routing and current StreamName not in streams_need_original_set let original_data = if value.is_object() { - if stream_routing_map.is_empty() - && !streams_need_original_set.contains(&stream_name) + // 2. current stream does not have pipeline + if stream_executable_pipelines + .get(&stream_name) + .unwrap() + .is_none() { - None + // current stream requires original + streams_need_original_set + .contains(&stream_name) + .then_some(value.to_string()) } else { - // otherwise, make a copy in case the routed stream needs original data - Some(value.to_string()) + // 3. with pipeline, storing original as long as streams_need_original_set is + // not empty + // because not sure the pipeline destinations + (!streams_need_original_set.is_empty()).then_some(value.to_string()) } } else { None // `item` won't be flattened, no need to store original }; - let main_stream_key = format!("{org_id}/{}/{stream_name}", StreamType::Logs); - // Start row based transform before flattening the value - if let Some(transforms) = stream_before_functions_map.get(&main_stream_key) { - if !transforms.is_empty() { - let mut ret_value = value.clone(); - ret_value = crate::service::ingestion::apply_stream_functions( - transforms, - ret_value, - &stream_vrl_map, + if stream_executable_pipelines + .get(&stream_name) + .unwrap() + .is_some() + { + // current stream has pipeline. buff the record for batch processing later + let inputs = stream_pipeline_inputs + .entry(stream_name.clone()) + .or_default(); + inputs.add_input(value, doc_id.to_owned(), original_data); + } else { + // JSON Flattening + value = flatten::flatten_with_level(value, cfg.limit.ingest_flatten_level)?; + + // get json object + let mut local_val = match value.take() { + json::Value::Object(v) => v, + _ => unreachable!(), + }; + + // set _id + if let Some(doc_id) = &doc_id { + local_val.insert("_id".to_string(), json::Value::String(doc_id.to_owned())); + } + + if let Some(fields) = user_defined_schema_map.get(&stream_name) { + local_val = crate::service::logs::refactor_map(local_val, fields); + } + + // add `_original` and '_record_id` if required by StreamSettings + if streams_need_original_set.contains(&stream_name) && original_data.is_some() { + local_val.insert( + ORIGINAL_DATA_COL_NAME.to_string(), + original_data.unwrap().into(), + ); + let record_id = crate::service::ingestion::generate_record_id( org_id, &stream_name, - &mut runtime, - )?; - - if ret_value.is_null() || !ret_value.is_object() { - bulk_res.errors = true; - add_record_status( - stream_name.clone(), - &doc_id, - action.clone(), - Some(value), - &mut bulk_res, - Some(TRANSFORM_FAILED.to_owned()), - Some(TRANSFORM_FAILED.to_owned()), - ); - continue; - } else { - value = ret_value; - } + &StreamType::Logs, + ); + local_val.insert( + ID_COL_NAME.to_string(), + json::Value::String(record_id.to_string()), + ); } - } - // end row based transformation - // JSON Flattening - let mut value = flatten::flatten_with_level(value, cfg.limit.ingest_flatten_level)?; - - let mut routed_stream_name = stream_name.clone(); - // Start re-routing if exists - if let Some(routing) = stream_routing_map.get(&stream_name) { - if !routing.is_empty() { - for route in routing { - let mut is_routed = true; - let val = &route.routing; - for q_condition in val.iter() { - if !q_condition.evaluate(value.as_object().unwrap()).await { - is_routed = false; - break; - } + // handle timestamp + let timestamp = match local_val.get(&cfg.common.column_timestamp) { + Some(v) => match parse_timestamp_micro_from_value(v) { + Ok(t) => t, + Err(_e) => { + bulk_res.errors = true; + add_record_status( + stream_name.clone(), + &doc_id, + action.clone(), + Some(value), + &mut bulk_res, + Some(TS_PARSE_FAILED.to_string()), + Some(TS_PARSE_FAILED.to_string()), + ); + continue; } - if is_routed && !val.is_empty() { - routed_stream_name = route.destination.clone(); - break; - } - } + }, + None => Utc::now().timestamp_micros(), + }; + + // check ingestion time + if timestamp < min_ts { + bulk_res.errors = true; + let failure_reason = Some(get_upto_discard_error().to_string()); + add_record_status( + stream_name.clone(), + &doc_id, + action.clone(), + Some(value), + &mut bulk_res, + Some(TS_PARSE_FAILED.to_string()), + failure_reason, + ); + continue; } - } - // End re-routing - - let key = format!("{org_id}/{}/{routed_stream_name}", StreamType::Logs); - - // Start row based transform - if let Some(transforms) = stream_after_functions_map.get(&key) { - if !transforms.is_empty() { - let mut ret_value = value.clone(); - ret_value = crate::service::ingestion::apply_stream_functions( - transforms, - ret_value, - &stream_vrl_map, - org_id, - &routed_stream_name, - &mut runtime, - )?; - - if ret_value.is_null() || !ret_value.is_object() { - bulk_res.errors = true; - add_record_status( - routed_stream_name.clone(), - &doc_id, - action.clone(), - Some(value), - &mut bulk_res, - Some(TRANSFORM_FAILED.to_owned()), - Some(TRANSFORM_FAILED.to_owned()), - ); - continue; - } else { - value = ret_value; - } - } - } - // End row based transform - - // get json object - let mut local_val = match value.take() { - json::Value::Object(v) => v, - _ => unreachable!(), - }; - - // set _id - if let Some(doc_id) = &doc_id { - local_val.insert("_id".to_string(), json::Value::String(doc_id.to_owned())); - } - - if let Some(fields) = user_defined_schema_map.get(&routed_stream_name) { - local_val = crate::service::logs::refactor_map(local_val, fields); - } - - // add `_original` and '_record_id` if required by StreamSettings - if streams_need_original_set.contains(&routed_stream_name) && original_data.is_some() { local_val.insert( - ORIGINAL_DATA_COL_NAME.to_string(), - original_data.unwrap().into(), - ); - let record_id = crate::service::ingestion::generate_record_id( - org_id, - &routed_stream_name, - &StreamType::Logs, - ); - local_val.insert( - ID_COL_NAME.to_string(), - json::Value::String(record_id.to_string()), + cfg.common.column_timestamp.clone(), + json::Value::Number(timestamp.into()), ); + + let (ts_data, fn_num) = json_data_by_stream + .entry(stream_name.clone()) + .or_insert((Vec::new(), None)); + ts_data.push((timestamp, local_val)); + *fn_num = Some(0); // no pl -> no func } - - // handle timestamp - let timestamp = match local_val.get(&cfg.common.column_timestamp) { - Some(v) => match parse_timestamp_micro_from_value(v) { - Ok(t) => t, - Err(_e) => { - bulk_res.errors = true; - add_record_status( - routed_stream_name.clone(), - &doc_id, - action.clone(), - Some(value), - &mut bulk_res, - Some(TS_PARSE_FAILED.to_string()), - Some(TS_PARSE_FAILED.to_string()), - ); - continue; - } - }, - None => Utc::now().timestamp_micros(), - }; - - // check ingestion time - if timestamp < min_ts { - bulk_res.errors = true; - let failure_reason = Some(get_upto_discard_error().to_string()); - add_record_status( - routed_stream_name.clone(), - &doc_id, - action.clone(), - Some(value), - &mut bulk_res, - Some(TS_PARSE_FAILED.to_string()), - failure_reason, - ); - continue; - } - local_val.insert( - cfg.common.column_timestamp.clone(), - json::Value::Number(timestamp.into()), - ); - - let fns_length = stream_before_functions_map - .get(&main_stream_key) - .map(|v| v.len()) - .unwrap_or_default() - + stream_after_functions_map - .get(&key) - .map(|v| v.len()) - .unwrap_or_default(); - - let (ts_data, fn_num) = json_data_by_stream - .entry(routed_stream_name.clone()) - .or_insert((Vec::new(), None)); - ts_data.push((timestamp, local_val)); - *fn_num = Some(fns_length); } } + // batch process records through pipeline + for (stream_name, exec_pl_option) in stream_executable_pipelines { + if let Some(exec_pl) = exec_pl_option { + let Some(pipeline_inputs) = stream_pipeline_inputs.remove(&stream_name) else { + log::error!( + "[Ingestion]: Stream {} has pipeline, but inputs failed to be buffered. BUG", + stream_name + ); + continue; + }; + let (records, doc_ids, originals) = pipeline_inputs.into_parts(); + match exec_pl.process_batch(org_id, records).await { + Err(e) => { + log::error!( + "[Pipeline] for stream {}/{}: Batch execution error: {}.", + org_id, + stream_name, + e + ); + bulk_res.errors = true; + add_record_status( + stream_name.clone(), + &None, + action.clone(), + None, + &mut bulk_res, + Some(PIPELINE_EXEC_FAILED.to_string()), + Some(PIPELINE_EXEC_FAILED.to_string()), + ); + continue; + } + Ok(pl_results) => { + let function_no = exec_pl.num_of_func(); + for (stream_params, stream_pl_results) in pl_results { + if stream_params.stream_type != StreamType::Logs { + continue; + } + + for (idx, mut res) in stream_pl_results { + // get json object + let mut local_val = match res.take() { + json::Value::Object(v) => v, + _ => unreachable!(), + }; + + // set _id + if let Some(doc_id) = &doc_ids[idx] { + local_val.insert( + "_id".to_string(), + json::Value::String(doc_id.to_owned()), + ); + } + + if let Some(fields) = + user_defined_schema_map.get(stream_params.stream_name.as_str()) + { + local_val = crate::service::logs::refactor_map(local_val, fields); + } + + // add `_original` and '_record_id` if required by StreamSettings + if streams_need_original_set + .contains(stream_params.stream_name.as_str()) + && originals[idx].is_some() + { + local_val.insert( + ORIGINAL_DATA_COL_NAME.to_string(), + originals[idx].clone().unwrap().into(), + ); + let record_id = crate::service::ingestion::generate_record_id( + org_id, + &stream_params.stream_name, + &StreamType::Logs, + ); + local_val.insert( + ID_COL_NAME.to_string(), + json::Value::String(record_id.to_string()), + ); + } + + // handle timestamp + let timestamp = match local_val.get(&cfg.common.column_timestamp) { + Some(v) => match parse_timestamp_micro_from_value(v) { + Ok(t) => t, + Err(_e) => { + bulk_res.errors = true; + add_record_status( + stream_params.stream_name.to_string(), + &doc_ids[idx], + action.clone(), + Some(res), + &mut bulk_res, + Some(TS_PARSE_FAILED.to_string()), + Some(TS_PARSE_FAILED.to_string()), + ); + continue; + } + }, + None => Utc::now().timestamp_micros(), + }; + + // check ingestion time + if timestamp < min_ts { + bulk_res.errors = true; + let failure_reason = Some(get_upto_discard_error().to_string()); + add_record_status( + stream_params.stream_name.to_string(), + &doc_ids[idx], + action.clone(), + Some(res), + &mut bulk_res, + Some(TS_PARSE_FAILED.to_string()), + failure_reason, + ); + continue; + } + local_val.insert( + cfg.common.column_timestamp.clone(), + json::Value::Number(timestamp.into()), + ); + + let (ts_data, fn_num) = json_data_by_stream + .entry(stream_params.stream_name.to_string()) + .or_insert((Vec::new(), None)); + ts_data.push((timestamp, local_val)); + *fn_num = Some(function_no) + } + } + } + } + } + } + + // drop memory-intensive variables + drop(stream_pipeline_inputs); + drop(streams_need_original_set); + drop(user_defined_schema_map); + let (metric_rpt_status_code, response_body) = { let mut status = IngestionStatus::Bulk(bulk_res); let write_result = super::write_logs_by_stream( diff --git a/src/service/logs/ingest.rs b/src/service/logs/ingest.rs index 84af0e38b3..57111a5d73 100644 --- a/src/service/logs/ingest.rs +++ b/src/service/logs/ingest.rs @@ -24,7 +24,7 @@ use chrono::{Duration, Utc}; use config::{ get_config, meta::{ - stream::{Routing, StreamParams, StreamType}, + stream::{StreamParams, StreamType}, usage::UsageType, }, metrics, @@ -39,16 +39,12 @@ use opentelemetry_proto::tonic::{ }; use prost::Message; use serde_json::json; -use vrl::compiler::runtime::Runtime; use crate::{ - common::meta::{ - functions::{StreamTransform, VRLResultResolver}, - ingestion::{ - AWSRecordType, GCPIngestionResponse, IngestionData, IngestionDataIter, IngestionError, - IngestionRequest, IngestionResponse, IngestionStatus, KinesisFHIngestionResponse, - StreamStatus, - }, + common::meta::ingestion::{ + AWSRecordType, GCPIngestionResponse, IngestionData, IngestionDataIter, IngestionError, + IngestionRequest, IngestionResponse, IngestionStatus, KinesisFHIngestionResponse, + StreamStatus, }, service::{ format_stream_name, get_formatted_stream_name, ingestion::check_ingestion_allowed, @@ -81,31 +77,23 @@ pub async fn ingest( let min_ts = (Utc::now() - Duration::try_hours(cfg.limit.ingest_allowed_upto).unwrap()) .timestamp_micros(); - let mut runtime = crate::service::ingestion::init_functions_runtime(); - let mut stream_vrl_map: HashMap = HashMap::new(); - let mut stream_before_functions_map: HashMap> = HashMap::new(); - let mut stream_after_functions_map: HashMap> = HashMap::new(); - let mut stream_params = vec![StreamParams::new(org_id, &stream_name, StreamType::Logs)]; - // Start get routing keys - let mut stream_routing_map: HashMap> = HashMap::new(); - crate::service::ingestion::get_stream_routing( - StreamParams::new(org_id, &stream_name, StreamType::Logs), - &mut stream_routing_map, + // Start retrieve associated pipeline and construct pipeline components + let executable_pipeline = crate::service::ingestion::get_stream_executable_pipeline( + org_id, + &stream_name, + &StreamType::Logs, ) .await; + let mut pipeline_inputs = Vec::new(); + let mut original_options = Vec::new(); + // End pipeline params construction - if let Some(routes) = stream_routing_map.get(&stream_name) { - for route in routes { - stream_params.push(StreamParams::new( - org_id, - &route.destination, - StreamType::Logs, - )); - } + if let Some(exec_pl) = &executable_pipeline { + let pl_destinations = exec_pl.get_all_destination_streams(); + stream_params.extend(pl_destinations); } - // End get routing keys // Start get user defined schema let mut user_defined_schema_map: HashMap> = HashMap::new(); @@ -118,16 +106,6 @@ pub async fn ingest( .await; // End get user defined schema - // Start Register functions for stream - crate::service::ingestion::get_stream_functions( - &stream_params, - &mut stream_before_functions_map, - &mut stream_after_functions_map, - &mut stream_vrl_map, - ) - .await; - // End Register functions for index - let json_req: Vec; // to hold json request because of borrow checker let (endpoint, usage_type, data) = match in_req { IngestionRequest::JSON(req) => { @@ -186,7 +164,6 @@ pub async fn ingest( return Err(anyhow::anyhow!("Failed processing: {:?}", e)); } }; - let mut routed_stream_name = stream_name.clone(); if let Some(extend) = extend_json.as_ref() { for (key, val) in extend.iter() { @@ -194,142 +171,149 @@ pub async fn ingest( } } - // store a copy of original data before it's being transformed and/or flattened, unless - // 1. original data is not an object -> won't be flattened. - // 2. no routing and current StreamName not in streams_need_original_set + // store a copy of original data before it's being transformed and/or flattened, when + // 1. original data is an object let original_data = if item.is_object() { - if stream_routing_map.is_empty() - && !streams_need_original_set.contains(&routed_stream_name) - { - None + // 2. current stream does not have pipeline + if executable_pipeline.is_none() { + // current stream requires original + streams_need_original_set + .contains(&stream_name) + .then_some(item.to_string()) } else { - // otherwise, make a copy in case the routed stream needs original data - Some(item.to_string()) + // 3. with pipeline, storing original as long as streams_need_original_set is not + // empty + // because not sure the pipeline destinations + (!streams_need_original_set.is_empty()).then_some(item.to_string()) } } else { None // `item` won't be flattened, no need to store original }; - let main_stream_key = format!("{org_id}/{}/{stream_name}", StreamType::Logs); - // Start row based transform before flattening the value - if let Some(transforms) = stream_before_functions_map.get(&main_stream_key) { - if !transforms.is_empty() { - item = match apply_functions( - item, - transforms, - &stream_vrl_map, + if executable_pipeline.is_some() { + // buffer the records and originals for pipeline batch processing + pipeline_inputs.push(item); + original_options.push(original_data); + } else { + // JSON Flattening + let mut res = flatten::flatten_with_level(item, cfg.limit.ingest_flatten_level)?; + // get json object + let mut local_val = match res.take() { + json::Value::Object(val) => val, + _ => unreachable!(), + }; + + if let Some(fields) = user_defined_schema_map.get(&stream_name) { + local_val = crate::service::logs::refactor_map(local_val, fields); + } + + // add `_original` and '_record_id` if required by StreamSettings + if streams_need_original_set.contains(&stream_name) && original_data.is_some() { + local_val.insert( + ORIGINAL_DATA_COL_NAME.to_string(), + original_data.unwrap().into(), + ); + let record_id = crate::service::ingestion::generate_record_id( org_id, - &routed_stream_name, - &mut runtime, - ) { - Ok(res) => res, - Err(e) => { - stream_status.status.failed += 1; - stream_status.status.error = e.to_string(); - continue; - } - } + &stream_name, + &StreamType::Logs, + ); + local_val.insert( + ID_COL_NAME.to_string(), + json::Value::String(record_id.to_string()), + ); } - } - // end row based transformation - // JSON Flattening - let item = flatten::flatten_with_level(item, cfg.limit.ingest_flatten_level)?; - - // Start re-routing if exists - if let Some(routings) = stream_routing_map.get(&routed_stream_name) { - if !routings.is_empty() { - for route in routings { - let mut is_routed = true; - let val = &route.routing; - for q_condition in val.iter() { - if !q_condition.evaluate(item.as_object().unwrap()).await { - is_routed = false; - break; - } - } - if !val.is_empty() && is_routed { - routed_stream_name = route.destination.clone(); - break; - } - } - } - } - // End re-routing - - let key = format!("{org_id}/{}/{routed_stream_name}", StreamType::Logs); - // Start row based transform - let mut res = if let Some(transforms) = stream_after_functions_map.get(&key) { - match apply_functions( - item, - transforms, - &stream_vrl_map, - org_id, - &routed_stream_name, - &mut runtime, - ) { - Ok(res) => res, + // handle timestamp + let timestamp = match handle_timestamp(&mut local_val, min_ts) { + Ok(ts) => ts, Err(e) => { stream_status.status.failed += 1; stream_status.status.error = e.to_string(); continue; } - } - } else { - item - }; - // end row based transform + }; - // get json object - let mut local_val = match res.take() { - json::Value::Object(val) => val, - _ => unreachable!(), - }; - - if let Some(fields) = user_defined_schema_map.get(&routed_stream_name) { - local_val = crate::service::logs::refactor_map(local_val, fields); + let (ts_data, fn_num) = json_data_by_stream + .entry(stream_name.clone()) + .or_insert_with(|| (Vec::new(), None)); + ts_data.push((timestamp, local_val)); + *fn_num = need_usage_report.then_some(0); // no pl -> no func } + } - // add `_original` and '_record_id` if required by StreamSettings - if streams_need_original_set.contains(&routed_stream_name) && original_data.is_some() { - local_val.insert( - ORIGINAL_DATA_COL_NAME.to_string(), - original_data.unwrap().into(), - ); - let record_id = crate::service::ingestion::generate_record_id( - org_id, - &routed_stream_name, - &StreamType::Logs, - ); - local_val.insert( - ID_COL_NAME.to_string(), - json::Value::String(record_id.to_string()), - ); - } - - // handle timestamp - let timestamp = match handle_timestamp(&mut local_val, min_ts) { - Ok(ts) => ts, + // batch process records through pipeline + if let Some(exec_pl) = &executable_pipeline { + let records_count = pipeline_inputs.len(); + match exec_pl.process_batch(org_id, pipeline_inputs).await { Err(e) => { - stream_status.status.failed += 1; - stream_status.status.error = e.to_string(); - continue; + log::error!( + "[Pipeline] for stream {}/{}: Batch execution error: {}.", + org_id, + stream_name, + e + ); + stream_status.status.failed += records_count as u32; + stream_status.status.error = format!("Pipeline batch execution error: {}", e); } - }; + Ok(pl_results) => { + let function_no = exec_pl.num_of_func(); + for (stream_params, stream_pl_results) in pl_results { + if stream_params.stream_type != StreamType::Logs { + continue; + } - let function_no = stream_before_functions_map - .get(&main_stream_key) - .map(|v| v.len()) - .unwrap_or_default() - + stream_after_functions_map - .get(&key) - .map(|v| v.len()) - .unwrap_or_default(); - let (ts_data, fn_num) = json_data_by_stream - .entry(routed_stream_name.clone()) - .or_insert((Vec::new(), None)); - ts_data.push((timestamp, local_val)); - *fn_num = need_usage_report.then_some(function_no); + for (idx, mut res) in stream_pl_results { + // get json object + let mut local_val = match res.take() { + json::Value::Object(val) => val, + _ => unreachable!(), + }; + + if let Some(fields) = + user_defined_schema_map.get(stream_params.stream_name.as_str()) + { + local_val = crate::service::logs::refactor_map(local_val, fields); + } + + // add `_original` and '_record_id` if required by StreamSettings + if streams_need_original_set.contains(stream_params.stream_name.as_str()) + && original_options[idx].is_some() + { + local_val.insert( + ORIGINAL_DATA_COL_NAME.to_string(), + original_options[idx].clone().unwrap().into(), + ); + let record_id = crate::service::ingestion::generate_record_id( + org_id, + &stream_params.stream_name, + &StreamType::Logs, + ); + local_val.insert( + ID_COL_NAME.to_string(), + json::Value::String(record_id.to_string()), + ); + } + + // handle timestamp + let timestamp = match handle_timestamp(&mut local_val, min_ts) { + Ok(ts) => ts, + Err(e) => { + stream_status.status.failed += 1; + stream_status.status.error = e.to_string(); + continue; + } + }; + + let (ts_data, fn_num) = json_data_by_stream + .entry(stream_params.stream_name.to_string()) + .or_insert_with(|| (Vec::new(), None)); + ts_data.push((timestamp, local_val)); + *fn_num = need_usage_report.then_some(function_no); + } + } + } + } } // if no data, fast return @@ -340,6 +324,12 @@ pub async fn ingest( )); } + // drop memory-intensive variables + drop(streams_need_original_set); + drop(executable_pipeline); + drop(original_options); + drop(user_defined_schema_map); + let (metric_rpt_status_code, response_body) = { let mut status = IngestionStatus::Record(stream_status.status); let write_result = super::write_logs_by_stream( @@ -386,45 +376,12 @@ pub async fn ingest( ]) .inc(); - // drop variables - drop(runtime); - drop(stream_vrl_map); - drop(stream_routing_map); - drop(user_defined_schema_map); - Ok(IngestionResponse::new( http::StatusCode::OK.into(), vec![response_body], )) } -pub fn apply_functions<'a>( - item: json::Value, - local_trans: &[StreamTransform], - stream_vrl_map: &'a HashMap, - org_id: &'a str, - stream_name: &'a str, - runtime: &mut Runtime, -) -> Result { - let mut value = item; - if !local_trans.is_empty() { - value = crate::service::ingestion::apply_stream_functions( - local_trans, - value, - stream_vrl_map, - org_id, - stream_name, - runtime, - )?; - } - - if value.is_null() || !value.is_object() { - Err(anyhow::Error::msg("apply functions failure")) - } else { - Ok(value) - } -} - pub fn handle_timestamp( local_val: &mut json::Map, min_ts: i64, diff --git a/src/service/logs/mod.rs b/src/service/logs/mod.rs index e020606532..f733415eb5 100644 --- a/src/service/logs/mod.rs +++ b/src/service/logs/mod.rs @@ -25,6 +25,7 @@ use arrow_schema::{DataType, Field}; use config::{ get_config, meta::{ + alerts::alert::Alert, stream::{PartitionTimeLevel, StreamParams, StreamPartition, StreamType}, usage::{RequestStats, UsageType}, }, @@ -42,9 +43,9 @@ use super::{ schema::stream_schema_exists, }; use crate::{ - common::meta::{alerts::alert::Alert, ingestion::IngestionStatus, stream::SchemaRecords}, + common::meta::{ingestion::IngestionStatus, stream::SchemaRecords}, service::{ - db, ingestion::get_write_partition_key, schema::check_for_schema, + alerts::alert::AlertExt, db, ingestion::get_write_partition_key, schema::check_for_schema, usage::report_request_usage_stats, }, }; @@ -57,7 +58,7 @@ pub mod syslog; static BULK_OPERATORS: [&str; 3] = ["create", "index", "update"]; -type LogJsonData = (Vec<(i64, Map)>, Option); +pub type O2IngestJsonData = (Vec<(i64, Map)>, Option); fn parse_bulk_index(v: &Value) -> Option<(String, String, Option)> { let local_val = v.as_object().unwrap(); @@ -190,7 +191,7 @@ async fn write_logs_by_stream( time_stats: (i64, &Instant), // started_at usage_type: UsageType, status: &mut IngestionStatus, - json_data_by_stream: HashMap, + json_data_by_stream: HashMap, ) -> Result<()> { for (stream_name, (json_data, fn_num)) in json_data_by_stream { // check if we are allowed to ingest diff --git a/src/service/logs/otlp_grpc.rs b/src/service/logs/otlp_grpc.rs index 955f7499e6..41de8c92d7 100644 --- a/src/service/logs/otlp_grpc.rs +++ b/src/service/logs/otlp_grpc.rs @@ -22,7 +22,7 @@ use chrono::{Duration, Utc}; use config::{ get_config, meta::{ - stream::{Routing, StreamParams, StreamType}, + stream::{StreamParams, StreamType}, usage::UsageType, }, metrics, @@ -36,10 +36,7 @@ use opentelemetry_proto::tonic::collector::logs::v1::{ use prost::Message; use crate::{ - common::meta::{ - functions::{StreamTransform, VRLResultResolver}, - ingestion::{IngestionStatus, StreamStatus}, - }, + common::meta::ingestion::{IngestionStatus, StreamStatus}, handler::http::request::CONTENT_TYPE_PROTO, service::{ format_stream_name, @@ -61,7 +58,6 @@ pub async fn handle_grpc_request( ) -> Result { let start = std::time::Instant::now(); let started_at = Utc::now().timestamp_micros(); - let cfg = get_config(); // check stream let stream_name = match in_stream_name { @@ -70,34 +66,28 @@ pub async fn handle_grpc_request( }; check_ingestion_allowed(org_id, Some(&stream_name))?; + let cfg = get_config(); let min_ts = (Utc::now() - Duration::try_hours(cfg.limit.ingest_allowed_upto).unwrap()) .timestamp_micros(); - let mut runtime = crate::service::ingestion::init_functions_runtime(); - let mut stream_vrl_map: HashMap = HashMap::new(); - let mut stream_before_functions_map: HashMap> = HashMap::new(); - let mut stream_after_functions_map: HashMap> = HashMap::new(); - let mut stream_params = vec![StreamParams::new(org_id, &stream_name, StreamType::Logs)]; - // Start get routing keys - let mut stream_routing_map: HashMap> = HashMap::new(); - crate::service::ingestion::get_stream_routing( - StreamParams::new(org_id, &stream_name, StreamType::Logs), - &mut stream_routing_map, + // Start retrieve associated pipeline and construct pipeline components + let executable_pipeline = crate::service::ingestion::get_stream_executable_pipeline( + org_id, + &stream_name, + &StreamType::Logs, ) .await; + let mut pipeline_inputs = Vec::new(); + let mut original_options = Vec::new(); + let mut timestamps = Vec::new(); + // End pipeline construction - if let Some(routes) = stream_routing_map.get(&stream_name) { - for route in routes { - stream_params.push(StreamParams::new( - org_id, - &route.destination, - StreamType::Logs, - )); - } + if let Some(exec_pl) = &executable_pipeline { + let pl_destinations = exec_pl.get_all_destination_streams(); + stream_params.extend(pl_destinations); } - // End get routing keys // Start get user defined schema let mut user_defined_schema_map: HashMap> = HashMap::new(); @@ -110,19 +100,13 @@ pub async fn handle_grpc_request( .await; // End get user defined schema - // Start Register functions for stream - crate::service::ingestion::get_stream_functions( - &stream_params, - &mut stream_before_functions_map, - &mut stream_after_functions_map, - &mut stream_vrl_map, - ) - .await; - // End Register functions for index - let mut stream_status = StreamStatus::new(&stream_name); let mut json_data_by_stream = HashMap::new(); + let mut res = ExportLogsServiceResponse { + partial_success: None, + }; + for resource_log in &request.resource_logs { for instrumentation_logs in &resource_log.scope_logs { for log_record in &instrumentation_logs.log_records { @@ -205,131 +189,149 @@ pub async fn handle_grpc_request( .into(); } }; - let main_stream_key = format!("{org_id}/{}/{stream_name}", StreamType::Logs); - // store a copy of original data before it's being transformed and/or flattened, - // unless - // 1. original data is not an object -> won't be flattened. - // 2. no routing and current StreamName not in streams_need_original_set + // store a copy of original data before it's modified, when + // 1. original data is an object let original_data = if rec.is_object() { - if stream_routing_map.is_empty() - && !streams_need_original_set.contains(&stream_name) - { - None + // 2. current stream does not have pipeline + if executable_pipeline.is_none() { + // current stream requires original + streams_need_original_set + .contains(&stream_name) + .then_some(rec.to_string()) } else { - // otherwise, make a copy in case the routed stream needs original data - Some(rec.to_string()) + // 3. with pipeline, storing original as long as streams_need_original_set + // is not empty + // because not sure the pipeline destinations + (!streams_need_original_set.is_empty()).then_some(rec.to_string()) } } else { None // `item` won't be flattened, no need to store original }; - // Start row based transform before flattening the value - if let Some(transforms) = stream_before_functions_map.get(&main_stream_key) { - if !transforms.is_empty() { - rec = crate::service::ingestion::apply_stream_functions( - transforms, - rec, - &stream_vrl_map, + if executable_pipeline.is_some() { + // buffer the records and originals for pipeline batch processing + pipeline_inputs.push(rec); + original_options.push(original_data); + timestamps.push(timestamp); + } else { + // flattening + rec = flatten::flatten_with_level(rec, cfg.limit.ingest_flatten_level)?; + + // get json object + let mut local_val = match rec.take() { + json::Value::Object(v) => v, + _ => unreachable!(), + }; + + if let Some(fields) = user_defined_schema_map.get(&stream_name) { + local_val = crate::service::logs::refactor_map(local_val, fields); + } + + // add `_original` and '_record_id` if required by StreamSettings + if streams_need_original_set.contains(&stream_name) && original_data.is_some() { + local_val.insert( + ORIGINAL_DATA_COL_NAME.to_string(), + original_data.unwrap().into(), + ); + let record_id = crate::service::ingestion::generate_record_id( org_id, &stream_name, - &mut runtime, - )?; + &StreamType::Logs, + ); + local_val.insert( + ID_COL_NAME.to_string(), + json::Value::String(record_id.to_string()), + ); } + + let (ts_data, fn_num) = json_data_by_stream + .entry(stream_name.clone()) + .or_insert((Vec::new(), None)); + ts_data.push((timestamp, local_val)); + *fn_num = Some(0); // no pl -> no func } - // end row based transformation - - // flattening - rec = flatten::flatten_with_level(rec, cfg.limit.ingest_flatten_level)?; - - let mut routed_stream_name = stream_name.clone(); - // Start re-routing if exists - if let Some(routings) = stream_routing_map.get(&stream_name) { - if !routings.is_empty() { - for route in routings { - let mut is_routed = true; - let val = &route.routing; - for q_condition in val.iter() { - if !q_condition.evaluate(rec.as_object().unwrap()).await { - is_routed = false; - break; - } - } - if !val.is_empty() && is_routed { - routed_stream_name = route.destination.clone(); - break; - } - } - } - } - // End re-routing - - let key = format!("{org_id}/{}/{routed_stream_name}", StreamType::Logs); - - // Start row based transform - if let Some(transforms) = stream_after_functions_map.get(&key) { - if !transforms.is_empty() { - rec = crate::service::ingestion::apply_stream_functions( - transforms, - rec, - &stream_vrl_map, - org_id, - &routed_stream_name, - &mut runtime, - )?; - } - } - // end row based transform - - // get json object - let mut local_val = match rec.take() { - json::Value::Object(v) => v, - _ => unreachable!(), - }; - - if let Some(fields) = user_defined_schema_map.get(&routed_stream_name) { - local_val = crate::service::logs::refactor_map(local_val, fields); - } - - // add `_original` and '_record_id` if required by StreamSettings - if streams_need_original_set.contains(&routed_stream_name) - && original_data.is_some() - { - local_val.insert( - ORIGINAL_DATA_COL_NAME.to_string(), - original_data.unwrap().into(), - ); - let record_id = crate::service::ingestion::generate_record_id( - org_id, - &routed_stream_name, - &StreamType::Logs, - ); - local_val.insert( - ID_COL_NAME.to_string(), - json::Value::String(record_id.to_string()), - ); - } - - let function_no = stream_before_functions_map - .get(&main_stream_key) - .map(|v| v.len()) - .unwrap_or_default() - + stream_after_functions_map - .get(&key) - .map(|v| v.len()) - .unwrap_or_default(); - let (ts_data, fn_num) = json_data_by_stream - .entry(routed_stream_name.clone()) - .or_insert((Vec::new(), None)); - ts_data.push((timestamp, local_val)); - *fn_num = Some(function_no); } } } - let mut res = ExportLogsServiceResponse { - partial_success: None, - }; + // batch process records through pipeline + if let Some(exec_pl) = &executable_pipeline { + let records_count = pipeline_inputs.len(); + match exec_pl.process_batch(org_id, pipeline_inputs).await { + Err(e) => { + log::error!( + "[Pipeline] for stream {}/{}: Batch execution error: {}.", + org_id, + stream_name, + e + ); + stream_status.status.failed += records_count as u32; + stream_status.status.error = format!("Pipeline batch execution error: {}", e); + } + Ok(pl_results) => { + let function_no = exec_pl.num_of_func(); + for (stream_params, stream_pl_results) in pl_results { + if stream_params.stream_type != StreamType::Logs { + continue; + } + + for (idx, mut res) in stream_pl_results { + // get json object + let mut local_val = match res.take() { + json::Value::Object(v) => v, + _ => unreachable!(), + }; + + if let Some(fields) = + user_defined_schema_map.get(stream_params.stream_name.as_str()) + { + local_val = crate::service::logs::refactor_map(local_val, fields); + } + + // add `_original` and '_record_id` if required by StreamSettings + if streams_need_original_set.contains(stream_params.stream_name.as_str()) + && original_options[idx].is_some() + { + local_val.insert( + ORIGINAL_DATA_COL_NAME.to_string(), + original_options[idx].clone().unwrap().into(), + ); + let record_id = crate::service::ingestion::generate_record_id( + org_id, + &stream_params.stream_name, + &StreamType::Logs, + ); + local_val.insert( + ID_COL_NAME.to_string(), + json::Value::String(record_id.to_string()), + ); + } + + let (ts_data, fn_num) = json_data_by_stream + .entry(stream_params.stream_name.to_string()) + .or_insert((Vec::new(), None)); + ts_data.push((timestamps[idx], local_val)); + *fn_num = Some(function_no); // no pl -> no func + } + } + } + } + } + + // drop variables + drop(executable_pipeline); + drop(original_options); + drop(timestamps); + drop(user_defined_schema_map); + + // Update partial success + if stream_status.status.failed > 0 { + res.partial_success = Some(ExportLogsPartialSuccess { + rejected_log_records: stream_status.status.failed as i64, + error_message: stream_status.status.error.clone(), + }); + } // if no data, fast return if json_data_by_stream.is_empty() { @@ -400,12 +402,6 @@ pub async fn handle_grpc_request( ]) .inc(); - // drop variables - drop(runtime); - drop(stream_vrl_map); - drop(stream_routing_map); - drop(user_defined_schema_map); - return Ok(HttpResponse::Ok() .status(http::StatusCode::OK) .content_type(CONTENT_TYPE_PROTO) diff --git a/src/service/logs/otlp_http.rs b/src/service/logs/otlp_http.rs index cdae0afbfe..503cc8c9f1 100644 --- a/src/service/logs/otlp_http.rs +++ b/src/service/logs/otlp_http.rs @@ -22,7 +22,7 @@ use chrono::{Duration, Utc}; use config::{ get_config, meta::{ - stream::{Routing, StreamParams, StreamType}, + stream::{StreamParams, StreamType}, usage::UsageType, }, metrics, @@ -37,7 +37,6 @@ use prost::Message; use crate::{ common::meta::{ - functions::{StreamTransform, VRLResultResolver}, http::HttpResponse as MetaHttpResponse, ingestion::{IngestionStatus, StreamStatus}, }, @@ -106,31 +105,24 @@ pub async fn logs_json_handler( let min_ts = (Utc::now() - Duration::try_hours(cfg.limit.ingest_allowed_upto).unwrap()) .timestamp_micros(); - let mut runtime = crate::service::ingestion::init_functions_runtime(); - let mut stream_vrl_map: HashMap = HashMap::new(); - let mut stream_before_functions_map: HashMap> = HashMap::new(); - let mut stream_after_functions_map: HashMap> = HashMap::new(); - let mut stream_params = vec![StreamParams::new(org_id, &stream_name, StreamType::Logs)]; - // Start get routing keys - let mut stream_routing_map: HashMap> = HashMap::new(); - crate::service::ingestion::get_stream_routing( - StreamParams::new(org_id, &stream_name, StreamType::Logs), - &mut stream_routing_map, + // Start retrieve associated pipeline and construct pipeline components + let executable_pipeline = crate::service::ingestion::get_stream_executable_pipeline( + org_id, + &stream_name, + &StreamType::Logs, ) .await; + let mut pipeline_inputs = Vec::new(); + let mut original_options = Vec::new(); + let mut timestamps = Vec::new(); + // End pipeline params construction - if let Some(routes) = stream_routing_map.get(&stream_name) { - for route in routes { - stream_params.push(StreamParams::new( - org_id, - &route.destination, - StreamType::Logs, - )); - } + if let Some(pl) = &executable_pipeline { + let pl_destinations = pl.get_all_destination_streams(); + stream_params.extend(pl_destinations); } - // End get routing keys // Start get user defined schema let mut user_defined_schema_map: HashMap> = HashMap::new(); @@ -143,16 +135,6 @@ pub async fn logs_json_handler( .await; // End get user defined schema - // Start Register functions for stream - crate::service::ingestion::get_stream_functions( - &stream_params, - &mut stream_before_functions_map, - &mut stream_after_functions_map, - &mut stream_vrl_map, - ) - .await; - // End Register functions for stream - let mut stream_status = StreamStatus::new(&stream_name); let mut json_data_by_stream = HashMap::new(); @@ -195,6 +177,10 @@ pub async fn logs_json_handler( }, }; + let mut res = ExportLogsServiceResponse { + partial_success: None, + }; + for res_log in logs.iter() { let mut service_att_map: json::Map = json::Map::new(); if res_log.get("resource").is_some() { @@ -328,133 +314,149 @@ pub async fn logs_json_handler( value = json::to_value(local_val)?; - // store a copy of original data before it's being transformed and/or flattened, - // unless - // 1. original data is not an object -> won't be flattened. - // 2. no routing and current StreamName not in streams_need_original_set + // store a copy of original data before it's modified, when + // 1. original data is an object let original_data = if value.is_object() { - if stream_routing_map.is_empty() - && !streams_need_original_set.contains(&stream_name) - { - None + // 2. current stream does not have pipeline + if executable_pipeline.is_none() { + // current stream requires original + streams_need_original_set + .contains(&stream_name) + .then_some(value.to_string()) } else { - // otherwise, make a copy in case the routed stream needs original data - Some(value.to_string()) + // 3. with pipeline, storing original as long as streams_need_original_set + // is not empty + // because not sure the pipeline destinations + (!streams_need_original_set.is_empty()).then_some(value.to_string()) } } else { None // `item` won't be flattened, no need to store original }; - let main_stream_key = format!("{org_id}/{}/{stream_name}", StreamType::Logs); + if executable_pipeline.is_some() { + // buffer the records and originals for pipeline batch processing + pipeline_inputs.push(value); + original_options.push(original_data); + timestamps.push(timestamp); + } else { + // JSON Flattening + value = + flatten::flatten_with_level(value, cfg.limit.ingest_flatten_level).unwrap(); - // Start row based transform before flattening the value - if let Some(transforms) = stream_before_functions_map.get(&main_stream_key) { - if !transforms.is_empty() { - value = crate::service::ingestion::apply_stream_functions( - transforms, - value, - &stream_vrl_map, + // get json object + let mut local_val = match value.take() { + json::Value::Object(v) => v, + _ => unreachable!(), + }; + + if let Some(fields) = user_defined_schema_map.get(&stream_name) { + local_val = crate::service::logs::refactor_map(local_val, fields); + } + + // add `_original` and '_record_id` if required by StreamSettings + if streams_need_original_set.contains(&stream_name) && original_data.is_some() { + local_val.insert( + ORIGINAL_DATA_COL_NAME.to_string(), + original_data.unwrap().into(), + ); + let record_id = crate::service::ingestion::generate_record_id( org_id, &stream_name, - &mut runtime, - ) - .unwrap(); + &StreamType::Logs, + ); + local_val.insert( + ID_COL_NAME.to_string(), + json::Value::String(record_id.to_string()), + ); } + + let (ts_data, fn_num) = json_data_by_stream + .entry(stream_name.clone()) + .or_insert((Vec::new(), None)); + ts_data.push((timestamp, local_val)); + *fn_num = Some(0); // no pl -> no func } - // end row based transformation - - // JSON Flattening - value = flatten::flatten_with_level(value, cfg.limit.ingest_flatten_level).unwrap(); - - let mut routed_stream_name = stream_name.clone(); - // Start re-routing if exists - if let Some(routing) = stream_routing_map.get(&stream_name) { - if !routing.is_empty() { - for route in routing { - let mut is_routed = true; - let val = &route.routing; - for q_condition in val.iter() { - if !q_condition.evaluate(value.as_object().unwrap()).await { - is_routed = false; - break; - } - } - if is_routed && !val.is_empty() { - routed_stream_name = route.destination.clone(); - break; - } - } - } - } - // End re-routing - - let key = format!("{org_id}/{}/{routed_stream_name}", StreamType::Logs); - - // Start row based transform - if let Some(transforms) = stream_after_functions_map.get(&key) { - if !transforms.is_empty() { - value = crate::service::ingestion::apply_stream_functions( - transforms, - value, - &stream_vrl_map, - org_id, - &routed_stream_name, - &mut runtime, - ) - .unwrap(); - } - } - // end row based transform - - // get json object - let mut local_val = match value.take() { - json::Value::Object(v) => v, - _ => unreachable!(), - }; - - if let Some(fields) = user_defined_schema_map.get(&routed_stream_name) { - local_val = crate::service::logs::refactor_map(local_val, fields); - } - - // add `_original` and '_record_id` if required by StreamSettings - if streams_need_original_set.contains(&routed_stream_name) - && original_data.is_some() - { - local_val.insert( - ORIGINAL_DATA_COL_NAME.to_string(), - original_data.unwrap().into(), - ); - let record_id = crate::service::ingestion::generate_record_id( - org_id, - &routed_stream_name, - &StreamType::Logs, - ); - local_val.insert( - ID_COL_NAME.to_string(), - json::Value::String(record_id.to_string()), - ); - } - - let function_no = stream_before_functions_map - .get(&main_stream_key) - .map(|v| v.len()) - .unwrap_or_default() - + stream_after_functions_map - .get(&key) - .map(|v| v.len()) - .unwrap_or_default(); - let (ts_data, fn_num) = json_data_by_stream - .entry(routed_stream_name.clone()) - .or_insert((Vec::new(), None)); - ts_data.push((timestamp, local_val)); - *fn_num = Some(function_no); } } } - let mut res = ExportLogsServiceResponse { - partial_success: None, - }; + // batch process records through pipeline + if let Some(exec_pl) = &executable_pipeline { + let records_count = pipeline_inputs.len(); + match exec_pl.process_batch(org_id, pipeline_inputs).await { + Err(e) => { + log::error!( + "[Pipeline] for stream {}/{}: Batch execution error: {}.", + org_id, + stream_name, + e + ); + stream_status.status.failed += records_count as u32; + stream_status.status.error = format!("Pipeline batch execution error: {}", e); + } + Ok(pl_results) => { + let function_no = exec_pl.num_of_func(); + for (stream_params, stream_pl_results) in pl_results { + if stream_params.stream_type != StreamType::Logs { + continue; + } + + for (idx, mut res) in stream_pl_results { + // get json object + let mut local_val = match res.take() { + json::Value::Object(v) => v, + _ => unreachable!(), + }; + + if let Some(fields) = + user_defined_schema_map.get(stream_params.stream_name.as_str()) + { + local_val = crate::service::logs::refactor_map(local_val, fields); + } + + // add `_original` and '_record_id` if required by StreamSettings + if streams_need_original_set.contains(stream_params.stream_name.as_str()) + && original_options[idx].is_some() + { + local_val.insert( + ORIGINAL_DATA_COL_NAME.to_string(), + original_options[idx].clone().unwrap().into(), + ); + let record_id = crate::service::ingestion::generate_record_id( + org_id, + &stream_params.stream_name, + &StreamType::Logs, + ); + local_val.insert( + ID_COL_NAME.to_string(), + json::Value::String(record_id.to_string()), + ); + } + + let (ts_data, fn_num) = json_data_by_stream + .entry(stream_params.stream_name.to_string()) + .or_insert((Vec::new(), None)); + ts_data.push((timestamps[idx], local_val)); + *fn_num = Some(function_no); // no pl -> no func + } + } + } + } + } + + // drop variables + drop(executable_pipeline); + drop(original_options); + drop(timestamps); + drop(user_defined_schema_map); + + // Update partial success + if stream_status.status.failed > 0 { + res.partial_success = Some(ExportLogsPartialSuccess { + rejected_log_records: stream_status.status.failed as i64, + error_message: stream_status.status.error.clone(), + }); + } // if no data, fast return if json_data_by_stream.is_empty() { @@ -520,12 +522,6 @@ pub async fn logs_json_handler( ]) .inc(); - // drop variables - drop(runtime); - drop(stream_vrl_map); - drop(stream_routing_map); - drop(user_defined_schema_map); - return Ok(HttpResponse::Ok() .status(http::StatusCode::OK) .content_type(CONTENT_TYPE_JSON) diff --git a/src/service/logs/syslog.rs b/src/service/logs/syslog.rs index 7fa488ad29..12df73376f 100644 --- a/src/service/logs/syslog.rs +++ b/src/service/logs/syslog.rs @@ -23,7 +23,10 @@ use anyhow::Result; use chrono::{Duration, Utc}; use config::{ get_config, - meta::stream::{Routing, StreamParams, StreamType}, + meta::{ + stream::{StreamParams, StreamType}, + usage::UsageType, + }, metrics, utils::{flatten, json}, ID_COL_NAME, ORIGINAL_DATA_COL_NAME, @@ -35,7 +38,6 @@ use crate::{ common::{ infra::config::SYSLOG_ROUTES, meta::{ - functions::{StreamTransform, VRLResultResolver}, http::HttpResponse as MetaHttpResponse, ingestion::{IngestionResponse, IngestionStatus, StreamStatus}, syslog::SyslogRoute, @@ -46,6 +48,7 @@ use crate::{ pub async fn ingest(msg: &str, addr: SocketAddr) -> Result { let start = std::time::Instant::now(); + let started_at: i64 = Utc::now().timestamp_micros(); let ip = addr.ip(); let matching_route = get_org_for_ip(ip).await; @@ -80,31 +83,23 @@ pub async fn ingest(msg: &str, addr: SocketAddr) -> Result { let min_ts = (Utc::now() - Duration::try_hours(cfg.limit.ingest_allowed_upto).unwrap()) .timestamp_micros(); - let mut runtime = crate::service::ingestion::init_functions_runtime(); - let mut stream_vrl_map: HashMap = HashMap::new(); - let mut stream_before_functions_map: HashMap> = HashMap::new(); - let mut stream_after_functions_map: HashMap> = HashMap::new(); - let mut stream_params = vec![StreamParams::new(org_id, &stream_name, StreamType::Logs)]; - // Start get routing keys - let mut stream_routing_map: HashMap> = HashMap::new(); - crate::service::ingestion::get_stream_routing( - StreamParams::new(org_id, &stream_name, StreamType::Logs), - &mut stream_routing_map, + // Start retrieve associated pipeline and construct pipeline components + let executable_pipeline = crate::service::ingestion::get_stream_executable_pipeline( + org_id, + &stream_name, + &StreamType::Logs, ) .await; + let mut pipeline_inputs = Vec::new(); + let mut original_options = Vec::new(); + // End pipeline construction - if let Some(routes) = stream_routing_map.get(&stream_name) { - for route in routes { - stream_params.push(StreamParams::new( - org_id, - &route.destination, - StreamType::Logs, - )); - } + if let Some(pl) = &executable_pipeline { + let pl_destinations = pl.get_all_destination_streams(); + stream_params.extend(pl_destinations); } - // End get routing keys // Start get user defined schema let mut user_defined_schema_map: HashMap> = HashMap::new(); @@ -117,153 +112,184 @@ pub async fn ingest(msg: &str, addr: SocketAddr) -> Result { .await; // End get user defined schema - // Start Register functions for stream - crate::service::ingestion::get_stream_functions( - &stream_params, - &mut stream_before_functions_map, - &mut stream_after_functions_map, - &mut stream_vrl_map, - ) - .await; - // End Register functions for stream - let mut stream_status = StreamStatus::new(&stream_name); + let mut json_data_by_stream = HashMap::new(); // parse msg to json::Value let parsed_msg = syslog_loose::parse_message(msg); let mut value = message_to_value(parsed_msg); - // store a copy of original data before it's being transformed and/or flattened, unless - // 1. original data is not an object -> won't be flattened. - // 2. no routing and current StreamName not in streams_need_original_set + // store a copy of original data before it's modified, when + // 1. original data is an object let original_data = if value.is_object() { - if stream_routing_map.is_empty() && !streams_need_original_set.contains(&stream_name) { - None + // 2. current stream does not have pipeline + if executable_pipeline.is_none() { + // current stream requires original + streams_need_original_set + .contains(&stream_name) + .then_some(value.to_string()) } else { - // otherwise, make a copy in case the routed stream needs original data - Some(value.to_string()) + // 3. with pipeline, storing original as long as streams_need_original_set is not empty + // because not sure the pipeline destinations + (!streams_need_original_set.is_empty()).then_some(value.to_string()) } } else { None // `item` won't be flattened, no need to store original }; - let main_stream_key = format!("{org_id}/{}/{stream_name}", StreamType::Logs); + if executable_pipeline.is_some() { + // buffer the records and originals for pipeline batch processing + pipeline_inputs.push(value); + original_options.push(original_data); + } else { + // JSON Flattening + value = flatten::flatten_with_level(value, cfg.limit.ingest_flatten_level).unwrap(); - // Start row based transform. Apply vrl functions with apply_before_flattening flag - if let Some(transforms) = stream_before_functions_map.get(&main_stream_key) { - if !transforms.is_empty() { - value = crate::service::ingestion::apply_stream_functions( - transforms, - value, - &stream_vrl_map, + // get json object + let mut local_val = match value.take() { + json::Value::Object(v) => v, + _ => unreachable!(), + }; + + if let Some(fields) = user_defined_schema_map.get(&stream_name) { + local_val = crate::service::logs::refactor_map(local_val, fields); + } + + // add `_original` and '_record_id` if required by StreamSettings + if streams_need_original_set.contains(&stream_name) && original_data.is_some() { + local_val.insert( + ORIGINAL_DATA_COL_NAME.to_string(), + original_data.unwrap().into(), + ); + let record_id = crate::service::ingestion::generate_record_id( org_id, &stream_name, - &mut runtime, - )?; + &StreamType::Logs, + ); + local_val.insert( + ID_COL_NAME.to_string(), + json::Value::String(record_id.to_string()), + ); } + + // handle timestamp + let timestamp = match handle_timestamp(&mut local_val, min_ts) { + Ok(ts) => ts, + Err(e) => { + stream_status.status.failed += 1; + stream_status.status.error = e.to_string(); + return Ok(HttpResponse::Ok().json(IngestionResponse::new( + http::StatusCode::OK.into(), + vec![stream_status], + ))); // just return + } + }; + + let (ts_data, fn_num) = json_data_by_stream + .entry(stream_name.clone()) + .or_insert((Vec::new(), None)); + ts_data.push((timestamp, local_val)); + *fn_num = Some(0); // no pl -> no func } - // end row based transformation - // JSON Flattening - value = flatten::flatten_with_level(value, cfg.limit.ingest_flatten_level).unwrap(); - - let mut routed_stream_name = stream_name.clone(); - // Start re-rerouting if exists - if let Some(routings) = stream_routing_map.get(&stream_name) { - if !routings.is_empty() { - for route in routings { - let mut is_routed = true; - let val = &route.routing; - for q_condition in val.iter() { - if !q_condition.evaluate(value.as_object().unwrap()).await { - is_routed = false; - break; + // batch process records through pipeline + if let Some(exec_pl) = &executable_pipeline { + let records_count = pipeline_inputs.len(); + match exec_pl.process_batch(org_id, pipeline_inputs).await { + Err(e) => { + log::error!( + "[Pipeline] for stream {}/{}: Batch execution error: {}.", + org_id, + stream_name, + e + ); + stream_status.status.failed += records_count as u32; + stream_status.status.error = format!("Pipeline batch execution error: {}", e); + } + Ok(pl_results) => { + let function_no = exec_pl.num_of_func(); + for (stream_params, stream_pl_results) in pl_results { + if stream_params.stream_type != StreamType::Logs { + continue; + } + + for (idx, mut res) in stream_pl_results { + // get json object + let mut local_val = match res.take() { + json::Value::Object(val) => val, + _ => unreachable!(), + }; + + if let Some(fields) = + user_defined_schema_map.get(stream_params.stream_name.as_str()) + { + local_val = crate::service::logs::refactor_map(local_val, fields); + } + + // add `_original` and '_record_id` if required by StreamSettings + if streams_need_original_set.contains(stream_params.stream_name.as_str()) + && original_options[idx].is_some() + { + local_val.insert( + ORIGINAL_DATA_COL_NAME.to_string(), + original_options[idx].clone().unwrap().into(), + ); + let record_id = crate::service::ingestion::generate_record_id( + org_id, + &stream_params.stream_name, + &StreamType::Logs, + ); + local_val.insert( + ID_COL_NAME.to_string(), + json::Value::String(record_id.to_string()), + ); + } + + // handle timestamp + let timestamp = match handle_timestamp(&mut local_val, min_ts) { + Ok(ts) => ts, + Err(e) => { + stream_status.status.failed += 1; + stream_status.status.error = e.to_string(); + continue; + } + }; + + let (ts_data, fn_num) = json_data_by_stream + .entry(stream_params.stream_name.to_string()) + .or_insert_with(|| (Vec::new(), None)); + ts_data.push((timestamp, local_val)); + *fn_num = Some(function_no); } - } - if !val.is_empty() && is_routed { - routed_stream_name = route.destination.clone(); - break; } } } } - // End re-routing - let key = format!("{org_id}/{}/{routed_stream_name}", StreamType::Logs); - - // Start row based transform - if let Some(transforms) = stream_after_functions_map.get(&key) { - if !transforms.is_empty() { - value = crate::service::ingestion::apply_stream_functions( - transforms, - value, - &stream_vrl_map, - org_id, - &routed_stream_name, - &mut runtime, - ) - .unwrap(); - } - } - // end row based transform - - if value.is_null() || !value.is_object() { - stream_status.status.failed += 1; // transform failed or dropped + // if no data, fast return + if json_data_by_stream.is_empty() { return Ok(HttpResponse::Ok().json(IngestionResponse::new( http::StatusCode::OK.into(), vec![stream_status], ))); // just return } - // End row based transform - // get json object - let mut local_val = match value.take() { - json::Value::Object(v) => v, - _ => unreachable!(), - }; - - if let Some(fields) = user_defined_schema_map.get(&routed_stream_name) { - local_val = crate::service::logs::refactor_map(local_val, fields); - } - - // add `_original` and '_record_id` if required by StreamSettings - if streams_need_original_set.contains(&routed_stream_name) && original_data.is_some() { - local_val.insert( - ORIGINAL_DATA_COL_NAME.to_string(), - original_data.unwrap().into(), - ); - let record_id = crate::service::ingestion::generate_record_id( - org_id, - &routed_stream_name, - &StreamType::Logs, - ); - local_val.insert( - ID_COL_NAME.to_string(), - json::Value::String(record_id.to_string()), - ); - } - - // handle timestamp - let timestamp = match handle_timestamp(&mut local_val, min_ts) { - Ok(ts) => ts, - Err(e) => { - stream_status.status.failed += 1; - stream_status.status.error = e.to_string(); - return Ok(HttpResponse::Ok().json(IngestionResponse::new( - http::StatusCode::OK.into(), - vec![stream_status], - ))); // just return - } - }; + // drop memory-intensive variables + drop(streams_need_original_set); + drop(executable_pipeline); + drop(original_options); + drop(user_defined_schema_map); let (metric_rpt_status_code, response_body) = { let mut status = IngestionStatus::Record(stream_status.status); - let write_result = super::write_logs( + let write_result = super::write_logs_by_stream( 0, org_id, - &routed_stream_name, + "", + (started_at, &start), + UsageType::Syslog, &mut status, - vec![(timestamp, local_val)], + json_data_by_stream, ) .await; stream_status.status = match status { @@ -299,12 +325,6 @@ pub async fn ingest(msg: &str, addr: SocketAddr) -> Result { ]) .inc(); - // drop variables - drop(runtime); - drop(stream_vrl_map); - drop(stream_routing_map); - drop(user_defined_schema_map); - Ok(HttpResponse::Ok().json(IngestionResponse::new( http::StatusCode::OK.into(), vec![response_body], diff --git a/src/service/metrics/json.rs b/src/service/metrics/json.rs index d1a332e73e..c33659f54f 100644 --- a/src/service/metrics/json.rs +++ b/src/service/metrics/json.rs @@ -20,7 +20,8 @@ use anyhow::{anyhow, Result}; use config::{ cluster::LOCAL_NODE, meta::{ - stream::{PartitioningDetails, StreamType}, + alerts::alert::Alert, + stream::{PartitioningDetails, StreamParams, StreamType}, usage::UsageType, }, metrics, @@ -28,7 +29,6 @@ use config::{ }; use datafusion::arrow::datatypes::Schema; use infra::schema::{unwrap_partition_time_level, SchemaCache}; -use vrl::compiler::runtime::Runtime; use super::get_exclude_labels; use crate::{ @@ -39,8 +39,10 @@ use crate::{ stream::SchemaRecords, }, service::{ + alerts::alert::AlertExt, db, format_stream_name, - ingestion::{get_write_partition_key, write_file}, + ingestion::{evaluate_trigger, get_write_partition_key, write_file, TriggerAlertData}, + pipeline::batch_execution::ExecutablePipeline, schema::check_for_schema, usage::report_request_usage_stats, }, @@ -72,12 +74,23 @@ pub async fn ingest(org_id: &str, body: web::Bytes) -> Result }); } - let mut runtime = crate::service::ingestion::init_functions_runtime(); let mut stream_schema_map: HashMap = HashMap::new(); let mut stream_status_map: HashMap = HashMap::new(); let mut stream_data_buf: HashMap> = HashMap::new(); let mut stream_partitioning_map: HashMap = HashMap::new(); + // associated pipeline + let mut stream_executable_pipelines: HashMap> = + HashMap::new(); + let mut stream_pipeline_inputs: HashMap> = HashMap::new(); + + // realtime alerts + let mut stream_alerts_map: HashMap> = HashMap::new(); + let mut stream_trigger_map: HashMap> = HashMap::new(); + + // records buffer + let mut json_data_by_stream: HashMap> = HashMap::new(); + let reader: Vec = json::from_slice(&body)?; for record in reader.into_iter() { // JSON Flattening @@ -97,6 +110,18 @@ pub async fn ingest(org_id: &str, body: web::Bytes) -> Result } }; + // Start retrieve associated pipeline and initialize ExecutablePipeline + if !stream_executable_pipelines.contains_key(&stream_name) { + let exec_pl_option = crate::service::ingestion::get_stream_executable_pipeline( + org_id, + &stream_name, + &StreamType::Metrics, + ) + .await; + stream_executable_pipelines.insert(stream_name.clone(), exec_pl_option); + } + // End pipeline params construction + // check metrics type for Histogram & Summary if metrics_type.to_lowercase() == "histogram" || metrics_type.to_lowercase() == "summary" { if !stream_schema_map.contains_key(&stream_name) { @@ -130,111 +155,91 @@ pub async fn ingest(org_id: &str, body: web::Bytes) -> Result continue; } - // apply functions let record = json::Value::Object(record.to_owned()); - let mut record = apply_func(&mut runtime, org_id, &stream_name, record)?; - let record = record.as_object_mut().unwrap(); + // ready to be buffered for downstream processing + if stream_executable_pipelines + .get(&stream_name) + .unwrap() + .is_some() + { + // buffer to pipeline for batch processing + stream_pipeline_inputs + .entry(stream_name.to_owned()) + .or_default() + .push((record, metrics_type)); + } else { + // buffer to downstream processing directly + json_data_by_stream + .entry(stream_name.clone()) + .or_default() + .push((record, metrics_type)); + } + } - let cfg = config::get_config(); - // check timestamp & value - let timestamp: i64 = match record.get(&cfg.common.column_timestamp) { - None => chrono::Utc::now().timestamp_micros(), - Some(json::Value::Number(s)) => { - time::parse_i64_to_timestamp_micros(s.as_f64().unwrap() as i64) - } - Some(_) => { - return Err(anyhow::anyhow!("invalid _timestamp, need to be number")); - } - }; - let value: f64 = match record.get(VALUE_LABEL).ok_or(anyhow!("missing value"))? { - json::Value::Number(s) => s.as_f64().unwrap(), - _ => { - return Err(anyhow::anyhow!("invalid value, need to be number")); - } - }; - // reset time & value - record.insert( - cfg.common.column_timestamp.clone(), - json::Value::Number(timestamp.into()), - ); - record.insert( - VALUE_LABEL.to_string(), - json::Number::from_f64(value).unwrap().into(), - ); - // remove type from labels - record.remove(TYPE_LABEL); - // add hash - let hash = super::signature_without_labels(record, &get_exclude_labels()); - record.insert(HASH_LABEL.to_string(), json::Value::String(hash.into())); - - // convert every label to string - for (k, v) in record.iter_mut() { - if k == NAME_LABEL - || k == TYPE_LABEL - || k == VALUE_LABEL - || k == &cfg.common.column_timestamp - { + // process records buffered for pipeline processing + for (stream_name, exec_pl_option) in &stream_executable_pipelines { + if let Some(exec_pl) = exec_pl_option { + let Some(pipeline_inputs) = stream_pipeline_inputs.remove(stream_name) else { + log::error!( + "[Ingestion]: Stream {} has pipeline, but inputs failed to be buffered. BUG", + stream_name + ); continue; - } - match v { - json::Value::String(_) => {} - _ => { - *v = json::Value::String(v.to_string()); + }; + let (records, metric_types): (Vec, Vec) = + pipeline_inputs.into_iter().unzip(); + let count = records.len(); + match exec_pl.process_batch(org_id, records).await { + Err(e) => { + let err_msg = format!( + "[Ingestion]: Stream {} pipeline batch processing failed: {}", + stream_name, e, + ); + log::error!("{err_msg}"); + // update status + let stream_status = stream_status_map + .entry(stream_name.clone()) + .or_insert_with(|| StreamStatus::new(stream_name)); + stream_status.status.failed += count as u32; + stream_status.status.error = err_msg; + continue; + } + Ok(pl_results) => { + for (stream_params, stream_pl_results) in pl_results { + if stream_params.stream_type != StreamType::Metrics { + continue; + } + // add partition keys + if !stream_partitioning_map.contains_key(stream_params.stream_name.as_str()) + { + let partition_det = + crate::service::ingestion::get_stream_partition_keys( + org_id, + &StreamType::Metrics, + &stream_params.stream_name, + ) + .await; + stream_partitioning_map.insert( + stream_params.stream_name.to_string(), + partition_det.clone(), + ); + } + for (idx, res) in stream_pl_results { + // buffer to downstream processing directly + json_data_by_stream + .entry(stream_params.stream_name.to_string()) + .or_default() + .push((res, metric_types[idx].to_owned())); + } + } } } } - let record_str = json::to_string(&record).unwrap(); + } - // check schema - if !stream_schema_map.contains_key(&stream_name) { - let mut schema = infra::schema::get(org_id, &stream_name, StreamType::Metrics).await?; - if schema.fields().is_empty() { - let mut schema_reader = BufReader::new(record_str.as_bytes()); - let inferred_schema = - infer_json_schema(&mut schema_reader, None, StreamType::Metrics).unwrap(); - let metadata = Metadata { - metric_family_name: stream_name.clone(), - metric_type: metrics_type.as_str().into(), - help: stream_name.clone().replace('_', " "), - unit: "".to_string(), - }; - let mut extra_metadata: HashMap = HashMap::new(); - extra_metadata.insert( - METADATA_LABEL.to_string(), - json::to_string(&metadata).unwrap(), - ); - schema = inferred_schema.with_metadata(extra_metadata); - db::schema::merge( - org_id, - &stream_name, - StreamType::Metrics, - &schema, - Some(timestamp), - ) - .await?; - crate::common::utils::auth::set_ownership( - org_id, - &StreamType::Metrics.to_string(), - Authz::new(&stream_name), - ) - .await; - } - stream_schema_map.insert(stream_name.clone(), SchemaCache::new(schema)); - } - - // check for schema evolution - let _ = check_for_schema( - org_id, - &stream_name, - StreamType::Metrics, - &mut stream_schema_map, - vec![record], - timestamp, - ) - .await; - - // write into buffer + let cfg = config::get_config(); + for (stream_name, json_data) in json_data_by_stream { if !stream_partitioning_map.contains_key(&stream_name) { let partition_det = crate::service::ingestion::get_stream_partition_keys( org_id, @@ -242,50 +247,183 @@ pub async fn ingest(org_id: &str, body: web::Bytes) -> Result &stream_name, ) .await; - stream_partitioning_map.insert(stream_name.to_string(), partition_det.clone()); + stream_partitioning_map.insert(stream_name.to_string(), partition_det); } - + // get partition key let partition_det = stream_partitioning_map.get(&stream_name).unwrap(); let partition_keys = partition_det.partition_keys.clone(); let partition_time_level = unwrap_partition_time_level(partition_det.partition_time_level, StreamType::Metrics); - let schema = stream_schema_map - .get(&stream_name) - .unwrap() - .schema() - .as_ref() - .clone() - .with_metadata(HashMap::new()); - let schema_key = schema.hash_key(); - let hour_key = get_write_partition_key( - timestamp, - &partition_keys, - partition_time_level, - record, - Some(&schema_key), - ); - let stream_buf = stream_data_buf.entry(stream_name.to_string()).or_default(); - let hour_buf = stream_buf.entry(hour_key).or_insert_with(|| SchemaRecords { - schema_key, - schema: Arc::new(schema), - records: vec![], - records_size: 0, - }); - hour_buf - .records - .push(Arc::new(json::Value::Object(record.to_owned()))); - hour_buf.records_size += record_str.len(); + for (mut record, metric_type) in json_data { + // Start get stream alerts + if !stream_alerts_map.contains_key(&stream_name) { + crate::service::ingestion::get_stream_alerts( + &[StreamParams { + org_id: org_id.to_owned().into(), + stream_name: stream_name.to_owned().into(), + stream_type: StreamType::Metrics, + }], + &mut stream_alerts_map, + ) + .await; + } + // End get stream alert - // update status - let stream_status = stream_status_map - .entry(stream_name.clone()) - .or_insert_with(|| StreamStatus::new(&stream_name)); - stream_status.status.successful += 1; + let record = record.as_object_mut().unwrap(); + + // check timestamp & value + let timestamp: i64 = match record.get(&cfg.common.column_timestamp) { + None => chrono::Utc::now().timestamp_micros(), + Some(json::Value::Number(s)) => { + time::parse_i64_to_timestamp_micros(s.as_f64().unwrap() as i64) + } + Some(_) => { + return Err(anyhow::anyhow!("invalid _timestamp, need to be number")); + } + }; + let value: f64 = match record.get(VALUE_LABEL).ok_or(anyhow!("missing value"))? { + json::Value::Number(s) => s.as_f64().unwrap(), + _ => { + return Err(anyhow::anyhow!("invalid value, need to be number")); + } + }; + // reset time & value + record.insert( + cfg.common.column_timestamp.clone(), + json::Value::Number(timestamp.into()), + ); + record.insert( + VALUE_LABEL.to_string(), + json::Number::from_f64(value).unwrap().into(), + ); + // remove type from labels + record.remove(TYPE_LABEL); + // add hash + let hash = super::signature_without_labels(record, &get_exclude_labels()); + record.insert(HASH_LABEL.to_string(), json::Value::String(hash.into())); + + // convert every label to string + for (k, v) in record.iter_mut() { + if k == NAME_LABEL + || k == TYPE_LABEL + || k == VALUE_LABEL + || k == &cfg.common.column_timestamp + { + continue; + } + match v { + json::Value::String(_) => {} + _ => { + *v = json::Value::String(v.to_string()); + } + } + } + let record_str = json::to_string(&record).unwrap(); + + // check schema + if !stream_schema_map.contains_key(&stream_name) { + let mut schema = + infra::schema::get(org_id, &stream_name, StreamType::Metrics).await?; + if schema.fields().is_empty() { + let mut schema_reader = BufReader::new(record_str.as_bytes()); + let inferred_schema = + infer_json_schema(&mut schema_reader, None, StreamType::Metrics).unwrap(); + let metadata = Metadata { + metric_family_name: stream_name.clone(), + metric_type: metric_type.as_str().into(), + help: stream_name.clone().replace('_', " "), + unit: "".to_string(), + }; + let mut extra_metadata: HashMap = HashMap::new(); + extra_metadata.insert( + METADATA_LABEL.to_string(), + json::to_string(&metadata).unwrap(), + ); + schema = inferred_schema.with_metadata(extra_metadata); + db::schema::merge( + org_id, + &stream_name, + StreamType::Metrics, + &schema, + Some(timestamp), + ) + .await?; + crate::common::utils::auth::set_ownership( + org_id, + &StreamType::Metrics.to_string(), + Authz::new(&stream_name), + ) + .await; + } + stream_schema_map.insert(stream_name.clone(), SchemaCache::new(schema)); + } + + // check for schema evolution + let _ = check_for_schema( + org_id, + &stream_name, + StreamType::Metrics, + &mut stream_schema_map, + vec![record], + timestamp, + ) + .await; + + // write into buffer + let schema = stream_schema_map + .get(&stream_name) + .unwrap() + .schema() + .as_ref() + .clone() + .with_metadata(HashMap::new()); + let schema_key = schema.hash_key(); + let hour_key = get_write_partition_key( + timestamp, + &partition_keys, + partition_time_level, + record, + Some(&schema_key), + ); + let stream_buf = stream_data_buf.entry(stream_name.to_string()).or_default(); + let hour_buf = stream_buf.entry(hour_key).or_insert_with(|| SchemaRecords { + schema_key, + schema: Arc::new(schema), + records: vec![], + records_size: 0, + }); + hour_buf + .records + .push(Arc::new(json::Value::Object(record.to_owned()))); + hour_buf.records_size += record_str.len(); + + // update status + let stream_status = stream_status_map + .entry(stream_name.clone()) + .or_insert_with(|| StreamStatus::new(&stream_name)); + stream_status.status.successful += 1; + + // realtime alert + let need_trigger = !stream_trigger_map.contains_key(&stream_name); + if need_trigger && !stream_alerts_map.is_empty() { + // start check for alert trigger + let key = format!("{}/{}/{}", org_id, StreamType::Metrics, stream_name); + if let Some(alerts) = stream_alerts_map.get(&key) { + let mut trigger_alerts: TriggerAlertData = Vec::new(); + for alert in alerts { + if let Ok((Some(v), _)) = alert.evaluate(Some(record), None).await { + trigger_alerts.push((alert.clone(), v)); + } + } + stream_trigger_map.insert(stream_name.clone(), Some(trigger_alerts)); + } + } + // End check for alert trigger + } } // write data to wal - let time = start.elapsed().as_secs_f64(); for (stream_name, stream_data) in stream_data_buf { // check if we are allowed to ingest if db::compact::retention::is_deleting_stream( @@ -301,23 +439,29 @@ pub async fn ingest(org_id: &str, body: web::Bytes) -> Result let writer = ingester::get_writer(0, org_id, &StreamType::Metrics.to_string(), &stream_name).await; let mut req_stats = write_file(&writer, &stream_name, stream_data).await; - // if let Err(e) = writer.sync().await { - // log::error!("ingestion error while syncing writer: {}", e); - // } - req_stats.response_time = time; + req_stats.response_time = start.elapsed().as_secs_f64(); + let fns_length: usize = + stream_executable_pipelines + .get(&stream_name) + .map_or(0, |exec_pl_option| { + exec_pl_option + .as_ref() + .map_or(0, |exec_pl| exec_pl.num_of_func()) + }); report_request_usage_stats( req_stats, org_id, &stream_name, StreamType::Metrics, UsageType::JsonMetrics, - 0, + fns_length as _, started_at, ) .await; } + let time = start.elapsed().as_secs_f64(); metrics::HTTP_RESPONSE_TIME .with_label_values(&[ "/api/org/ingest/metrics/_json", @@ -337,30 +481,15 @@ pub async fn ingest(org_id: &str, body: web::Bytes) -> Result ]) .inc(); + // only one trigger per request, as it updates etcd + for (_, entry) in stream_trigger_map { + if let Some(entry) = entry { + evaluate_trigger(entry).await; + } + } + Ok(IngestionResponse::new( http::StatusCode::OK.into(), stream_status_map.values().map(|v| v.to_owned()).collect(), )) } - -fn apply_func( - runtime: &mut Runtime, - org_id: &str, - metric_name: &str, - value: json::Value, -) -> Result { - let (_, local_trans, stream_vrl_map) = crate::service::ingestion::register_stream_functions( - org_id, - &StreamType::Metrics, - metric_name, - ); - - crate::service::ingestion::apply_stream_functions( - &local_trans, - value, - &stream_vrl_map, - org_id, - metric_name, - runtime, - ) -} diff --git a/src/service/metrics/otlp_grpc.rs b/src/service/metrics/otlp_grpc.rs index 13779bdba8..9e0fe5a7a6 100644 --- a/src/service/metrics/otlp_grpc.rs +++ b/src/service/metrics/otlp_grpc.rs @@ -22,6 +22,7 @@ use config::{ cluster::LOCAL_NODE, get_config, meta::{ + alerts::alert, stream::{PartitioningDetails, StreamParams, StreamType}, usage::UsageType, }, @@ -32,16 +33,17 @@ use hashbrown::HashSet; use infra::schema::{unwrap_partition_time_level, update_setting, SchemaCache}; use opentelemetry::trace::{SpanId, TraceId}; use opentelemetry_proto::tonic::{ - collector::metrics::v1::{ExportMetricsServiceRequest, ExportMetricsServiceResponse}, + collector::metrics::v1::{ + ExportMetricsPartialSuccess, ExportMetricsServiceRequest, ExportMetricsServiceResponse, + }, metrics::v1::{metric::Data, *}, }; use prost::Message; use crate::{ - common::meta::{ - alerts::alert, http::HttpResponse as MetaHttpResponse, prom::*, stream::SchemaRecords, - }, + common::meta::{http::HttpResponse as MetaHttpResponse, prom::*, stream::SchemaRecords}, service::{ + alerts::alert::AlertExt, db, format_stream_name, ingestion::{ evaluate_trigger, @@ -49,6 +51,7 @@ use crate::{ write_file, TriggerAlertData, }, metrics::{format_label_name, get_exclude_labels}, + pipeline::batch_execution::ExecutablePipeline, schema::{check_for_schema, stream_schema_exists}, usage::report_request_usage_stats, }, @@ -89,15 +92,29 @@ pub async fn handle_grpc_request( let start = std::time::Instant::now(); let started_at = Utc::now().timestamp_micros(); - let mut runtime = crate::service::ingestion::init_functions_runtime(); + let mut metric_data_map: HashMap> = HashMap::new(); let mut metric_schema_map: HashMap = HashMap::new(); let mut schema_evolved: HashMap = HashMap::new(); - let mut stream_alerts_map: HashMap> = HashMap::new(); - let mut stream_trigger_map: HashMap> = HashMap::new(); let mut stream_partitioning_map: HashMap = HashMap::new(); + // associated pipeline + let mut stream_executable_pipelines: HashMap> = + HashMap::new(); + let mut stream_pipeline_inputs: HashMap> = HashMap::new(); + + // realtime alerts + let mut stream_alerts_map: HashMap> = HashMap::new(); + let mut stream_trigger_map: HashMap> = HashMap::new(); + let cfg = get_config(); + let mut response = ExportMetricsServiceResponse { + partial_success: None, + }; + + // records buffer + let mut json_data_by_stream: HashMap> = HashMap::new(); + for resource_metric in &request.resource_metrics { for scope_metric in &resource_metric.scope_metrics { for metric in &scope_metric.metrics { @@ -122,12 +139,6 @@ pub async fn handle_grpc_request( stream_partitioning_map .insert(metric_name.clone().to_owned(), partition_det.clone()); } - let mut partition_det = stream_partitioning_map.get(metric_name).unwrap(); - let mut partition_keys = partition_det.partition_keys.clone(); - let mut partition_time_level = unwrap_partition_time_level( - partition_det.partition_time_level, - StreamType::Metrics, - ); // Start get stream alerts crate::service::ingestion::get_stream_alerts( @@ -141,14 +152,17 @@ pub async fn handle_grpc_request( .await; // End get stream alert - // Start Register Transforms for stream - let (_, mut local_trans, mut stream_vrl_map) = - crate::service::ingestion::register_stream_functions( - org_id, - &StreamType::Metrics, - metric_name, - ); - // End Register Transforms for stream + // get stream pipeline + if !stream_executable_pipelines.contains_key(metric_name) { + let pipeline_params = + crate::service::ingestion::get_stream_executable_pipeline( + org_id, + metric_name, + &StreamType::Metrics, + ) + .await; + stream_executable_pipelines.insert(metric_name.clone(), pipeline_params); + } let mut rec = json::json!({}); match &resource_metric.resource { @@ -243,12 +257,6 @@ pub async fn handle_grpc_request( stream_partitioning_map .insert(local_metric_name.to_owned(), partition_det.clone()); } - partition_det = stream_partitioning_map.get(local_metric_name).unwrap(); - partition_keys = partition_det.partition_keys.clone(); - partition_time_level = unwrap_partition_time_level( - partition_det.partition_time_level, - StreamType::Metrics, - ); // Start get stream alerts crate::service::ingestion::get_stream_alerts( @@ -262,132 +270,221 @@ pub async fn handle_grpc_request( .await; // End get stream alert - // Start Register Transforms for stream - (_, local_trans, stream_vrl_map) = - crate::service::ingestion::register_stream_functions( - org_id, - &StreamType::Metrics, - local_metric_name, - ); - // End Register Transforms for stream - } - - if !local_trans.is_empty() { - rec = crate::service::ingestion::apply_stream_functions( - &local_trans, - rec, - &stream_vrl_map, - org_id, - local_metric_name, - &mut runtime, - )?; - } - - // get json object - let val_map: &mut serde_json::Map = - rec.as_object_mut().unwrap(); - - let timestamp = val_map - .get(&cfg.common.column_timestamp) - .unwrap() - .as_i64() - .unwrap_or(Utc::now().timestamp_micros()); - - let value_str = json::to_string(&val_map).unwrap(); - - // check for schema evolution - let schema_fields = match metric_schema_map.get(local_metric_name) { - Some(schema) => schema - .schema() - .fields() - .iter() - .map(|f| f.name()) - .collect::>(), - None => HashSet::default(), - }; - let mut need_schema_check = !schema_evolved.contains_key(local_metric_name); - for key in val_map.keys() { - if !schema_fields.contains(&key) { - need_schema_check = true; - break; + // get stream pipeline + if !stream_executable_pipelines.contains_key(local_metric_name) { + let pipeline_params = + crate::service::ingestion::get_stream_executable_pipeline( + org_id, + local_metric_name, + &StreamType::Metrics, + ) + .await; + stream_executable_pipelines + .insert(local_metric_name.clone(), pipeline_params); } } - drop(schema_fields); - if need_schema_check - && check_for_schema( - org_id, - local_metric_name, - StreamType::Metrics, - &mut metric_schema_map, - vec![val_map], - timestamp, - ) - .await - .is_ok() - { - schema_evolved.insert(local_metric_name.to_owned(), true); - } - let buf = metric_data_map - .entry(local_metric_name.to_owned()) - .or_default(); - let schema = metric_schema_map + // ready to be buffered for downstream processing + if stream_executable_pipelines .get(local_metric_name) .unwrap() - .schema() - .as_ref() - .clone() - .with_metadata(HashMap::new()); - let schema_key = schema.hash_key(); - // get hour key - let hour_key = crate::service::ingestion::get_write_partition_key( - timestamp, - &partition_keys, - partition_time_level, - val_map, - Some(&schema_key), - ); - let hour_buf = buf.entry(hour_key).or_insert_with(|| SchemaRecords { - schema_key, - schema: Arc::new(schema), - records: vec![], - records_size: 0, - }); - hour_buf - .records - .push(Arc::new(json::Value::Object(val_map.to_owned()))); - hour_buf.records_size += value_str.len(); - - // real time alert - let need_trigger = !stream_trigger_map.contains_key(local_metric_name); - if need_trigger && !stream_alerts_map.is_empty() { - // Start check for alert trigger - let key = format!( - "{}/{}/{}", - &org_id, - StreamType::Metrics, - local_metric_name.clone() - ); - if let Some(alerts) = stream_alerts_map.get(&key) { - let mut trigger_alerts: TriggerAlertData = Vec::new(); - for alert in alerts { - if let Ok((Some(v), _)) = alert.evaluate(Some(val_map), None).await - { - trigger_alerts.push((alert.clone(), v)); - } - } - stream_trigger_map - .insert(local_metric_name.clone(), Some(trigger_alerts)); - } - // End check for alert trigger + .is_some() + { + stream_pipeline_inputs + .entry(local_metric_name.to_string()) + .or_default() + .push(rec); + } else { + json_data_by_stream + .entry(local_metric_name.to_string()) + .or_default() + .push(rec); } } } } } + // process records buffered for pipeline processing + for (stream_name, exec_pl_option) in &stream_executable_pipelines { + if let Some(exec_pl) = exec_pl_option { + let Some(pipeline_inputs) = stream_pipeline_inputs.remove(stream_name) else { + let err_msg = format!( + "[Ingestion]: Stream {} has pipeline, but inputs failed to be buffered. BUG", + stream_name + ); + log::error!("{err_msg}"); + let partial_resp = + response + .partial_success + .get_or_insert(ExportMetricsPartialSuccess { + rejected_data_points: 0, + error_message: String::new(), + }); + partial_resp.error_message = err_msg; + continue; + }; + let count = pipeline_inputs.len(); + match exec_pl.process_batch(org_id, pipeline_inputs).await { + Err(e) => { + let err_msg = format!( + "[Ingestion]: Stream {} pipeline batch processing failed: {}", + stream_name, e, + ); + log::error!("{err_msg}"); + // update status + let partial_resp = + response + .partial_success + .get_or_insert(ExportMetricsPartialSuccess { + rejected_data_points: 0, + error_message: String::new(), + }); + partial_resp.rejected_data_points += count as i64; + partial_resp.error_message = err_msg; + continue; + } + Ok(pl_results) => { + for (stream_params, stream_pl_results) in pl_results { + if stream_params.stream_type != StreamType::Metrics { + continue; + } + // add partition keys + if !stream_partitioning_map.contains_key(stream_params.stream_name.as_str()) + { + let partition_det = + crate::service::ingestion::get_stream_partition_keys( + org_id, + &StreamType::Metrics, + &stream_params.stream_name, + ) + .await; + stream_partitioning_map.insert( + stream_params.stream_name.to_string(), + partition_det.clone(), + ); + } + for (_, res) in stream_pl_results { + // buffer to downstream processing directly + json_data_by_stream + .entry(stream_params.stream_name.to_string()) + .or_default() + .push(res); + } + } + } + } + } + } + + for (local_metric_name, json_data) in json_data_by_stream { + // get partition keys + let partition_det = stream_partitioning_map.get(&local_metric_name).unwrap(); + let partition_keys = partition_det.partition_keys.clone(); + let partition_time_level = + unwrap_partition_time_level(partition_det.partition_time_level, StreamType::Metrics); + + for mut rec in json_data { + // get json object + let val_map: &mut serde_json::Map = + rec.as_object_mut().unwrap(); + + let timestamp = val_map + .get(&cfg.common.column_timestamp) + .unwrap() + .as_i64() + .unwrap_or(Utc::now().timestamp_micros()); + + let value_str = json::to_string(&val_map).unwrap(); + + // check for schema evolution + let schema_fields = match metric_schema_map.get(&local_metric_name) { + Some(schema) => schema + .schema() + .fields() + .iter() + .map(|f| f.name()) + .collect::>(), + None => HashSet::default(), + }; + let mut need_schema_check = !schema_evolved.contains_key(&local_metric_name); + for key in val_map.keys() { + if !schema_fields.contains(&key) { + need_schema_check = true; + break; + } + } + drop(schema_fields); + if need_schema_check + && check_for_schema( + org_id, + &local_metric_name, + StreamType::Metrics, + &mut metric_schema_map, + vec![val_map], + timestamp, + ) + .await + .is_ok() + { + schema_evolved.insert(local_metric_name.to_owned(), true); + } + + let buf = metric_data_map + .entry(local_metric_name.to_owned()) + .or_default(); + let schema = metric_schema_map + .get(&local_metric_name) + .unwrap() + .schema() + .as_ref() + .clone() + .with_metadata(HashMap::new()); + let schema_key = schema.hash_key(); + // get hour key + let hour_key = crate::service::ingestion::get_write_partition_key( + timestamp, + &partition_keys, + partition_time_level, + val_map, + Some(&schema_key), + ); + let hour_buf = buf.entry(hour_key).or_insert_with(|| SchemaRecords { + schema_key, + schema: Arc::new(schema), + records: vec![], + records_size: 0, + }); + hour_buf + .records + .push(Arc::new(json::Value::Object(val_map.to_owned()))); + hour_buf.records_size += value_str.len(); + + // real time alert + let need_trigger = !stream_trigger_map.contains_key(&local_metric_name); + if need_trigger && !stream_alerts_map.is_empty() { + // Start check for alert trigger + let key = format!( + "{}/{}/{}", + &org_id, + StreamType::Metrics, + local_metric_name.clone() + ); + if let Some(alerts) = stream_alerts_map.get(&key) { + let mut trigger_alerts: TriggerAlertData = Vec::new(); + for alert in alerts { + if let Ok((Some(v), _)) = alert.evaluate(Some(val_map), None).await { + trigger_alerts.push((alert.clone(), v)); + } + } + stream_trigger_map.insert(local_metric_name.clone(), Some(trigger_alerts)); + } + } + // End check for alert trigger + } + } + // write data to wal - let time = start.elapsed().as_secs_f64(); for (stream_name, stream_data) in metric_data_map { // stream_data could be empty if metric value is nan, check it if stream_data.is_empty() { @@ -409,49 +506,53 @@ pub async fn handle_grpc_request( let writer = ingester::get_writer(0, org_id, &StreamType::Metrics.to_string(), &stream_name).await; let mut req_stats = write_file(&writer, &stream_name, stream_data).await; - // if let Err(e) = writer.sync().await { - // log::error!("ingestion error while syncing writer: {}", e); - // } - req_stats.response_time += time; + let fns_length: usize = + stream_executable_pipelines + .get(&stream_name) + .map_or(0, |exec_pl_option| { + exec_pl_option + .as_ref() + .map_or(0, |exec_pl| exec_pl.num_of_func()) + }); + req_stats.response_time = start.elapsed().as_secs_f64(); report_request_usage_stats( req_stats, org_id, &stream_name, StreamType::Metrics, UsageType::Metrics, - 0, + fns_length as _, started_at, ) .await; - - let ep = if is_grpc { - "/grpc/otlp/metrics" - } else { - "/api/otlp/v1/metrics" - }; - - let time = start.elapsed().as_secs_f64(); - metrics::HTTP_RESPONSE_TIME - .with_label_values(&[ - ep, - "200", - org_id, - &stream_name, - StreamType::Metrics.to_string().as_str(), - ]) - .observe(time); - metrics::HTTP_INCOMING_REQUESTS - .with_label_values(&[ - ep, - "200", - org_id, - &stream_name, - StreamType::Metrics.to_string().as_str(), - ]) - .inc(); } + let ep = if is_grpc { + "/grpc/otlp/metrics" + } else { + "/api/otlp/v1/metrics" + }; + let time_took = start.elapsed().as_secs_f64(); + metrics::HTTP_RESPONSE_TIME + .with_label_values(&[ + ep, + "200", + org_id, + "", + StreamType::Metrics.to_string().as_str(), + ]) + .observe(time_took); + metrics::HTTP_INCOMING_REQUESTS + .with_label_values(&[ + ep, + "200", + org_id, + "", + StreamType::Metrics.to_string().as_str(), + ]) + .inc(); + // only one trigger per request, as it updates etcd for (_, entry) in stream_trigger_map { if let Some(entry) = entry { diff --git a/src/service/metrics/otlp_http.rs b/src/service/metrics/otlp_http.rs index 3721261f8a..40ce09b58d 100644 --- a/src/service/metrics/otlp_http.rs +++ b/src/service/metrics/otlp_http.rs @@ -22,6 +22,7 @@ use config::{ cluster::LOCAL_NODE, get_config, meta::{ + alerts::alert::Alert, stream::{PartitioningDetails, StreamParams, StreamType}, usage::UsageType, }, @@ -32,7 +33,9 @@ use hashbrown::HashSet; use infra::schema::{unwrap_partition_time_level, update_setting, SchemaCache}; use opentelemetry::trace::{SpanId, TraceId}; use opentelemetry_proto::tonic::{ - collector::metrics::v1::{ExportMetricsServiceRequest, ExportMetricsServiceResponse}, + collector::metrics::v1::{ + ExportMetricsPartialSuccess, ExportMetricsServiceRequest, ExportMetricsServiceResponse, + }, metrics::v1::*, }; use prost::Message; @@ -40,16 +43,17 @@ use prost::Message; use crate::{ common::meta::{ self, - alerts::alert::Alert, http::HttpResponse as MetaHttpResponse, prom::{self, MetricType, HASH_LABEL, NAME_LABEL, VALUE_LABEL}, stream::SchemaRecords, }, handler::http::request::CONTENT_TYPE_JSON, service::{ + alerts::alert::AlertExt, db, format_stream_name, ingestion::{evaluate_trigger, get_val_for_attr, write_file, TriggerAlertData}, metrics::{format_label_name, get_exclude_labels, otlp_grpc::handle_grpc_request}, + pipeline::batch_execution::ExecutablePipeline, schema::{check_for_schema, stream_schema_exists}, usage::report_request_usage_stats, }, @@ -110,13 +114,20 @@ pub async fn metrics_json_handler( let start = std::time::Instant::now(); let started_at = Utc::now().timestamp_micros(); - let mut runtime = crate::service::ingestion::init_functions_runtime(); + let mut metric_data_map: HashMap> = HashMap::new(); let mut metric_schema_map: HashMap = HashMap::new(); let mut schema_evolved: HashMap = HashMap::new(); + let mut stream_partitioning_map: HashMap = HashMap::new(); + + // associated pipeline + let mut stream_executable_pipelines: HashMap> = + HashMap::new(); + let mut stream_pipeline_inputs: HashMap> = HashMap::new(); + + // realtime alerts let mut stream_alerts_map: HashMap> = HashMap::new(); let mut stream_trigger_map: HashMap> = HashMap::new(); - let mut stream_partitioning_map: HashMap = HashMap::new(); let body: json::Value = match json::from_slice(body.as_ref()) { Ok(v) => v, @@ -147,6 +158,13 @@ pub async fn metrics_json_handler( }; let cfg = get_config(); + let mut response = ExportMetricsServiceResponse { + partial_success: None, + }; + + // records buffer + let mut json_data_by_stream: HashMap> = HashMap::new(); + for res_metric in res_metrics.iter() { let mut service_att_map: json::Map = json::Map::new(); if res_metric.get("resource").is_some() { @@ -200,12 +218,6 @@ pub async fn metrics_json_handler( stream_partitioning_map .insert(metric_name.to_owned(), partition_det.clone()); } - let mut partition_det = stream_partitioning_map.get(metric_name).unwrap(); - let mut partition_keys = partition_det.partition_keys.clone(); - let mut partition_time_level = unwrap_partition_time_level( - partition_det.partition_time_level, - StreamType::Metrics, - ); // Start get stream alerts crate::service::ingestion::get_stream_alerts( @@ -219,14 +231,17 @@ pub async fn metrics_json_handler( .await; // End get stream alert - // Start Register Transforms for stream - let (_, mut local_trans, mut stream_vrl_map) = - crate::service::ingestion::register_stream_functions( - org_id, - &StreamType::Metrics, - metric_name, - ); - // End Register Transforms for stream + // get stream pipeline + if !stream_executable_pipelines.contains_key(metric_name) { + let pipeline_params = + crate::service::ingestion::get_stream_executable_pipeline( + org_id, + metric_name, + &StreamType::Metrics, + ) + .await; + stream_executable_pipelines.insert(metric_name.clone(), pipeline_params); + } let mut rec = json::json!({}); @@ -328,12 +343,6 @@ pub async fn metrics_json_handler( stream_partitioning_map .insert(local_metric_name.to_owned(), partition_det.clone()); } - partition_det = stream_partitioning_map.get(local_metric_name).unwrap(); - partition_keys = partition_det.partition_keys.clone(); - partition_time_level = unwrap_partition_time_level( - partition_det.partition_time_level, - StreamType::Metrics, - ); // Start get stream alerts crate::service::ingestion::get_stream_alerts( @@ -347,126 +356,35 @@ pub async fn metrics_json_handler( .await; // End get stream alert - // Start Register Transforms for stream - (_, local_trans, stream_vrl_map) = - crate::service::ingestion::register_stream_functions( - org_id, - &StreamType::Metrics, - local_metric_name, - ); - // End Register Transforms for stream - } - - if !local_trans.is_empty() { - rec = crate::service::ingestion::apply_stream_functions( - &local_trans, - rec, - &stream_vrl_map, - org_id, - local_metric_name, - &mut runtime, - ) - .unwrap(); - } - - let val_map: &mut serde_json::Map = - rec.as_object_mut().unwrap(); - - let timestamp = val_map - .get(&cfg.common.column_timestamp) - .unwrap() - .as_i64() - .unwrap_or(Utc::now().timestamp_micros()); - - let value_str = json::to_string(&val_map).unwrap(); - - // check for schema evolution - let schema_fields = match metric_schema_map.get(local_metric_name) { - Some(schema) => schema - .schema() - .fields() - .iter() - .map(|f| f.name()) - .collect::>(), - None => HashSet::default(), - }; - let mut need_schema_check = !schema_evolved.contains_key(local_metric_name); - for key in val_map.keys() { - if !schema_fields.contains(&key) { - need_schema_check = true; - break; + // get stream pipeline + if !stream_executable_pipelines.contains_key(local_metric_name) { + let pipeline_params = + crate::service::ingestion::get_stream_executable_pipeline( + org_id, + local_metric_name, + &StreamType::Metrics, + ) + .await; + stream_executable_pipelines + .insert(local_metric_name.clone(), pipeline_params); } } - drop(schema_fields); - if need_schema_check - && check_for_schema( - org_id, - local_metric_name, - StreamType::Metrics, - &mut metric_schema_map, - vec![val_map], - timestamp, - ) - .await - .is_ok() - { - schema_evolved.insert(local_metric_name.to_owned(), true); - } - let schema = metric_schema_map + // ready to be buffered for downstream processing + if stream_executable_pipelines .get(local_metric_name) .unwrap() - .schema() - .as_ref() - .clone() - .with_metadata(HashMap::new()); - let schema_key = schema.hash_key(); - - let buf = metric_data_map - .entry(local_metric_name.to_owned()) - .or_default(); - // get hour key - let hour_key = crate::service::ingestion::get_write_partition_key( - timestamp, - &partition_keys, - partition_time_level, - val_map, - Some(&schema_key), - ); - let hour_buf = buf.entry(hour_key).or_insert_with(|| SchemaRecords { - schema_key, - schema: Arc::new(schema), - records: vec![], - records_size: 0, - }); - hour_buf - .records - .push(Arc::new(json::Value::Object(val_map.to_owned()))); - hour_buf.records_size += value_str.len(); - - // real time alert - let need_trigger = !stream_trigger_map.contains_key(local_metric_name); - if need_trigger && !stream_alerts_map.is_empty() { - // Start check for alert trigger - let key = format!( - "{}/{}/{}", - &org_id, - StreamType::Metrics, - local_metric_name - ); - if let Some(alerts) = stream_alerts_map.get(&key) { - let mut trigger_alerts: TriggerAlertData = Vec::new(); - for alert in alerts { - if let Ok((Some(v), _)) = - alert.evaluate(Some(val_map), None).await - { - trigger_alerts.push((alert.clone(), v)); - } - } - stream_trigger_map - .insert(local_metric_name.clone(), Some(trigger_alerts)); - } - // End check for alert trigger + .is_some() + { + stream_pipeline_inputs + .entry(local_metric_name.to_string()) + .or_default() + .push(rec); + } else { + json_data_by_stream + .entry(local_metric_name.to_string()) + .or_default() + .push(rec); } } } @@ -474,8 +392,181 @@ pub async fn metrics_json_handler( } } + // process records buffered for pipeline processing + for (stream_name, exec_pl_option) in &stream_executable_pipelines { + if let Some(exec_pl) = exec_pl_option { + let Some(pipeline_inputs) = stream_pipeline_inputs.remove(stream_name) else { + let err_msg = format!( + "[Ingestion]: Stream {} has pipeline, but inputs failed to be buffered. BUG", + stream_name + ); + log::error!("{err_msg}"); + let partial_resp = + response + .partial_success + .get_or_insert(ExportMetricsPartialSuccess { + rejected_data_points: 0, + error_message: String::new(), + }); + partial_resp.error_message = err_msg; + continue; + }; + let count = pipeline_inputs.len(); + match exec_pl.process_batch(org_id, pipeline_inputs).await { + Err(e) => { + let err_msg = format!( + "[Ingestion]: Stream {} pipeline batch processing failed: {}", + stream_name, e, + ); + log::error!("{err_msg}"); + // update status + let partial_resp = + response + .partial_success + .get_or_insert(ExportMetricsPartialSuccess { + rejected_data_points: 0, + error_message: String::new(), + }); + partial_resp.rejected_data_points += count as i64; + partial_resp.error_message = err_msg; + continue; + } + Ok(pl_results) => { + for (stream_params, stream_pl_results) in pl_results { + if stream_params.stream_type != StreamType::Metrics { + continue; + } + // add partition keys + if !stream_partitioning_map.contains_key(stream_params.stream_name.as_str()) + { + let partition_det = + crate::service::ingestion::get_stream_partition_keys( + org_id, + &StreamType::Metrics, + &stream_params.stream_name, + ) + .await; + stream_partitioning_map.insert( + stream_params.stream_name.to_string(), + partition_det.clone(), + ); + } + for (_, res) in stream_pl_results { + // buffer to downstream processing directly + json_data_by_stream + .entry(stream_params.stream_name.to_string()) + .or_default() + .push(res); + } + } + } + } + } + } + + for (local_metric_name, json_data) in json_data_by_stream { + // get partition keys + let partition_det = stream_partitioning_map.get(&local_metric_name).unwrap(); + let partition_keys = partition_det.partition_keys.clone(); + let partition_time_level = + unwrap_partition_time_level(partition_det.partition_time_level, StreamType::Metrics); + + for mut rec in json_data { + let val_map: &mut serde_json::Map = + rec.as_object_mut().unwrap(); + + let timestamp = val_map + .get(&cfg.common.column_timestamp) + .unwrap() + .as_i64() + .unwrap_or(Utc::now().timestamp_micros()); + + let value_str = json::to_string(&val_map).unwrap(); + + // check for schema evolution + let schema_fields = match metric_schema_map.get(&local_metric_name) { + Some(schema) => schema + .schema() + .fields() + .iter() + .map(|f| f.name()) + .collect::>(), + None => HashSet::default(), + }; + let mut need_schema_check = !schema_evolved.contains_key(&local_metric_name); + for key in val_map.keys() { + if !schema_fields.contains(&key) { + need_schema_check = true; + break; + } + } + drop(schema_fields); + if need_schema_check + && check_for_schema( + org_id, + &local_metric_name, + StreamType::Metrics, + &mut metric_schema_map, + vec![val_map], + timestamp, + ) + .await + .is_ok() + { + schema_evolved.insert(local_metric_name.to_owned(), true); + } + + let schema = metric_schema_map + .get(&local_metric_name) + .unwrap() + .schema() + .as_ref() + .clone() + .with_metadata(HashMap::new()); + let schema_key = schema.hash_key(); + + let buf = metric_data_map + .entry(local_metric_name.to_owned()) + .or_default(); + // get hour key + let hour_key = crate::service::ingestion::get_write_partition_key( + timestamp, + &partition_keys, + partition_time_level, + val_map, + Some(&schema_key), + ); + let hour_buf = buf.entry(hour_key).or_insert_with(|| SchemaRecords { + schema_key, + schema: Arc::new(schema), + records: vec![], + records_size: 0, + }); + hour_buf + .records + .push(Arc::new(json::Value::Object(val_map.to_owned()))); + hour_buf.records_size += value_str.len(); + + // real time alert + let need_trigger = !stream_trigger_map.contains_key(&local_metric_name); + if need_trigger && !stream_alerts_map.is_empty() { + // Start check for alert trigger + let key = format!("{}/{}/{}", &org_id, StreamType::Metrics, local_metric_name); + if let Some(alerts) = stream_alerts_map.get(&key) { + let mut trigger_alerts: TriggerAlertData = Vec::new(); + for alert in alerts { + if let Ok((Some(v), _)) = alert.evaluate(Some(val_map), None).await { + trigger_alerts.push((alert.clone(), v)); + } + } + stream_trigger_map.insert(local_metric_name.clone(), Some(trigger_alerts)); + } + } + // End check for alert trigger + } + } + // write data to wal - let time = start.elapsed().as_secs_f64(); for (stream_name, stream_data) in metric_data_map { // stream_data could be empty if metric value is nan, check it if stream_data.is_empty() { @@ -497,43 +588,48 @@ pub async fn metrics_json_handler( let writer = ingester::get_writer(0, org_id, &StreamType::Metrics.to_string(), &stream_name).await; let mut req_stats = write_file(&writer, &stream_name, stream_data).await; - // if let Err(e) = writer.sync().await { - // log::error!("ingestion error while syncing writer: {}", e); - // } - req_stats.response_time += time; + let fns_length: usize = + stream_executable_pipelines + .get(&stream_name) + .map_or(0, |exec_pl_option| { + exec_pl_option + .as_ref() + .map_or(0, |exec_pl| exec_pl.num_of_func()) + }); + req_stats.response_time = start.elapsed().as_secs_f64(); report_request_usage_stats( req_stats, org_id, &stream_name, StreamType::Metrics, UsageType::Metrics, - 0, + fns_length as _, started_at, ) .await; - - let time = start.elapsed().as_secs_f64(); - metrics::HTTP_RESPONSE_TIME - .with_label_values(&[ - "/api/otlp/v1/metrics", - "200", - org_id, - &stream_name, - StreamType::Metrics.to_string().as_str(), - ]) - .observe(time); - metrics::HTTP_INCOMING_REQUESTS - .with_label_values(&[ - "/api/otlp/v1/metrics", - "200", - org_id, - &stream_name, - StreamType::Metrics.to_string().as_str(), - ]) - .inc(); } + let time = start.elapsed().as_secs_f64(); + metrics::HTTP_RESPONSE_TIME + .with_label_values(&[ + "/api/otlp/v1/metrics", + "200", + org_id, + "", + StreamType::Metrics.to_string().as_str(), + ]) + .observe(time); + metrics::HTTP_INCOMING_REQUESTS + .with_label_values(&[ + "/api/otlp/v1/metrics", + "200", + org_id, + "", + StreamType::Metrics.to_string().as_str(), + ]) + .inc(); + // only one trigger per request, as it updates etcd for (_, entry) in stream_trigger_map { if let Some(entry) = entry { @@ -541,11 +637,8 @@ pub async fn metrics_json_handler( } } - let res = ExportMetricsServiceResponse { - partial_success: None, - }; - let mut out = BytesMut::with_capacity(res.encoded_len()); - res.encode(&mut out).expect("Out of memory"); + let mut out = BytesMut::with_capacity(response.encoded_len()); + response.encode(&mut out).expect("Out of memory"); return Ok(HttpResponse::Ok() .status(http::StatusCode::OK) diff --git a/src/service/metrics/prom.rs b/src/service/metrics/prom.rs index 71fb8acbd3..984cfef1d4 100644 --- a/src/service/metrics/prom.rs +++ b/src/service/metrics/prom.rs @@ -21,6 +21,7 @@ use config::{ cluster::LOCAL_NODE, get_config, meta::{ + alerts::alert, stream::{PartitioningDetails, StreamParams, StreamType}, usage::UsageType, }, @@ -42,12 +43,14 @@ use proto::prometheus_rpc; use crate::{ common::{ infra::config::{METRIC_CLUSTER_LEADER, METRIC_CLUSTER_MAP}, - meta::{alerts::alert, functions::StreamTransform, prom::*, stream::SchemaRecords}, + meta::{prom::*, stream::SchemaRecords}, }, service::{ + alerts::alert::AlertExt, db, format_stream_name, ingestion::{evaluate_trigger, write_file, TriggerAlertData}, metrics::format_label_name, + pipeline::batch_execution::ExecutablePipeline, schema::{check_for_schema, stream_schema_exists}, search as search_service, usage::report_request_usage_stats, @@ -79,8 +82,6 @@ pub async fn remote_write( } let cfg = get_config(); - // let min_ts = (Utc::now() - - // Duration::try_hours(cfg.limit.ingest_allowed_upto)).unwrap().timestamp_micros(); let dedup_enabled = cfg.common.metrics_dedup_enabled; let election_interval = cfg.limit.metrics_leader_election_interval * 1000000; let mut last_received: i64 = 0; @@ -90,10 +91,16 @@ pub async fn remote_write( let mut metric_data_map: HashMap> = HashMap::new(); let mut metric_schema_map: HashMap = HashMap::new(); let mut schema_evolved: HashMap = HashMap::new(); + let mut stream_partitioning_map: HashMap = HashMap::new(); + + // associated pipeline + let mut stream_executable_pipelines: HashMap> = + HashMap::new(); + let mut stream_pipeline_inputs: HashMap> = HashMap::new(); + + // realtime alerts let mut stream_alerts_map: HashMap> = HashMap::new(); let mut stream_trigger_map: HashMap> = HashMap::new(); - let mut stream_transform_map: HashMap> = HashMap::new(); - let mut stream_partitioning_map: HashMap = HashMap::new(); let decoded = snap::raw::Decoder::new() .decompress_vec(&body) @@ -101,6 +108,9 @@ pub async fn remote_write( let request = prometheus_rpc::WriteRequest::decode(bytes::Bytes::from(decoded)) .map_err(|e| anyhow::anyhow!("Invalid protobuf: {}", e.to_string()))?; + // records buffer + let mut json_data_by_stream: HashMap> = HashMap::new(); + // parse metadata for item in request.metadata { let metric_name = format_stream_name(&item.metric_family_name.clone()); @@ -197,8 +207,6 @@ pub async fn remote_write( None => continue, }; - let buf = metric_data_map.entry(metric_name.to_owned()).or_default(); - // parse samples for sample in event.samples { let mut sample_val = sample.value; @@ -218,8 +226,6 @@ pub async fn remote_write( value: sample_val, }; - let timestamp = parse_i64_to_timestamp_micros(sample.timestamp); - if first_line && dedup_enabled && !cluster_name.is_empty() { let lock = METRIC_CLUSTER_LEADER.read().await; match lock.get(&cluster_name) { @@ -292,12 +298,6 @@ pub async fn remote_write( .await; stream_partitioning_map.insert(metric_name.clone(), partition_det.clone()); } - let partition_det = stream_partitioning_map.get(&metric_name).unwrap(); - let partition_keys = partition_det.partition_keys.clone(); - let partition_time_level = unwrap_partition_time_level( - partition_det.partition_time_level, - StreamType::Metrics, - ); // Start get stream alerts crate::service::ingestion::get_stream_alerts( @@ -311,34 +311,103 @@ pub async fn remote_write( .await; // End get stream alert - let mut runtime = crate::service::ingestion::init_functions_runtime(); - - // Start Register Transforms for stream - let (_, local_trans, stream_vrl_map) = - crate::service::ingestion::register_stream_functions( + // get stream pipeline + if !stream_executable_pipelines.contains_key(&metric_name) { + let pipeline_params = crate::service::ingestion::get_stream_executable_pipeline( org_id, - &StreamType::Metrics, &metric_name, + &StreamType::Metrics, + ) + .await; + stream_executable_pipelines.insert(metric_name.clone(), pipeline_params); + } + + let value: json::Value = json::to_value(&metric).unwrap(); + let timestamp = parse_i64_to_timestamp_micros(sample.timestamp); + + // ready to be buffered for downstream processing + if stream_executable_pipelines + .get(&metric_name) + .unwrap() + .is_some() + { + // buffer to pipeline for batch processing + stream_pipeline_inputs + .entry(metric_name.to_owned()) + .or_default() + .push((value, timestamp)); + } else { + // buffer to downstream processing directly + json_data_by_stream + .entry(metric_name.clone()) + .or_default() + .push((value, timestamp)); + } + } + } + + // process records buffered for pipeline processing + for (stream_name, exec_pl_option) in &stream_executable_pipelines { + if let Some(exec_pl) = exec_pl_option { + let Some(pipeline_inputs) = stream_pipeline_inputs.remove(stream_name) else { + log::error!( + "[Ingestion]: Stream {} has pipeline, but inputs failed to be buffered. BUG", + stream_name ); + continue; + }; + let (records, timestamps): (Vec, Vec) = + pipeline_inputs.into_iter().unzip(); + match exec_pl.process_batch(org_id, records).await { + Err(e) => { + log::error!( + "[Ingestion]: Stream {} pipeline batch processing failed: {}", + stream_name, + e, + ); + continue; + } + Ok(pl_results) => { + for (stream_params, stream_pl_results) in pl_results { + if stream_params.stream_type != StreamType::Metrics { + continue; + } + // add partition keys + if !stream_partitioning_map.contains_key(stream_params.stream_name.as_str()) + { + let partition_det = + crate::service::ingestion::get_stream_partition_keys( + org_id, + &StreamType::Metrics, + &stream_params.stream_name, + ) + .await; + stream_partitioning_map.insert( + stream_params.stream_name.to_string(), + partition_det.clone(), + ); + } + for (idx, res) in stream_pl_results { + // buffer to downstream processing directly + json_data_by_stream + .entry(stream_params.stream_name.to_string()) + .or_default() + .push((res, timestamps[idx])); + } + } + } + } + } + } - stream_transform_map.insert(metric_name.to_owned(), local_trans.clone()); - // End Register Transforms for stream + for (stream_name, json_data) in json_data_by_stream { + // get partition keys + let partition_det = stream_partitioning_map.get(&stream_name).unwrap(); + let partition_keys = partition_det.partition_keys.clone(); + let partition_time_level = + unwrap_partition_time_level(partition_det.partition_time_level, StreamType::Metrics); - let mut value: json::Value = json::to_value(&metric).unwrap(); - - // Start row based transform - value = crate::service::ingestion::apply_stream_functions( - &local_trans, - value, - &stream_vrl_map, - org_id, - &metric_name, - &mut runtime, - )?; - - // End row based transform - - // get json object + for (mut value, timestamp) in json_data { let val_map = value.as_object_mut().unwrap(); let hash = super::signature_without_labels(val_map, &[VALUE_LABEL]); val_map.insert(HASH_LABEL.to_string(), json::Value::String(hash.into())); @@ -349,7 +418,7 @@ pub async fn remote_write( let value_str = config::utils::json::to_string(&val_map).unwrap(); // check for schema evolution - let schema_fields = match metric_schema_map.get(&metric_name) { + let schema_fields = match metric_schema_map.get(&stream_name) { Some(schema) => schema .schema() .fields() @@ -358,7 +427,7 @@ pub async fn remote_write( .collect::>(), None => HashSet::default(), }; - let mut need_schema_check = !schema_evolved.contains_key(&metric_name); + let mut need_schema_check = !schema_evolved.contains_key(&stream_name); for key in val_map.keys() { if !schema_fields.contains(&key) { need_schema_check = true; @@ -369,7 +438,7 @@ pub async fn remote_write( if need_schema_check && check_for_schema( org_id, - &metric_name, + &stream_name, StreamType::Metrics, &mut metric_schema_map, vec![val_map], @@ -378,11 +447,11 @@ pub async fn remote_write( .await .is_ok() { - schema_evolved.insert(metric_name.clone(), true); + schema_evolved.insert(stream_name.clone(), true); } let schema = metric_schema_map - .get(&metric_name) + .get(&stream_name) .unwrap() .schema() .as_ref() @@ -398,6 +467,7 @@ pub async fn remote_write( val_map, Some(&schema_key), ); + let buf = metric_data_map.entry(stream_name.to_owned()).or_default(); let hour_buf = buf.entry(hour_key).or_insert_with(|| SchemaRecords { schema_key, schema: Arc::new(schema), @@ -410,14 +480,14 @@ pub async fn remote_write( hour_buf.records_size += value_str.len(); // real time alert - let need_trigger = !stream_trigger_map.contains_key(&metric_name); + let need_trigger = !stream_trigger_map.contains_key(&stream_name); if need_trigger && !stream_alerts_map.is_empty() { // Start check for alert trigger let key = format!( "{}/{}/{}", &org_id, StreamType::Metrics, - metric_name.clone() + stream_name.clone() ); if let Some(alerts) = stream_alerts_map.get(&key) { let mut trigger_alerts: TriggerAlertData = Vec::new(); @@ -426,15 +496,14 @@ pub async fn remote_write( trigger_alerts.push((alert.clone(), v)); } } - stream_trigger_map.insert(metric_name.clone(), Some(trigger_alerts)); + stream_trigger_map.insert(stream_name.clone(), Some(trigger_alerts)); } - // End check for alert trigger } + // End check for alert trigger } } // write data to wal - let time = start.elapsed().as_secs_f64(); for (stream_name, stream_data) in metric_data_map { // stream_data could be empty if metric value is nan, check it if stream_data.is_empty() { @@ -456,12 +525,16 @@ pub async fn remote_write( let writer = ingester::get_writer(0, org_id, &StreamType::Metrics.to_string(), &stream_name).await; let mut req_stats = write_file(&writer, &stream_name, stream_data).await; - // if let Err(e) = writer.sync().await { - // log::error!("ingestion error while syncing writer: {}", e); - // } - let fns_length: usize = stream_transform_map.values().map(|v| v.len()).sum(); - req_stats.response_time += time; + let fns_length: usize = + stream_executable_pipelines + .get(&stream_name) + .map_or(0, |exec_pl_option| { + exec_pl_option + .as_ref() + .map_or(0, |exec_pl| exec_pl.num_of_func()) + }); + req_stats.response_time = start.elapsed().as_secs_f64(); report_request_usage_stats( req_stats, org_id, @@ -474,13 +547,7 @@ pub async fn remote_write( .await; } - // only one trigger per request, as it updates etcd - for (_, entry) in stream_trigger_map { - if let Some(entry) = entry { - evaluate_trigger(entry).await; - } - } - + let time = start.elapsed().as_secs_f64(); metrics::HTTP_RESPONSE_TIME .with_label_values(&[ "/prometheus/api/v1/write", @@ -500,6 +567,13 @@ pub async fn remote_write( ]) .inc(); + // only one trigger per request, as it updates etcd + for (_, entry) in stream_trigger_map { + if let Some(entry) = entry { + evaluate_trigger(entry).await; + } + } + Ok(()) } diff --git a/src/service/mod.rs b/src/service/mod.rs index 2bcde0e644..74a813f447 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -32,7 +32,7 @@ pub mod logs; pub mod metadata; pub mod metrics; pub mod organization; -pub mod pipelines; +pub mod pipeline; pub mod promql; pub mod schema; pub mod search; diff --git a/src/service/pipeline/batch_execution.rs b/src/service/pipeline/batch_execution.rs new file mode 100644 index 0000000000..a32f7d25a3 --- /dev/null +++ b/src/service/pipeline/batch_execution.rs @@ -0,0 +1,547 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::collections::{HashMap, HashSet}; + +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use config::{ + meta::{ + function::{Transform, VRLResultResolver}, + pipeline::{components::NodeData, Pipeline}, + stream::{StreamParams, StreamType}, + }, + utils::{flatten, json::Value}, +}; +use futures::future::try_join_all; +use tokio::sync::mpsc::{channel, Receiver, Sender}; + +use crate::{ + common::infra::config::QUERY_FUNCTIONS, + service::ingestion::{apply_vrl_fn, compile_vrl_function}, +}; + +#[async_trait] +pub trait PipelineExt: Sync + Send + 'static { + /// Registers the function of all the FunctionNode of this pipeline once for execution. + /// Returns a map of node_id -> VRLResultResolver for quick lookup + async fn register_functions(&self) -> Result>; +} + +#[async_trait] +impl PipelineExt for Pipeline { + async fn register_functions(&self) -> Result> { + let mut vrl_map = HashMap::new(); + for node in &self.nodes { + if let NodeData::Function(func_params) = &node.data { + let transform = get_transforms(&self.org, &func_params.name).await?; + if let Ok(vrl_runtime_config) = compile_vrl_function(&transform.function, &self.org) + { + let registry = vrl_runtime_config + .config + .get_custom::() + .unwrap(); + registry.finish_load(); + vrl_map.insert( + node.get_node_id(), + VRLResultResolver { + program: vrl_runtime_config.program, + fields: vrl_runtime_config.fields, + }, + ); + } + } + } + Ok(vrl_map) + } +} + +#[derive(Debug, Clone)] +pub struct ExecutablePipeline { + source_node_id: String, + sorted_nodes: Vec, + vrl_map: HashMap, + node_map: HashMap, +} + +#[derive(Debug, Clone)] +pub struct ExecutableNode { + node_data: NodeData, + children: Vec, +} + +#[derive(Debug)] +pub struct ExecutablePipelineBulkInputs { + records: Vec, + doc_ids: Vec>, + originals: Vec>, +} + +#[derive(Debug)] +pub struct ExecutablePipelineTraceInputs { + records: Vec, + timestamps: Vec, + services: Vec, + span_names: Vec, + span_status_for_spanmetrics: Vec, + span_kinds: Vec, + span_durations: Vec, +} + +impl ExecutablePipeline { + pub async fn new(pipeline: &Pipeline) -> Result { + let node_map = pipeline + .nodes + .iter() + .map(|node| { + ( + node.get_node_id(), + ExecutableNode { + node_data: node.get_node_data(), + children: pipeline + .edges + .iter() + .filter(|edge| edge.source == node.id) + .map(|edge| edge.target.clone()) + .collect(), + }, + ) + }) + .collect(); + + let vrl_map = pipeline.register_functions().await?; + let sorted_nodes = topological_sort(&node_map)?; + let source_node_id = sorted_nodes[0].to_owned(); + + Ok(Self { + source_node_id, + node_map, + sorted_nodes, + vrl_map, + }) + } + + pub async fn process_batch( + &self, + org_id: &str, + records: Vec, + ) -> Result>> { + let batch_size = records.len(); + log::debug!("[Pipeline]: process batch of size {}", batch_size); + + let (result_sender, mut result_receiver) = + channel::<(usize, StreamParams, Value)>(batch_size); + + let mut node_senders = HashMap::new(); + let mut node_receivers = HashMap::new(); + + for node_id in &self.sorted_nodes { + let (sender, receiver) = channel::<(usize, Value, bool)>(batch_size); + node_senders.insert(node_id.to_string(), sender); + node_receivers.insert(node_id.to_string(), receiver); + } + + // Spawn tasks for each node + let mut node_tasks = Vec::new(); + for (idx, node_id) in self.sorted_nodes.iter().enumerate() { + let org_id_cp = org_id.to_string(); + let node = self.node_map.get(node_id).unwrap().clone(); + let node_receiver = node_receivers.remove(node_id).unwrap(); + let child_senders: Vec<_> = node + .children + .iter() + .map(|child| node_senders.get(child).unwrap().clone()) + .collect(); + let result_sender_cp = node.children.is_empty().then_some(result_sender.clone()); + let vrl_runtime = self.vrl_map.get(node_id).cloned(); + + let task = tokio::spawn(async move { + process_node( + idx, + org_id_cp, + node, + node_receiver, + child_senders, + vrl_runtime, + result_sender_cp, + ) + .await + }); + node_tasks.push(task); + } + + let result_task = tokio::spawn(async move { + // Collect results + log::debug!("[Pipeline]: starts result collecting job"); + let mut count: usize = 0; + let mut results = HashMap::new(); + while let Some((idx, stream_params, record)) = result_receiver.recv().await { + results + .entry(stream_params) + .or_insert(Vec::new()) + .push((idx, record)); + count += 1; + } + log::debug!("[Pipeline]: collected {count} records"); + results + }); + + // Send records to the source node to begin processing + let flattened = { + let source_node = self.node_map.get(&self.source_node_id).unwrap(); + matches!(&source_node.node_data, NodeData::Stream(stream_params) if stream_params.stream_type == StreamType::Metrics) + }; + let source_sender = node_senders.remove(&self.source_node_id).unwrap(); + for (idx, record) in records.into_iter().enumerate() { + if let Err(send_err) = source_sender.send((idx, record, flattened)).await { + log::error!( + "[Pipeline]: Error sending original records into source Node for {send_err}" + ); + break; + } + } + drop(source_sender); + drop(result_sender); + drop(node_senders); + log::debug!("[Pipeline]: All records send into pipeline for processing"); + + // Wait for all node tasks to complete + if let Err(e) = try_join_all(node_tasks).await { + log::error!("[Pipeline] node processing jobs failed: {}", e); + } + + let results = result_task.await.map_err(|e| { + log::error!("[Pipeline] result collecting job failed: {}", e); + anyhow!("[Pipeline] result collecting job failed: {}", e) + })?; + + Ok(results) + } + + pub fn get_all_destination_streams(&self) -> Vec { + self.node_map + .values() + .filter_map(|exec_node| { + if exec_node.children.is_empty() { + if let NodeData::Stream(stream_params) = &exec_node.node_data { + Some(stream_params.clone()) + } else { + None + } + } else { + None + } + }) + .collect() + } + + pub fn num_of_func(&self) -> usize { + self.node_map + .values() + .filter(|exec_node| matches!(exec_node.node_data, NodeData::Function(_))) + .count() + } +} + +impl ExecutablePipelineBulkInputs { + pub fn new() -> Self { + Self { + records: Vec::new(), + doc_ids: Vec::new(), + originals: Vec::new(), + } + } + + pub fn add_input( + &mut self, + record: Value, + doc_id: Option, + original_data: Option, + ) { + self.records.push(record); + self.doc_ids.push(doc_id); + self.originals.push(original_data); + } + + pub fn into_parts(self) -> (Vec, Vec>, Vec>) { + (self.records, self.doc_ids, self.originals) + } +} + +impl Default for ExecutablePipelineBulkInputs { + fn default() -> Self { + Self::new() + } +} + +impl ExecutablePipelineTraceInputs { + pub fn new() -> Self { + Self { + records: Vec::new(), + timestamps: Vec::new(), + services: Vec::new(), + span_names: Vec::new(), + span_status_for_spanmetrics: Vec::new(), + span_kinds: Vec::new(), + span_durations: Vec::new(), + } + } + + #[allow(clippy::too_many_arguments)] + pub fn add_input( + &mut self, + record: Value, + ts: i64, + service: String, + span_name: String, + span_status_for_spanmetric: String, + span_kind: String, + duration: f64, + ) { + self.records.push(record); + self.timestamps.push(ts); + self.services.push(service); + self.span_names.push(span_name); + self.span_status_for_spanmetrics + .push(span_status_for_spanmetric); + self.span_kinds.push(span_kind); + self.span_durations.push(duration); + } + + #[allow(clippy::type_complexity)] + pub fn into_parts( + self, + ) -> ( + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, + ) { + ( + self.records, + self.timestamps, + self.services, + self.span_names, + self.span_status_for_spanmetrics, + self.span_kinds, + self.span_durations, + ) + } +} + +impl Default for ExecutablePipelineTraceInputs { + fn default() -> Self { + Self::new() + } +} + +async fn process_node( + node_id: usize, + org_id: String, + node: ExecutableNode, + mut receiver: Receiver<(usize, Value, bool)>, + mut child_senders: Vec>, + vrl_runtime: Option, + result_sender: Option>, +) -> Result<()> { + let cfg = config::get_config(); + let mut count: usize = 0; + match &node.node_data { + NodeData::Stream(stream_params) => { + if node.children.is_empty() { + log::debug!("[Pipeline]: Leaf node {node_id} starts processing"); + // leaf node: `result_sender` guaranteed to be Some() + // send received results directly via `result_sender` for collection + let result_sender = result_sender.unwrap(); + while let Some((idx, mut record, flattened)) = receiver.recv().await { + if !flattened { + record = + flatten::flatten_with_level(record, cfg.limit.ingest_flatten_level) + .map_err(|e| anyhow!("LeafNode error with flattening: {}", e))?; + } + if let Err(send_err) = result_sender + .send((idx, stream_params.clone(), record)) + .await + { + log::error!( + "[Pipeline]: Leaf node errors sending result for collection caused by: {send_err}" + ); + break; + } + count += 1; + } + log::debug!("[Pipeline]: leaf node {node_id} done processing {count} records"); + } else { + log::debug!("[Pipeline]: source node {node_id} starts processing"); + // source stream node: send received record to all its children + while let Some(item) = receiver.recv().await { + send_to_children(&mut child_senders, item, "StreamNode").await; + count += 1; + } + log::debug!("[Pipeline]: source node {node_id} done processing {count} records"); + } + } + NodeData::Condition(condition_params) => { + log::debug!("[Pipeline]: cond node {node_id} starts processing"); + while let Some((idx, mut record, mut flattened)) = receiver.recv().await { + // value must be flattened before condition params can take effect + if !flattened { + record = flatten::flatten_with_level(record, cfg.limit.ingest_flatten_level) + .map_err(|e| anyhow!("ConditionNode error with flattening: {}", e))?; + flattened = true; + } + // only send to children when passing all condition evaluations + if condition_params + .conditions + .iter() + .all(|cond| cond.evaluate(record.as_object().unwrap())) + { + send_to_children( + &mut child_senders, + (idx, record, flattened), + "ConditionNode", + ) + .await; + count += 1; + } + } + log::debug!("[Pipeline]: cond node {node_id} done processing {count} records"); + } + NodeData::Function(func_params) => { + log::debug!("[Pipeline]: func node {node_id} starts processing"); + let mut runtime = crate::service::ingestion::init_functions_runtime(); + while let Some((idx, mut record, mut flattened)) = receiver.recv().await { + if let Some(vrl_runtime) = &vrl_runtime { + if func_params.after_flatten && !flattened { + record = + flatten::flatten_with_level(record, cfg.limit.ingest_flatten_level) + .map_err(|e| { + anyhow!("FunctionNode error with flattening: {}", e) + })?; + } + record = apply_vrl_fn( + &mut runtime, + vrl_runtime, + record, + &org_id, + &["pipeline".to_string()], + ); + flattened = false; // since apply_vrl_fn can produce unflattened data + } + send_to_children(&mut child_senders, (idx, record, flattened), "FunctionNode") + .await; + count += 1; + } + log::debug!("[Pipeline]: func node {node_id} done processing {count} records"); + } + NodeData::Query(_) => { + // source node for Scheduled pipeline. Directly send to children nodes + log::debug!("[Pipeline]: query node {node_id} starts processing"); + while let Some(item) = receiver.recv().await { + send_to_children(&mut child_senders, item, "QueryNode").await; + count += 1; + } + log::debug!("[Pipeline]: query node {node_id} done processing {count} records"); + } + } + + // all cloned senders dropped when function goes out of scope -> close the channel + + Ok(()) +} + +async fn send_to_children( + child_senders: &mut [Sender<(usize, Value, bool)>], + item: (usize, Value, bool), + node_type: &str, +) { + if child_senders.len() == 1 { + // HACK to avoid cloning + if let Err(send_err) = child_senders[0].send(item).await { + log::error!( + "[Pipeline]: {node_type} errors sending record to its children caused by: {send_err}" + ); + } + } else { + for child_sender in child_senders.iter_mut() { + if let Err(send_err) = child_sender.send(item.clone()).await { + log::error!( + "[Pipeline]: {node_type} errors sending record to its children caused by: {send_err}" + ); + break; + } + } + } +} + +fn topological_sort(node_map: &HashMap) -> Result> { + let mut result = Vec::new(); + let mut visited = HashSet::new(); + let mut temp = HashSet::new(); + + let graph: HashMap> = node_map + .iter() + .map(|(k, v)| (k.clone(), v.children.clone())) + .collect(); + + for node in node_map.keys() { + if let Err(e) = dfs(node, &graph, &mut visited, &mut temp, &mut result) { + return Err(anyhow!( + "[Pipeline] Error to topological sort the nodes in preparing execution: {}", + e + )); + } + } + + result.reverse(); + Ok(result) +} + +fn dfs( + current_node_id: &str, + graph: &HashMap>, + visited: &mut HashSet, + temp: &mut HashSet, + result: &mut Vec, +) -> Result<()> { + if temp.contains(current_node_id) { + return Err(anyhow!("Cyclical pipeline detected.")); + } + if visited.contains(current_node_id) { + return Ok(()); + } + temp.insert(current_node_id.to_string()); + if let Some(neighbors) = graph.get(current_node_id) { + for neighbor in neighbors { + dfs(neighbor, graph, visited, temp, result)?; + } + } + temp.remove(current_node_id); + visited.insert(current_node_id.to_string()); + result.push(current_node_id.to_string()); + Ok(()) +} + +async fn get_transforms(org_id: &str, fn_name: &str) -> Result { + let func_key = format!("{org_id}/{fn_name}"); + if let Some(trans) = QUERY_FUNCTIONS.get(&func_key) { + return Ok(trans.value().clone()); + } + // get from database + crate::service::db::functions::get(org_id, fn_name).await +} diff --git a/src/service/pipeline/mod.rs b/src/service/pipeline/mod.rs new file mode 100644 index 0000000000..640f323444 --- /dev/null +++ b/src/service/pipeline/mod.rs @@ -0,0 +1,286 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::io::Error; + +use actix_web::{ + http::{self, StatusCode}, + HttpResponse, +}; +use config::meta::{ + pipeline::{components::PipelineSource, Pipeline, PipelineList}, + search::SearchEventType, + stream::ListStreamParams, +}; + +use super::db; +use crate::common::{ + meta::{authz::Authz, http::HttpResponse as MetaHttpResponse}, + utils::auth::{remove_ownership, set_ownership}, +}; + +pub mod batch_execution; + +#[tracing::instrument(skip(pipeline))] +pub async fn save_pipeline(mut pipeline: Pipeline) -> Result { + // validate pipeline + if let Err(e) = pipeline.validate() { + return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( + http::StatusCode::BAD_REQUEST.into(), + format!("Invalid Pipeline: {e}"), + ))); + } + + // Save DerivedStream details if there's any + if let PipelineSource::Scheduled(ref mut derived_stream) = &mut pipeline.source { + derived_stream.query_condition.search_event_type = Some(SearchEventType::DerivedStream); + derived_stream.org_id = pipeline.org.clone(); + // save derived_stream to triggers table + if let Err(e) = super::alerts::derived_streams::save( + derived_stream.clone(), + &pipeline.name, + &pipeline.id, + ) + .await + { + return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( + http::StatusCode::BAD_REQUEST.into(), + format!("Failed to save DerivedStream details error: {}", e), + ))); + } + } + + match db::pipeline::set(&pipeline).await { + Err(error) => Ok( + HttpResponse::InternalServerError().json(MetaHttpResponse::message( + http::StatusCode::INTERNAL_SERVER_ERROR.into(), + error.to_string(), + )), + ), + Ok(_) => { + set_ownership(&pipeline.org, "pipelines", Authz::new(&pipeline.id)).await; + Ok(HttpResponse::Ok().json(MetaHttpResponse::message( + http::StatusCode::OK.into(), + "Pipeline saved successfully".to_string(), + ))) + } + } +} + +#[tracing::instrument(skip(pipeline))] +pub async fn update_pipeline(mut pipeline: Pipeline) -> Result { + match db::pipeline::get_by_id(&pipeline.id).await { + Err(_err) => { + return Ok(HttpResponse::NotFound().json(MetaHttpResponse::error( + StatusCode::NOT_FOUND.into(), + format!("Existing Pipeline with ID {} not found", pipeline.id), + ))); + } + Ok(existing_pipeline) => { + if existing_pipeline == pipeline { + return Ok(HttpResponse::Ok().json("No changes found".to_string())); + } + // check version + if existing_pipeline.version != pipeline.version { + return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( + http::StatusCode::BAD_REQUEST.into(), + format!( + "Pipeline with ID {} modified by someone else. Please refresh", + pipeline.id + ), + ))); + } + // if the source is changed, check if the new source exists in another pipeline + if existing_pipeline.source != pipeline.source { + if let Ok(similar_pl) = db::pipeline::get_with_same_source_stream(&pipeline).await { + return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( + http::StatusCode::BAD_REQUEST.into(), + format!( + "The new source already exists in another pipeline in org: {} with name: {}", + similar_pl.org, + similar_pl.name + ), + ))); + } + } + } + }; + + // validate pipeline + if let Err(e) = pipeline.validate() { + return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( + http::StatusCode::BAD_REQUEST.into(), + format!("Invalid Pipeline: {e}"), + ))); + } + + // update the pipeline version + pipeline.version += 1; + + // Save DerivedStream details if there's any + if let PipelineSource::Scheduled(ref mut derived_stream) = &mut pipeline.source { + derived_stream.query_condition.search_event_type = Some(SearchEventType::DerivedStream); + // save derived_stream to triggers table + if let Err(e) = super::alerts::derived_streams::save( + derived_stream.clone(), + &pipeline.name, + &pipeline.id, + ) + .await + { + return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( + http::StatusCode::BAD_REQUEST.into(), + format!("Failed to save DerivedStream details error: {}", e), + ))); + } + } + + match db::pipeline::update(&pipeline).await { + Err(error) => Ok( + HttpResponse::InternalServerError().json(MetaHttpResponse::message( + http::StatusCode::INTERNAL_SERVER_ERROR.into(), + error.to_string(), + )), + ), + Ok(_) => Ok(HttpResponse::Ok().json(MetaHttpResponse::message( + http::StatusCode::OK.into(), + "Pipeline updated successfully".to_string(), + ))), + } +} + +#[tracing::instrument] +pub async fn list_pipelines( + org_id: String, + permitted: Option>, +) -> Result { + if let Ok(pipelines) = db::pipeline::list_by_org(&org_id).await { + let mut result = Vec::new(); + for pipeline in pipelines { + if permitted.is_none() + || permitted + .as_ref() + .unwrap() + .contains(&format!("pipelines:{}", pipeline.id)) + || permitted + .as_ref() + .unwrap() + .contains(&format!("pipelines:_all_{}", org_id)) + { + result.push(pipeline) + } + } + + Ok(HttpResponse::Ok().json(PipelineList { list: result })) + } else { + Ok(HttpResponse::Ok().json(PipelineList { list: vec![] })) + } +} + +#[tracing::instrument] +pub async fn list_streams_with_pipeline(org: &str) -> Result { + match db::pipeline::list_streams_with_pipeline(org).await { + Ok(stream_params) => Ok(HttpResponse::Ok().json(ListStreamParams { + list: stream_params, + })), + Err(_) => Ok(HttpResponse::Ok().json(PipelineList { list: vec![] })), + } +} + +#[tracing::instrument] +pub async fn enable_pipeline( + org_id: &str, + pipeline_id: &str, + value: bool, +) -> Result { + let mut pipeline = match db::pipeline::get_by_id(pipeline_id).await { + Ok(pipeline) => pipeline, + Err(_) => { + return Ok(HttpResponse::NotFound().json(MetaHttpResponse::error( + StatusCode::NOT_FOUND.into(), + format!("Pipeline with ID {pipeline_id} not found"), + ))); + } + }; + + pipeline.enabled = value; + match db::pipeline::update(&pipeline).await { + Err(error) => Ok( + HttpResponse::InternalServerError().json(MetaHttpResponse::message( + http::StatusCode::INTERNAL_SERVER_ERROR.into(), + error.to_string(), + )), + ), + Ok(_) => Ok(HttpResponse::Ok().json(MetaHttpResponse::message( + http::StatusCode::OK.into(), + format!("Pipeline enabled: {value}"), + ))), + } +} + +#[tracing::instrument] +pub async fn delete_pipeline(pipeline_id: &str) -> Result { + let existing_pipeline = match db::pipeline::get_by_id(pipeline_id).await { + Ok(pipeline) => pipeline, + Err(_) => { + return Ok(HttpResponse::NotFound().json(MetaHttpResponse::error( + StatusCode::NOT_FOUND.into(), + format!("Pipeline with ID {pipeline_id} not found"), + ))); + } + }; + + // delete DerivedStream details if there's any + if let PipelineSource::Scheduled(derived_stream) = existing_pipeline.source { + if let Err(error) = super::alerts::derived_streams::delete( + derived_stream, + &existing_pipeline.name, + &existing_pipeline.id, + ) + .await + { + let err_msg = format!( + "Failed to delete due to error deleting associated DerivedStream: {}", + error + ); + return Ok( + HttpResponse::InternalServerError().json(MetaHttpResponse::message( + http::StatusCode::INTERNAL_SERVER_ERROR.into(), + err_msg, + )), + ); + } + } + + let result = db::pipeline::delete(pipeline_id).await; + match result { + Ok(_) => { + remove_ownership( + &existing_pipeline.org, + "pipelines", + Authz::new(&existing_pipeline.id), + ) + .await; + Ok(HttpResponse::Ok().json(MetaHttpResponse::message( + http::StatusCode::OK.into(), + "Pipeline deleted".to_string(), + ))) + } + Err(e) => Ok(HttpResponse::NotFound().json(MetaHttpResponse::error( + http::StatusCode::NOT_FOUND.into(), + e.to_string(), + ))), + } +} diff --git a/src/service/pipelines.rs b/src/service/pipelines.rs deleted file mode 100644 index 6ecddd6481..0000000000 --- a/src/service/pipelines.rs +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright 2024 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -use std::io::Error; - -use actix_web::{ - http::{self, StatusCode}, - HttpResponse, -}; -use config::meta::{ - search::SearchEventType, - stream::{StreamParams, StreamType}, -}; - -use super::db; -use crate::common::{ - infra::config::STREAM_FUNCTIONS, - meta::{ - http::HttpResponse as MetaHttpResponse, - pipelines::{PipeLine, PipeLineList}, - }, -}; - -#[tracing::instrument(skip(pipeline))] -pub async fn save_pipeline(org_id: String, mut pipeline: PipeLine) -> Result { - if check_existing_pipeline( - &org_id, - pipeline.stream_type, - &pipeline.stream_name, - &pipeline.name, - ) - .await - .is_some() - { - return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( - StatusCode::BAD_REQUEST.into(), - "Pipeline already exits".to_string(), - ))); - } - - // Save DerivedStream details if there's any - if let Some(ref mut derived_streams) = &mut pipeline.derived_streams { - for derived_stream in derived_streams { - derived_stream.source = - StreamParams::new(&org_id, &pipeline.stream_name, pipeline.stream_type); - derived_stream.query_condition.search_event_type = Some(SearchEventType::DerivedStream); - if !derived_stream.is_valid() { - return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( - http::StatusCode::BAD_REQUEST.into(), - "Invalid DerivedStream details. Name, destination, and Trigger Period required" - .to_string(), - ))); - } - if let Err(e) = - super::alerts::derived_streams::save(derived_stream.clone(), &pipeline.name).await - { - return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( - http::StatusCode::BAD_REQUEST.into(), - format!("Failed to save DerivedStream details error: {}", e), - ))); - } - } - } - - if let Err(error) = db::pipelines::set(&org_id, &pipeline).await { - return Ok( - HttpResponse::InternalServerError().json(MetaHttpResponse::message( - http::StatusCode::INTERNAL_SERVER_ERROR.into(), - error.to_string(), - )), - ); - } else { - Ok(HttpResponse::Ok().json(MetaHttpResponse::message( - http::StatusCode::OK.into(), - "Pipeline saved successfully".to_string(), - ))) - } -} - -#[tracing::instrument(skip(pipeline))] -pub async fn update_pipeline(org_id: &str, mut pipeline: PipeLine) -> Result { - let existing_pipeline = match check_existing_pipeline( - org_id, - pipeline.stream_type, - &pipeline.stream_name, - &pipeline.name, - ) - .await - { - Some(pipeline) => pipeline, - None => { - return Ok(HttpResponse::NotFound().json(MetaHttpResponse::error( - StatusCode::NOT_FOUND.into(), - "Pipeline not found".to_string(), - ))); - } - }; - if pipeline.eq(&existing_pipeline) { - return Ok(HttpResponse::Ok().json(pipeline)); - } - - // Update DerivedStream details if there's any - if let Some(ref mut derived_streams) = &mut pipeline.derived_streams { - for derived_stream in derived_streams { - derived_stream.source = - StreamParams::new(org_id, &pipeline.stream_name, pipeline.stream_type); - derived_stream.query_condition.search_event_type = Some(SearchEventType::DerivedStream); - if !derived_stream.is_valid() { - return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( - http::StatusCode::BAD_REQUEST.into(), - "Invalid DerivedStream details. Name, destination, and Trigger Period required" - .to_string(), - ))); - } - // derived_streams::save updates existing triggers when found - if let Err(e) = - super::alerts::derived_streams::save(derived_stream.clone(), &pipeline.name).await - { - return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error( - http::StatusCode::BAD_REQUEST.into(), - format!("Failed to update DerivedStream details with error {}", e), - ))); - } - } - } - - if let Err(error) = db::pipelines::set(org_id, &pipeline).await { - return Ok( - HttpResponse::InternalServerError().json(MetaHttpResponse::message( - http::StatusCode::INTERNAL_SERVER_ERROR.into(), - error.to_string(), - )), - ); - } - Ok(HttpResponse::Ok().json(MetaHttpResponse::message( - http::StatusCode::OK.into(), - "Pipeline updated successfully".to_string(), - ))) -} - -#[tracing::instrument] -pub async fn list_pipelines( - org_id: String, - permitted: Option>, -) -> Result { - if let Ok(pipelines) = db::pipelines::list(&org_id).await { - let mut result = Vec::new(); - for pipeline in pipelines { - if permitted.is_none() - || permitted - .as_ref() - .unwrap() - .contains(&format!("logs:{}", pipeline.stream_name)) - || permitted - .as_ref() - .unwrap() - .contains(&format!("logs:_all_{}", org_id)) - { - let fn_list = STREAM_FUNCTIONS - .get(&format!( - "{}/{}/{}", - org_id, &pipeline.stream_type, &pipeline.stream_name - )) - .map(|val| val.value().clone()); - result.push(pipeline.into_response(fn_list)); - } - } - - Ok(HttpResponse::Ok().json(PipeLineList { list: result })) - } else { - Ok(HttpResponse::Ok().json(PipeLineList { list: vec![] })) - } -} - -#[tracing::instrument] -pub async fn delete_pipeline( - pipeline_name: &str, - source: StreamParams, -) -> Result { - let existing_pipeline = match check_existing_pipeline( - &source.org_id, - source.stream_type, - &source.stream_name, - pipeline_name, - ) - .await - { - Some(pipeline) => pipeline, - None => { - return Ok(HttpResponse::NotFound().json(MetaHttpResponse::error( - StatusCode::NOT_FOUND.into(), - "Pipeline not found".to_string(), - ))); - } - }; - - // delete DerivedStream details if there's any - if let Some(derived_streams) = existing_pipeline.derived_streams { - for derived_stream in derived_streams { - if let Err(error) = - super::alerts::derived_streams::delete(derived_stream, &existing_pipeline.name) - .await - { - return Ok( - HttpResponse::InternalServerError().json(MetaHttpResponse::message( - http::StatusCode::INTERNAL_SERVER_ERROR.into(), - error.to_string(), - )), - ); - } - } - } - - let result = db::pipelines::delete( - &source.org_id, - source.stream_type, - &source.stream_name, - pipeline_name, - ) - .await; - match result { - Ok(_) => Ok(HttpResponse::Ok().json(MetaHttpResponse::message( - http::StatusCode::OK.into(), - "Pipeline deleted".to_string(), - ))), - Err(e) => Ok(HttpResponse::NotFound().json(MetaHttpResponse::error( - http::StatusCode::NOT_FOUND.into(), - e.to_string(), - ))), - } -} - -async fn check_existing_pipeline( - org_id: &str, - stream_type: StreamType, - stream_name: &str, - name: &str, -) -> Option { - match db::pipelines::get(org_id, stream_type, stream_name, name).await { - Ok(pipeline) => Some(pipeline), - Err(_) => None, - } -} diff --git a/src/service/search/cluster/http.rs b/src/service/search/cluster/http.rs index 70b399457e..7b78e2391c 100644 --- a/src/service/search/cluster/http.rs +++ b/src/service/search/cluster/http.rs @@ -17,7 +17,7 @@ use std::sync::Arc; use ::datafusion::arrow::record_batch::RecordBatch; use config::{ - meta::search, + meta::{function::VRLResultResolver, search}, utils::{ arrow::record_batches_to_json_rows, flatten, @@ -28,10 +28,7 @@ use infra::errors::{Error, ErrorCodes, Result}; use proto::cluster_rpc::SearchQuery; use vector_enrichment::TableRegistry; -use crate::{ - common::meta::functions::VRLResultResolver, - service::search::{cluster::flight, request::Request, sql::Sql}, -}; +use crate::service::search::{cluster::flight, request::Request, sql::Sql}; #[tracing::instrument(name = "service:search:cluster", skip_all)] pub async fn search( @@ -135,7 +132,7 @@ pub async fn search( program: program.program.clone(), fields: program.fields.clone(), }, - &json::Value::Array( + json::Value::Array( json_rows .into_iter() .filter(|v| !v.is_empty()) @@ -164,7 +161,7 @@ pub async fn search( program: program.program.clone(), fields: program.fields.clone(), }, - &json::Value::Object(hit), + json::Value::Object(hit), &sql.org_id, &sql.stream_names, ); diff --git a/src/service/search/datafusion/udf/mod.rs b/src/service/search/datafusion/udf/mod.rs index 7ec3e7fa33..6ff1b03975 100644 --- a/src/service/search/datafusion/udf/mod.rs +++ b/src/service/search/datafusion/udf/mod.rs @@ -13,9 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use config::utils::json; - -use crate::common::meta::functions::ZoFunction; +use config::{meta::function::ZoFunction, utils::json}; pub(crate) mod arr_descending_udf; pub(crate) mod arrcount_udf; diff --git a/src/service/search/mod.rs b/src/service/search/mod.rs index a3bb3aa952..d583294b32 100644 --- a/src/service/search/mod.rs +++ b/src/service/search/mod.rs @@ -62,7 +62,7 @@ use { use super::usage::report_request_usage_stats; use crate::{ - common::{infra::cluster as infra_cluster, meta, utils::functions}, + common::{infra::cluster as infra_cluster, utils}, handler::grpc::request::search::Searcher, }; @@ -169,7 +169,7 @@ pub async fn search_multi( req.query.query_fn = query_fn.clone(); } - for fn_name in functions::get_all_transform_keys(org_id).await { + for fn_name in utils::functions::get_all_transform_keys(org_id).await { if req.query.sql.contains(&format!("{}(", fn_name)) { req.query.uses_zo_fn = true; break; @@ -234,7 +234,7 @@ pub async fn search_multi( if apply_over_hits { input_fn = RESULT_ARRAY.replace(&input_fn, "").to_string(); } - let mut runtime = crate::common::utils::functions::init_vrl_runtime(); + let mut runtime = utils::functions::init_vrl_runtime(); let program = match crate::service::ingestion::compile_vrl_function(&input_fn, org_id) { Ok(program) => { let registry = program @@ -256,11 +256,11 @@ pub async fn search_multi( if apply_over_hits { let ret_val = crate::service::ingestion::apply_vrl_fn( &mut runtime, - &meta::functions::VRLResultResolver { + &config::meta::function::VRLResultResolver { program: program.program.clone(), fields: program.fields.clone(), }, - &json::Value::Array(multi_res.hits), + json::Value::Array(multi_res.hits), org_id, &[stream_name.clone()], ); @@ -292,11 +292,11 @@ pub async fn search_multi( .filter_map(|hit| { let ret_val = crate::service::ingestion::apply_vrl_fn( &mut runtime, - &meta::functions::VRLResultResolver { + &config::meta::function::VRLResultResolver { program: program.program.clone(), fields: program.fields.clone(), }, - &hit, + hit, org_id, &[stream_name.clone()], ); diff --git a/src/service/traces/mod.rs b/src/service/traces/mod.rs index e48d242cdb..a224ee9a76 100644 --- a/src/service/traces/mod.rs +++ b/src/service/traces/mod.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{collections::HashMap, io::Error, sync::Arc}; +use std::{collections::HashMap, io::Error, sync::Arc, time::Instant}; use actix_web::{http, web, HttpResponse}; use bytes::BytesMut; @@ -22,6 +22,7 @@ use config::{ cluster::LOCAL_NODE, get_config, meta::{ + alerts::alert::Alert, stream::{PartitionTimeLevel, StreamParams, StreamPartition, StreamType}, usage::{RequestStats, UsageType}, }, @@ -40,14 +41,15 @@ use opentelemetry_proto::tonic::{ }; use prost::Message; +use super::{logs::O2IngestJsonData, pipeline::batch_execution::ExecutablePipelineTraceInputs}; use crate::{ common::meta::{ - alerts::alert::Alert, http::HttpResponse as MetaHttpResponse, stream::SchemaRecords, traces::{Event, Span, SpanLink, SpanLinkContext, SpanRefType}, }, service::{ + alerts::alert::AlertExt, db, format_stream_name, ingestion::{evaluate_trigger, grpc::get_val, write_file, TriggerAlertData}, metadata::{ @@ -176,18 +178,19 @@ pub async fn handle_trace_request( .expect("configuration error: too large ingest_allowed_upto")) .timestamp_micros(); - // Start Register Transforms for stream - let mut runtime = crate::service::ingestion::init_functions_runtime(); - let (_, local_trans, stream_vrl_map) = crate::service::ingestion::register_stream_functions( + // Start retrieving associated pipeline and construct pipeline params + let executable_pipeline = crate::service::ingestion::get_stream_executable_pipeline( org_id, - &StreamType::Traces, &traces_stream_name, - ); - // End Register Transforms for stream + &StreamType::Traces, + ) + .await; + let mut stream_pipeline_inputs = ExecutablePipelineTraceInputs::new(); + // End pipeline params construction let mut service_name: String = traces_stream_name.to_string(); let res_spans = request.resource_spans; - let mut json_data = Vec::with_capacity(res_spans.len()); + let mut json_data_by_stream = HashMap::new(); let mut span_metrics = Vec::with_capacity(res_spans.len()); let mut partial_success = ExportTracePartialSuccess::default(); for res_span in res_spans { @@ -340,93 +343,166 @@ pub async fn handle_trace_request( let mut value: json::Value = json::to_value(local_val).unwrap(); - // JSON Flattening - value = flatten::flatten(value).map_err(|e| { - std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()) - })?; - - // Start row based transform - if !local_trans.is_empty() { - value = crate::service::ingestion::apply_stream_functions( - &local_trans, + if executable_pipeline.is_some() { + stream_pipeline_inputs.add_input( value, - &stream_vrl_map, - org_id, - &traces_stream_name, - &mut runtime, - ) - .map_err(|e| { + timestamp, + service_name.to_owned(), + span.name, + span_status_for_spanmetric, + span.kind.to_string(), + ((end_time - start_time) / 1_000_000) as f64, + ); + } else { + // JSON Flattening + value = flatten::flatten(value).map_err(|e| { std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()) })?; + + // get json object + let mut record_val = match value.take() { + json::Value::Object(mut v) => { + // build span metrics item + let sm = crate::job::metrics::TraceMetricsItem { + organization: org_id.to_string(), + traces_stream_name: traces_stream_name.clone(), + service_name: service_name.clone(), + span_name: v + .remove("o2_span_metrics_name") + .map_or(span.name.clone(), |name| { + name.as_str().unwrap().to_string() + }), + span_status: span_status_for_spanmetric, + span_kind: span.kind.to_string(), + duration: ((end_time - start_time) / 1_000_000) as f64, /* milliseconds */ + span_id: v["span_id"].to_string(), + }; + span_metrics.push(sm); + v + } + _ => { + log::error!( + "[TRACE] stream did not receive a valid json object, trace_id: {}", + trace_id + ); + return Ok(HttpResponse::InternalServerError().json( + MetaHttpResponse::error( + http::StatusCode::INTERNAL_SERVER_ERROR.into(), + "stream did not receive a valid json objectt".into(), + ), + )); + } + }; + + // add timestamp + record_val.insert( + cfg.common.column_timestamp.clone(), + json::Value::Number(timestamp.into()), + ); + let (ts_data, _) = json_data_by_stream + .entry(traces_stream_name.to_string()) + .or_insert((Vec::new(), None)); + ts_data.push((timestamp, record_val)); } - // End row based transform + } + } + } - // get json object - let mut record_val = match value.take() { - json::Value::Object(mut v) => { - // build span metrics item - let sm = crate::job::metrics::TraceMetricsItem { - organization: org_id.to_string(), - traces_stream_name: traces_stream_name.clone(), - service_name: service_name.clone(), - span_name: v - .remove("o2_span_metrics_name") - .map_or(span.name.clone(), |name| { - name.as_str().unwrap().to_string() - }), - span_status: span_status_for_spanmetric, - span_kind: span.kind.to_string(), - duration: ((end_time - start_time) / 1_000_000) as f64, // milliseconds - span_id: v["span_id"].to_string(), - }; - span_metrics.push(sm); - v - } - _ => { - log::error!( - "[TRACE] stream functions did not return valid json object, trace_id: {}", - trace_id - ); - return Ok(HttpResponse::InternalServerError().json( - MetaHttpResponse::error( - http::StatusCode::INTERNAL_SERVER_ERROR.into(), - "stream functions did not return valid json object".into(), - ), - )); - } - }; - - // add timestamp - record_val.insert( - cfg.common.column_timestamp.clone(), - json::Value::Number(timestamp.into()), + // batch process records through pipeline + if let Some(exec_pl) = &executable_pipeline { + let ( + records, + timestamps, + services, + span_names, + span_status_for_spanmetrics, + span_kinds, + span_durations, + ) = stream_pipeline_inputs.into_parts(); + let records_count = records.len(); + match exec_pl.process_batch(org_id, records).await { + Err(e) => { + log::error!( + "[Trace] pipeline({}/{}) batch execution error: {}.", + org_id, + traces_stream_name, + e ); - json_data.push((timestamp, record_val)); + partial_success.rejected_spans += records_count as i64; + partial_success.error_message = format!("Pipeline batch execution error: {}", e); + } + Ok(pl_results) => { + for (stream_params, stream_pl_results) in pl_results { + if stream_params.stream_type != StreamType::Traces { + continue; + } + + for (idx, mut res) in stream_pl_results { + // get json object + let mut record_val = match res.take() { + json::Value::Object(mut v) => { + // build span metrics item + let sm = crate::job::metrics::TraceMetricsItem { + organization: org_id.to_string(), + traces_stream_name: stream_params.stream_name.to_string(), + service_name: services[idx].to_owned(), + span_name: v + .remove("o2_span_metrics_name") + .map_or(span_names[idx].to_owned(), |name| { + name.as_str().unwrap().to_string() + }), + span_status: span_status_for_spanmetrics[idx].to_owned(), + span_kind: span_kinds[idx].to_owned(), + duration: span_durations[idx], // milliseconds + span_id: v["span_id"].to_string(), + }; + span_metrics.push(sm); + v + } + _ => { + log::error!("[TRACE] stream did not receive a valid json object"); + return Ok(HttpResponse::InternalServerError().json( + MetaHttpResponse::error( + http::StatusCode::INTERNAL_SERVER_ERROR.into(), + "stream did not receive a valid json object".into(), + ), + )); + } + }; + + // add timestamp + let timestamp = timestamps[idx]; + record_val.insert( + cfg.common.column_timestamp.clone(), + json::Value::Number(timestamp.into()), + ); + let (ts_data, _) = json_data_by_stream + .entry(traces_stream_name.to_string()) + .or_insert((Vec::new(), None)); + ts_data.push((timestamp, record_val)); + } + } } } } // if no data, fast return - if json_data.is_empty() { - log::error!("[TRACE] no data to write"); + if json_data_by_stream.is_empty() { return format_response(partial_success, req_type); } - let mut req_stats = match write_traces(org_id, &traces_stream_name, json_data).await { - Ok(v) => v, - Err(e) => { - log::error!("Error while writing traces: {}", e); - return Ok( - HttpResponse::InternalServerError().json(MetaHttpResponse::error( - http::StatusCode::INTERNAL_SERVER_ERROR.into(), - format!("error while writing trace data: {e}",), - )), - ); - } - }; - let time = start.elapsed().as_secs_f64(); - req_stats.response_time = time; + if let Err(e) = write_traces_by_stream(org_id, (started_at, &start), json_data_by_stream).await + { + log::error!("Error while writing traces: {}", e); + return Ok( + HttpResponse::InternalServerError().json(MetaHttpResponse::error( + http::StatusCode::INTERNAL_SERVER_ERROR.into(), + format!("error while writing trace data: {e}",), + )), + ); + } + let time = start.elapsed().as_secs_f64(); let ep = match req_type { RequestType::Grpc => "/grpc/otlp/traces", _ => "/api/otlp/v1/traces", @@ -472,18 +548,6 @@ pub async fn handle_trace_request( ]) .inc(); - // metric + data usage - report_request_usage_stats( - req_stats, - org_id, - &traces_stream_name, - StreamType::Traces, - UsageType::Traces, - 0, - started_at, - ) - .await; - format_response(partial_success, req_type) } @@ -532,6 +596,35 @@ fn format_response( } } +async fn write_traces_by_stream( + org_id: &str, + time_stats: (i64, &Instant), + json_data_by_stream: HashMap, +) -> Result<(), Error> { + for (traces_stream_name, (json_data, fn_num)) in json_data_by_stream { + let mut req_stats = match write_traces(org_id, &traces_stream_name, json_data).await { + Ok(v) => v, + Err(e) => { + return Err(e); + } + }; + let time = time_stats.1.elapsed().as_secs_f64(); + req_stats.response_time = time; + // metric + data usage + report_request_usage_stats( + req_stats, + org_id, + &traces_stream_name, + StreamType::Traces, + UsageType::Traces, + fn_num.map_or(0, |cnt| cnt as u16), + time_stats.0, + ) + .await; + } + Ok(()) +} + async fn write_traces( org_id: &str, stream_name: &str, diff --git a/tests/api-testing/tests/test_functions.py b/tests/api-testing/tests/test_functions.py index 225e9c5e3b..b05d58fe8f 100644 --- a/tests/api-testing/tests/test_functions.py +++ b/tests/api-testing/tests/test_functions.py @@ -47,7 +47,8 @@ def test_e2e_invalidfunction(create_session, base_url): session = create_session # Create a function org_id = "default" - payload = {"name": "...", "function": "...", "params": "row", "transType": 0} + payload = {"name": "...", "function": "...", + "params": "row", "transType": 0} resp_create_function = session.post( f"{base_url}api/{org_id}/functions", json=payload @@ -170,23 +171,6 @@ def test_e2e_functionnotfound(create_session, base_url): ), f"Function not found, but got {resp_delete_function.status_code} {resp_delete_function.content}" -def test_e2e_allfunctionstreams(create_session, base_url): - """Running an E2E test for all streams under functions.""" - - session = create_session - org_id = "default" - stream_name = "test" - - resp_get_streamfunction = session.get( - f"{base_url}api/{org_id}/streams/{stream_name}/functions" - ) - - print(resp_get_streamfunction.content) - assert ( - resp_get_streamfunction.status_code == 200 - ), f"Get all functions streams list 200, but got {resp_get_streamfunction.status_code} {resp_get_streamfunction.content}" - - def test_e2e_addDeleteStreamFunction(create_session, base_url): """Running an E2E test for add stream to a function and delete.""" @@ -212,31 +196,6 @@ def test_e2e_addDeleteStreamFunction(create_session, base_url): resp_create_function.status_code == 200 ), f"Expected 200, but got {resp_create_function.status_code} {resp_create_function.content}" - resp_get_streamfunction = session.get( - f"{base_url}api/{org_id}/streams/{stream_name}/functions" - ) - - print(resp_get_streamfunction.content) - assert ( - resp_get_streamfunction.status_code == 200 - ), f"Get all functions streams list 200, but got {resp_get_streamfunction.status_code} {resp_get_streamfunction.content}" - - payload = {"order": 2} - resp_add_streamfunction = session.put( - f"{base_url}api/{org_id}/streams/{stream_name}/functions/pytestfunction", - json=payload, - ) - - print(resp_add_streamfunction.content) - assert ( - resp_add_streamfunction.status_code == 200 - ), f"Add stream to function 200, but got {resp_add_streamfunction.status_code} {resp_add_streamfunction.content}" - resp_delete_streamfunction = session.delete( - f"{base_url}api/{org_id}/streams/{stream_name}/functions/pytestfunction" - ) - assert ( - resp_delete_streamfunction.status_code == 200 - ), f"Deleted the function from stream 200, but got {resp_delete_streamfunction.status_code} {resp_delete_streamfunction.content}" resp_delete_function = session.delete( f"{base_url}api/{org_id}/functions/pytestfunction" ) diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 825eac8da6..db9386644e 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -22,17 +22,18 @@ mod tests { use arrow_flight::flight_service_server::FlightServiceServer; use bytes::{Bytes, BytesMut}; use chrono::{Duration, Utc}; - use config::{get_config, utils::json}; + use config::{ + get_config, + meta::alerts::{ + alert::Alert, + destinations::{Destination, DestinationType}, + Operator, QueryCondition, TriggerCondition, + }, + utils::json, + }; use infra::scheduler::Trigger; use openobserve::{ - common::meta::{ - alerts::{ - alert::Alert, - destinations::{Destination, DestinationType}, - QueryCondition, TriggerCondition, - }, - dashboards::{v1, Dashboard, Dashboards}, - }, + common::meta::dashboards::{v1, Dashboard, Dashboards}, handler::{ grpc::{auth::check_auth, flight::FlightServiceImpl}, http::router::*, @@ -150,10 +151,7 @@ mod tests { // functions e2e_post_function().await; - e2e_add_stream_function().await; e2e_list_functions().await; - e2e_list_stream_functions().await; - e2e_remove_stream_function().await; e2e_delete_function().await; // search @@ -480,34 +478,6 @@ mod tests { assert!(resp.status().is_success()); } - async fn e2e_add_stream_function() { - let auth = setup(); - let body_str = r#"{ - "order":1 - }"#; - let app = test::init_service( - App::new() - .app_data(web::JsonConfig::default().limit(get_config().limit.req_json_limit)) - .app_data(web::PayloadConfig::new( - get_config().limit.req_payload_limit, - )) - .configure(get_service_routes) - .configure(get_basic_routes), - ) - .await; - let req = test::TestRequest::put() - .uri(&format!( - "/api/{}/streams/{}/functions/{}", - "e2e", "olympics_schema", "e2etestfn" - )) - .insert_header(ContentType::json()) - .append_header(auth) - .set_payload(body_str) - .to_request(); - let resp = test::call_service(&app, req).await; - assert!(resp.status().is_success()); - } - async fn e2e_list_functions() { let auth = setup(); let app = test::init_service( @@ -529,30 +499,6 @@ mod tests { assert!(resp.status().is_success()); } - async fn e2e_list_stream_functions() { - let auth = setup(); - let app = test::init_service( - App::new() - .app_data(web::JsonConfig::default().limit(get_config().limit.req_json_limit)) - .app_data(web::PayloadConfig::new( - get_config().limit.req_payload_limit, - )) - .configure(get_service_routes) - .configure(get_basic_routes), - ) - .await; - let req = test::TestRequest::get() - .uri(&format!( - "/api/{}/streams/{}/functions", - "e2e", "olympics_schema" - )) - .insert_header(ContentType::json()) - .append_header(auth) - .to_request(); - let resp = test::call_service(&app, req).await; - assert!(resp.status().is_success()); - } - async fn e2e_delete_function() { let auth = setup(); let app = test::init_service( @@ -574,30 +520,6 @@ mod tests { assert!(resp.status().is_success()); } - async fn e2e_remove_stream_function() { - let auth = setup(); - let app = test::init_service( - App::new() - .app_data(web::JsonConfig::default().limit(get_config().limit.req_json_limit)) - .app_data(web::PayloadConfig::new( - get_config().limit.req_payload_limit, - )) - .configure(get_service_routes) - .configure(get_basic_routes), - ) - .await; - let req = test::TestRequest::delete() - .uri(&format!( - "/api/{}/streams/{}/functions/{}", - "e2e", "olympics_schema", "e2etestfn" - )) - .insert_header(ContentType::json()) - .append_header(auth) - .to_request(); - let resp = test::call_service(&app, req).await; - assert!(resp.status().is_success()); - } - async fn e2e_search() { let auth = setup(); let body_str = r#"{ @@ -2015,7 +1937,7 @@ mod tests { threshold: 1, silence: 0, frequency: 3600, - operator: openobserve::common::meta::alerts::Operator::GreaterThanEquals, + operator: Operator::GreaterThanEquals, ..Default::default() }, destinations: vec!["slack".to_string()], diff --git a/tests/ui-testing/playwright-tests/pipeline.spec.js b/tests/ui-testing/playwright-tests/pipeline.spec.js index 22866b631d..ca0b5e5d34 100644 --- a/tests/ui-testing/playwright-tests/pipeline.spec.js +++ b/tests/ui-testing/playwright-tests/pipeline.spec.js @@ -118,7 +118,7 @@ test.describe("Pipeline testcases", () => { // const streams = page.waitForResponse("**/api/default/streams**"); }); - test('display error if mandatory fields not added on pipeline creation UI ', async ({ page }) => { + test.skip('display error if mandatory fields not added on pipeline creation UI ', async ({ page }) => { await page.locator('[data-test="menu-link-\\/pipeline-item"]').click(); await page.getByRole('main').locator('div').filter({ hasText: 'Enrichment TablesFunctionsStream AssociationStream' }).first().click(); await page.locator('[data-test="stream-pipelines-tab"]').click(); @@ -129,7 +129,7 @@ test.describe("Pipeline testcases", () => { await page.getByText('Field is required!').click(); }); - test('should close Add pipeline UI on clicking close icon', async ({ page }) => { + test.skip('should close Add pipeline UI on clicking close icon', async ({ page }) => { await page.locator('[data-test="menu-link-\\/pipeline-item"]').click(); await page.getByRole('main').locator('div').filter({ hasText: 'Enrichment TablesFunctionsStream AssociationStream' }).first().click(); await page.locator('[data-test="stream-pipelines-tab"]').click(); @@ -144,7 +144,7 @@ test.describe("Pipeline testcases", () => { }); - test('should add and delete pipeline', async ({ page }) => { + test.skip('should add and delete pipeline', async ({ page }) => { await page.locator('[data-test="menu-link-\\/pipeline-item"]').click(); await page.getByRole('main').locator('div').filter({ hasText: 'Enrichment TablesFunctionsStream AssociationStream' }).first().click(); await page.locator('[data-test="stream-pipelines-tab"]').click(); @@ -168,7 +168,7 @@ test.describe("Pipeline testcases", () => { await page.locator('[data-test="confirm-button"]').click(); }); - test('should add function to pipeline', async ({ page }) => { + test.skip('should add function to pipeline', async ({ page }) => { await page.locator('[data-test="menu-link-\\/pipeline-item"]').click(); await page.getByRole('main').locator('div').filter({ hasText: 'Enrichment TablesFunctionsStream AssociationStream' }).first().click(); await page.locator('[data-test="stream-pipelines-tab"]').click(); @@ -229,7 +229,7 @@ test.describe("Pipeline testcases", () => { - test('should display error when no name added while adding function to pipeline', async ({ page }) => { + test.skip('should display error when no name added while adding function to pipeline', async ({ page }) => { await page.locator('[data-test="menu-link-\\/pipeline-item"]').click(); await page.getByRole('main').locator('div').filter({ hasText: 'Enrichment TablesFunctionsStream AssociationStream' }).first().click(); await page.locator('[data-test="stream-pipelines-tab"]').click(); @@ -278,7 +278,7 @@ test.describe("Pipeline testcases", () => { }); - test('should add streamroute node', async ({ page }) => { + test.skip('should add streamroute node', async ({ page }) => { await page.locator('[data-test="menu-link-\\/pipeline-item"]').click(); await page.getByRole('main').locator('div').filter({ hasText: 'Enrichment TablesFunctionsStream AssociationStream' }).first().click(); await page.locator('[data-test="stream-pipelines-tab"]').click(); @@ -335,7 +335,7 @@ test.describe("Pipeline testcases", () => { }); - test('should display error when only blank spaces added in stream route name', async ({ page }) => { + test.skip('should display error when only blank spaces added in stream route name', async ({ page }) => { await page.locator('[data-test="menu-link-\\/pipeline-item"]').click(); await page.getByRole('main').locator('div').filter({ hasText: 'Enrichment TablesFunctionsStream AssociationStream' }).first().click(); await page.locator('[data-test="stream-pipelines-tab"]').click(); @@ -386,5 +386,5 @@ test.describe("Pipeline testcases", () => { await page.locator('[data-test="confirm-button"]').click(); await deletePipeline(page, randomPipelineName); }); - + }) \ No newline at end of file diff --git a/web/package-lock.json b/web/package-lock.json index 032c746494..b214b9d75f 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -18,6 +18,9 @@ "@quasar/extras": "^1.16.12", "@tanstack/vue-table": "^8.19.3", "@tanstack/vue-virtual": "^3.8.4", + "@vue-flow/background": "^1.3.0", + "@vue-flow/controls": "^1.1.2", + "@vue-flow/core": "^1.41.1", "cron-parser": "^4.9.0", "d3-scale": "^4.0.2", "date-fns": "^3.6.0", @@ -2859,6 +2862,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/web-bluetooth": { + "version": "0.0.20", + "resolved": "https://registry.npmjs.org/@types/web-bluetooth/-/web-bluetooth-0.0.20.tgz", + "integrity": "sha512-g9gZnnXVq7gM7v3tJCWV/qw7w+KeOlSHAhgF9RytFyifW6AF61hdT2ucrYhPq9hLs5JIryeupHV3qGk95dH9ow==", + "license": "MIT" + }, "node_modules/@types/webpack": { "version": "4.41.38", "resolved": "https://registry.npmjs.org/@types/webpack/-/webpack-4.41.38.tgz", @@ -3298,6 +3307,41 @@ "vscode-uri": "^3.0.8" } }, + "node_modules/@vue-flow/background": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@vue-flow/background/-/background-1.3.0.tgz", + "integrity": "sha512-fu/8s9wzSOQIitnSTI10XT3bzTtagh4h8EF2SWwtlDklOZjAaKy75lqv4htHa3wigy/r4LGCOGwLw3Pk88/AxA==", + "license": "MIT", + "peerDependencies": { + "@vue-flow/core": "^1.23.0", + "vue": "^3.3.0" + } + }, + "node_modules/@vue-flow/controls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@vue-flow/controls/-/controls-1.1.2.tgz", + "integrity": "sha512-6dtl/JnwDBNau5h3pDBdOCK6tdxiVAOL3cyruRL61gItwq5E97Hmjmj2BIIqX2p7gU1ENg3z80Z4zlu58fGlsg==", + "license": "MIT", + "peerDependencies": { + "@vue-flow/core": "^1.23.0", + "vue": "^3.3.0" + } + }, + "node_modules/@vue-flow/core": { + "version": "1.41.1", + "resolved": "https://registry.npmjs.org/@vue-flow/core/-/core-1.41.1.tgz", + "integrity": "sha512-/w/FHXxnOE1QFsEQMVJGD4DGinlhj8ZDJmumWMFNJztk94d28iK8BWRQD4Cy305Jnt29sf3owHgliGXHIZDLTg==", + "license": "MIT", + "dependencies": { + "@vueuse/core": "^10.5.0", + "d3-drag": "^3.0.0", + "d3-selection": "^3.0.0", + "d3-zoom": "^3.0.0" + }, + "peerDependencies": { + "vue": "^3.3.0" + } + }, "node_modules/@vue/babel-helper-vue-transform-on": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/@vue/babel-helper-vue-transform-on/-/babel-helper-vue-transform-on-1.2.2.tgz", @@ -3603,6 +3647,94 @@ "dev": true, "license": "MIT" }, + "node_modules/@vueuse/core": { + "version": "10.11.1", + "resolved": "https://registry.npmjs.org/@vueuse/core/-/core-10.11.1.tgz", + "integrity": "sha512-guoy26JQktXPcz+0n3GukWIy/JDNKti9v6VEMu6kV2sYBsWuGiTU8OWdg+ADfUbHg3/3DlqySDe7JmdHrktiww==", + "license": "MIT", + "dependencies": { + "@types/web-bluetooth": "^0.0.20", + "@vueuse/metadata": "10.11.1", + "@vueuse/shared": "10.11.1", + "vue-demi": ">=0.14.8" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/core/node_modules/vue-demi": { + "version": "0.14.10", + "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.10.tgz", + "integrity": "sha512-nMZBOwuzabUO0nLgIcc6rycZEebF6eeUfaiQx9+WSk8e29IbLvPU9feI6tqW4kTo3hvoYAJkMh8n8D0fuISphg==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "vue-demi-fix": "bin/vue-demi-fix.js", + "vue-demi-switch": "bin/vue-demi-switch.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@vue/composition-api": "^1.0.0-rc.1", + "vue": "^3.0.0-0 || ^2.6.0" + }, + "peerDependenciesMeta": { + "@vue/composition-api": { + "optional": true + } + } + }, + "node_modules/@vueuse/metadata": { + "version": "10.11.1", + "resolved": "https://registry.npmjs.org/@vueuse/metadata/-/metadata-10.11.1.tgz", + "integrity": "sha512-IGa5FXd003Ug1qAZmyE8wF3sJ81xGLSqTqtQ6jaVfkeZ4i5kS2mwQF61yhVqojRnenVew5PldLyRgvdl4YYuSw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/shared": { + "version": "10.11.1", + "resolved": "https://registry.npmjs.org/@vueuse/shared/-/shared-10.11.1.tgz", + "integrity": "sha512-LHpC8711VFZlDaYUXEBbFBCQ7GS3dVU9mjOhhMhXP6txTV4EhYQg/KGnQuvt/sPAtoUKq7VVUnL6mVtFoL42sA==", + "license": "MIT", + "dependencies": { + "vue-demi": ">=0.14.8" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/shared/node_modules/vue-demi": { + "version": "0.14.10", + "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.10.tgz", + "integrity": "sha512-nMZBOwuzabUO0nLgIcc6rycZEebF6eeUfaiQx9+WSk8e29IbLvPU9feI6tqW4kTo3hvoYAJkMh8n8D0fuISphg==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "vue-demi-fix": "bin/vue-demi-fix.js", + "vue-demi-switch": "bin/vue-demi-switch.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@vue/composition-api": "^1.0.0-rc.1", + "vue": "^3.0.0-0 || ^2.6.0" + }, + "peerDependenciesMeta": { + "@vue/composition-api": { + "optional": true + } + } + }, "node_modules/@webassemblyjs/ast": { "version": "1.12.1", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz", @@ -5875,6 +6007,38 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "license": "BSD-3-Clause", "engines": { "node": ">=12" } @@ -5891,6 +6055,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "license": "ISC", "dependencies": { "d3-color": "1 - 3" }, @@ -5913,6 +6078,15 @@ "node": ">=12" } }, + "node_modules/d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/d3-time": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", @@ -5935,6 +6109,50 @@ "node": ">=12" } }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "d3-selection": "2 - 3" + } + }, + "node_modules/d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", diff --git a/web/package.json b/web/package.json index 238f51992d..87a9b80f38 100644 --- a/web/package.json +++ b/web/package.json @@ -32,6 +32,9 @@ "@quasar/extras": "^1.16.12", "@tanstack/vue-table": "^8.19.3", "@tanstack/vue-virtual": "^3.8.4", + "@vue-flow/background": "^1.3.0", + "@vue-flow/controls": "^1.1.2", + "@vue-flow/core": "^1.41.1", "cron-parser": "^4.9.0", "d3-scale": "^4.0.2", "date-fns": "^3.6.0", diff --git a/web/src/assets/images/pipeline/condition.svg b/web/src/assets/images/pipeline/condition.svg index e87fb6ae8f..968e33856e 100644 --- a/web/src/assets/images/pipeline/condition.svg +++ b/web/src/assets/images/pipeline/condition.svg @@ -1,4 +1 @@ - - - - + \ No newline at end of file diff --git a/web/src/assets/images/pipeline/function.svg b/web/src/assets/images/pipeline/function.svg index 8e1d412c94..f8a8725a4a 100644 --- a/web/src/assets/images/pipeline/function.svg +++ b/web/src/assets/images/pipeline/function.svg @@ -1,4 +1 @@ - - - - + \ No newline at end of file diff --git a/web/src/assets/images/pipeline/outputStream.svg b/web/src/assets/images/pipeline/outputStream.svg new file mode 100644 index 0000000000..44facb4116 --- /dev/null +++ b/web/src/assets/images/pipeline/outputStream.svg @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + diff --git a/web/src/assets/images/pipeline/query.svg b/web/src/assets/images/pipeline/query.svg new file mode 100644 index 0000000000..5eaefe5c15 --- /dev/null +++ b/web/src/assets/images/pipeline/query.svg @@ -0,0 +1,37 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/web/src/assets/images/pipeline/stream.svg b/web/src/assets/images/pipeline/stream.svg index be7c2b53d1..71a91c1c8a 100644 --- a/web/src/assets/images/pipeline/stream.svg +++ b/web/src/assets/images/pipeline/stream.svg @@ -1,4 +1 @@ - - - - + \ No newline at end of file diff --git a/web/src/components/alerts/FieldsInput.vue b/web/src/components/alerts/FieldsInput.vue index 4039d624b7..460a4fe9fd 100644 --- a/web/src/components/alerts/FieldsInput.vue +++ b/web/src/components/alerts/FieldsInput.vue @@ -67,6 +67,7 @@ along with this program. If not, see . behavior="menu" :rules="[(val: any) => !!val || 'Field is required!']" style="min-width: 220px" + v-bind="newValueMode" />
. diff --git a/web/src/components/pipeline/AssociateFunction.vue b/web/src/components/pipeline/NodeForm/AssociateFunction.vue similarity index 60% rename from web/src/components/pipeline/AssociateFunction.vue rename to web/src/components/pipeline/NodeForm/AssociateFunction.vue index 5aa0988aa3..47723be753 100644 --- a/web/src/components/pipeline/AssociateFunction.vue +++ b/web/src/components/pipeline/NodeForm/AssociateFunction.vue @@ -24,6 +24,59 @@ along with this program. If not, see . {{ t("pipeline.associateFunction") }}
+
+ + + + +
. " />
-
+
. outlined filled dense + use-input + input-debounce="300" :rules="[(val: any) => !!val || 'Field is required!']" style="min-width: 220px" v-bind:readonly="isUpdating" @@ -74,45 +129,34 @@ along with this program. If not, see . :error-message=" selectedFunction ? 'Function is already associated' : '' " + @filter="filterFunctions" :error="functionExists" /> -
- -
- +
+
+ +
+
+ {{t("alerts.newFunctionAssociationMsg")}} +
+
. /> . type="submit" /> import("../functions/AddFunction.vue") + () => import("@/components/functions/AddFunction.vue"), ); const props = defineProps({ - defaultOrder: { - type: Number, - required: false, - default: 1, - }, - functionData: { - type: Object, - required: false, - default: () => { - return null; - }, - }, - loading: { - type: Boolean, - required: false, - default: false, - }, functions: { type: Array, required: true, @@ -233,21 +270,31 @@ const emit = defineEmits([ const { t } = useI18n(); +const { addNode, pipelineObj , deletePipelineNode, formattedOptions, filteredOptions, filterOptions,getParentNode ,currentSelectedParentNode} = useDragAndDrop(); + const addFunctionRef: any = ref(null); const isUpdating = ref(false); -const selectedFunction = ref(""); +const selectedFunction = ref((pipelineObj.currentSelectedNodeData?.data as { name?: string })?.name || ""); -const functionOrder = ref(props.defaultOrder); +const loading = ref(false); + +const afterFlattening = ref((pipelineObj.currentSelectedNodeData?.data as { after_flatten?: boolean })?.after_flatten || false); const filteredFunctions: Ref = ref([]); const createNewFunction = ref(false); +const q = useQuasar(); const store = useStore(); const functionExists = ref(false); +const selected = ref(null); + +watch(selected, (newValue:any) => { + pipelineObj.userSelectedNode = newValue; +}); const nodeLink = ref({ from: "", @@ -262,34 +309,40 @@ const dialog = ref({ }); watch( - () => props.functions, - (newVal) => { - filteredFunctions.value = [...newVal]; - }, - { - deep: true, - immediate: true, - } -); + () => props.functions, + (newVal) => { + filteredFunctions.value = [...newVal].sort((a:any, b:any) => { + return a.localeCompare(b); + }); + }, + { + deep: true, + immediate: true, + } + ); +onMounted(()=>{ + if(pipelineObj.isEditNode){ + const selectedParentNode = currentSelectedParentNode(); + if(selectedParentNode){ + selected.value = selectedParentNode; -onBeforeMount(() => { - filteredFunctions.value = [...props.functions]; - - if (props.functionData) { - isUpdating.value = true; - selectedFunction.value = props.functionData.name; - functionOrder.value = props.functionData.order; + } } -}); + else{ + pipelineObj.userSelectedNode = {}; + selected.value = null; + } +}) + const openCancelDialog = () => { - if ( - selectedFunction.value === (props.functionData?.name || "") && - functionOrder.value === (props.functionData?.order || 1) - ) { - emit("cancel:hideform"); + if(!isUpdating){ + if(createNewFunction.value == true && addFunctionRef.value.formData.name == "" && addFunctionRef.value.formData.function == "") { + createNewFunction.value = false; return; } + } + dialog.value.show = true; dialog.value.title = "Discard Changes"; @@ -307,8 +360,20 @@ const openDeleteDialog = () => { const saveFunction = () => { functionExists.value = false; - + if (createNewFunction.value) { + if(addFunctionRef.value.formData.name == "" ){ + return; + } + if(addFunctionRef.value.formData.function == ""){ + q.notify({ + message: "Function is required", + color: "negative", + position: "bottom", + timeout: 2000, + }); + return; + } addFunctionRef.value.onSubmit(); return; } @@ -322,10 +387,15 @@ const saveFunction = () => { return; } - emit("update:node", { - data: { name: selectedFunction.value, order: functionOrder.value }, - link: nodeLink.value, - }); + const functionNode = { + name: selectedFunction.value, + after_flatten: afterFlattening.value, + }; + addNode(functionNode); + // emit("update:node", { + // data: { name: selectedFunction.value, order: functionOrder.value }, + // link: nodeLink.value, + // }); emit("cancel:hideform"); }; @@ -346,9 +416,23 @@ const saveUpdatedLink = (link: { from: string; to: string }) => { }; const deleteFunction = () => { - emit("delete:node", { data: props.functionData, type: "function" }); + deletePipelineNode (pipelineObj.currentSelectedNodeID); + emit("cancel:hideform"); }; +const filterFunctions = (val:any, update:any) => { + const filtered = props.functions + .filter((func:any) => + func.toLowerCase().includes(val.toLowerCase()) + ) + .sort((a:any, b:any) => a.localeCompare(b)); + + + + update(() => { + filteredFunctions.value = filtered; + }); + }; diff --git a/web/src/components/pipeline/StreamRouting.vue b/web/src/components/pipeline/NodeForm/Query.vue similarity index 50% rename from web/src/components/pipeline/StreamRouting.vue rename to web/src/components/pipeline/NodeForm/Query.vue index f02d85991f..ea51ceb225 100644 --- a/web/src/components/pipeline/StreamRouting.vue +++ b/web/src/components/pipeline/NodeForm/Query.vue @@ -21,98 +21,26 @@ along with this program. If not, see . :class="store.state.theme === 'dark' ? 'bg-dark' : 'bg-white'" >
- {{ t("pipeline.routing") }} + {{ t("pipeline.query") }}
- -
- -
- -
- - -
- -
- -
-
+ +
. :disableVrlFunction="true" :isValidSqlQuery="isValidSqlQuery" :disableQueryTypeSelection="true" - :showTimezoneWarning="showTimezoneWarning" v-model:trigger="streamRoute.trigger_condition" v-model:sql="streamRoute.query_condition.sql" v-model:query_type="streamRoute.query_condition.type" v-model:aggregation="streamRoute.query_condition.aggregation" v-model:isAggregationEnabled="isAggregationEnabled" - @field:add="addField" - @field:remove="removeField" @validate-sql="validateSqlQuery" class="q-mt-sm" /> - -
-
{{ t("alerts.additionalVariables") }}
- -
+ . type="submit" /> . /> diff --git a/web/src/components/pipeline/ScheduledPipeline.vue b/web/src/components/pipeline/NodeForm/ScheduledPipeline.vue similarity index 98% rename from web/src/components/pipeline/ScheduledPipeline.vue rename to web/src/components/pipeline/NodeForm/ScheduledPipeline.vue index f9966a6e93..fdf22e988e 100644 --- a/web/src/components/pipeline/ScheduledPipeline.vue +++ b/web/src/components/pipeline/NodeForm/ScheduledPipeline.vue @@ -518,7 +518,7 @@ along with this program. If not, see . max-width="300px" > Configure the option to enable a cron job.Configure the option to enable a cron expression. @@ -636,12 +636,13 @@ along with this program. If not, see . v-model="triggerData.cron" dense filled - :label="t('reports.cron') + ' *'" + :label="t('reports.cronExpression') + ' *'" style="background: none; width: 180px" class="showLabelOnTop" stack-label outlined @update:model-value="updateCron" + required /> . data-test="scheduled-alert-period-warning-text" v-else class="text-primary q-pt-xs" - style="font-size: 12px; line-height: 12px" + style="font-size: 12px; line-height: 12px ;padding: 8px 0px;" > - Note: The period should be the same as the cron expression. + Note: The period should be the same as frequency.
@@ -1170,7 +1171,8 @@ const filterFunctionOptions = (val: string, update: any) => { const onBlurQueryEditor = () => { queryEditorPlaceholderFlag.value = true; - emits("validate-sql"); + + // emits("validate-sql"); }; const validateInputs = (notify: boolean = true) => { @@ -1224,6 +1226,7 @@ const validateInputs = (notify: boolean = true) => { return true; }; + defineExpose({ tab, validateInputs, @@ -1256,10 +1259,12 @@ defineExpose({ } .scheduled-alerts { .monaco-editor { - width: 500px !important; - height: 100px !important; - border: 1px solid $border-color; - } + min-width: 500px !important; + min-height: 100px !important; + border: 1px solid $border-color; + resize: both; + overflow: auto; /* Ensure content adjusts when resized */ +} .q-btn { &.icon-dark { diff --git a/web/src/components/pipeline/NodeForm/Stream.vue b/web/src/components/pipeline/NodeForm/Stream.vue new file mode 100644 index 0000000000..aab697c187 --- /dev/null +++ b/web/src/components/pipeline/NodeForm/Stream.vue @@ -0,0 +1,441 @@ + + + + + + diff --git a/web/src/components/pipeline/NodeSidebar.vue b/web/src/components/pipeline/NodeSidebar.vue new file mode 100644 index 0000000000..a4f621befb --- /dev/null +++ b/web/src/components/pipeline/NodeSidebar.vue @@ -0,0 +1,67 @@ + + + + + diff --git a/web/src/components/pipeline/PipelineEditor.vue b/web/src/components/pipeline/PipelineEditor.vue index 093808d9a6..9d2af20367 100644 --- a/web/src/components/pipeline/PipelineEditor.vue +++ b/web/src/components/pipeline/PipelineEditor.vue @@ -31,7 +31,18 @@ along with this program. If not, see . >
-
{{ pipeline.name }}
+
+ {{ pipelineObj.currentSelectedPipeline.name }} +
+
+ +
@@ -69,79 +80,51 @@ along with this program. If not, see . - +
+ +
-
- Note: During execution of pipeline, routes will get executed before - functions -
- - - +
- +
- + + + +
@@ -159,13 +142,14 @@ import { computed, defineAsyncComponent, onBeforeMount, + onMounted, ref, type Ref, } from "vue"; import { getImageURL } from "@/utils/zincutils"; -import StreamRouting from "./StreamRouting.vue"; -import AssociateFunction from "./AssociateFunction.vue"; +import AssociateFunction from "@/components/pipeline/NodeForm/AssociateFunction.vue"; import functionsService from "@/services/jstransform"; + import { useStore } from "vuex"; import pipelineService from "@/services/pipelines"; import { useRouter } from "vue-router"; @@ -173,16 +157,27 @@ import ConfirmDialog from "@/components/ConfirmDialog.vue"; import { useI18n } from "vue-i18n"; import { useQuasar } from "quasar"; import jstransform from "@/services/jstransform"; +import NodeSidebar from "@/components/pipeline/NodeSidebar.vue"; +import useDragAndDrop from "@/plugins/pipelines/useDnD"; +import StreamNode from "@/components/pipeline/NodeForm/Stream.vue"; +import QueryForm from "@/components/pipeline/NodeForm/Query.vue"; +import ConditionForm from "@/components/pipeline/NodeForm/Condition.vue"; +import {MarkerType} from "@vue-flow/core"; + const functionImage = getImageURL("images/pipeline/function.svg"); const streamImage = getImageURL("images/pipeline/stream.svg"); +const streamOutputImage = getImageURL("images/pipeline/outputStream.svg"); const streamRouteImage = getImageURL("images/pipeline/route.svg"); const conditionImage = getImageURL("images/pipeline/condition.svg"); +const queryImage = getImageURL("images/pipeline/query.svg"); + +const PipelineFlow = defineAsyncComponent( + () => import("@/plugins/pipelines/PipelineFlow.vue"), -const ChartRenderer = defineAsyncComponent( - () => import("@/components/dashboards/panels/ChartRenderer.vue") ); + interface Routing { [key: string]: RouteCondition[]; } @@ -201,6 +196,7 @@ interface Function { } interface Pipeline { + pipeline_id: string; name: string; description: string; stream_name: string; @@ -239,7 +235,7 @@ const plotChart: any = ref({ position: "bottom", }, draggable: true, - edgeSymbol: ["circle", "arrow"], + edgeSymbol: [ "arrow"], edgeSymbolSize: [10, 10], edgeLabel: { fontSize: 20, @@ -257,6 +253,7 @@ const plotChart: any = ref({ }); const pipeline = ref({ + pipeline_id: "", name: "", stream_type: "", description: "", @@ -267,7 +264,6 @@ const pipeline = ref({ }); const router = useRouter(); - const store = useStore(); const confirmDialogMeta: any = ref({ @@ -278,17 +274,84 @@ const confirmDialogMeta: any = ref({ onConfirm: () => {}, }); -const nodeTypes = [ - { label: "Function", value: "function" }, - { label: "Route", value: "streamRoute" }, + +const nodeTypes: any = [ + { + label: "Source", + icon: "input", + isSectionHeader: true, + }, + { + label: "Stream", + subtype: "stream", + io_type: "input", + icon: "img:" + streamImage, + tooltip: "Source: Stream Node", + isSectionHeader: false, + }, + { + label: "Query", + subtype: "query", + io_type: "input", + icon: "img:" + queryImage, + tooltip: "Source: Query Node", + isSectionHeader: false, + }, + { + label: "Transform", + icon: "processing", + isSectionHeader: true, + }, + { + label: "Function", + subtype: "function", + io_type: "default", + icon: "img:" + functionImage, + tooltip: "Function Node", + isSectionHeader: false, + }, + { + label: "Condition", + subtype: "condition", + io_type: "default", + icon: "img:" + conditionImage, + tooltip: "Condition Node", + isSectionHeader: false, + }, + { + label: "Destination", + icon: "input", + isSectionHeader: true, + }, + { + label: "Stream", + subtype: "stream", + io_type: "output", + icon: "img:" + streamOutputImage, + tooltip: "Destination: Stream Node", + isSectionHeader: false, + }, ]; +const functions = ref<{ [key: string]: Function }>({}); + + +const { pipelineObj, resetPipelineData } = useDragAndDrop(); +pipelineObj.nodeTypes = nodeTypes; +pipelineObj.functions = functions; const nodes: Ref = ref([]); +const hasInputType = computed(() => { + return pipelineObj.currentSelectedPipeline.nodes.some( + (node : any) => node.io_type === "input", + ); +}); + const nodeLinks = ref<{ [key: string]: NodeLink }>({}); -const functions = ref<{ [key: string]: Function }>({}); - +const refreshFunctionList = () =>{ + getFunctions(); +} const functionOptions = ref([]); const streamRoutes = ref<{ [key: string]: any }>({}); @@ -317,47 +380,60 @@ const dialog = ref({ okCallback: () => {}, }); -const onChartClick = (params: any) => { - const { type, name } = params.data; - - resetDialog(); - - if (type === "streamRoute" || type === "condition") { - if (type === "condition") editingStreamRouteName.value = name.split(":")[0]; - else editingStreamRouteName.value = name; - dialog.value.show = true; - dialog.value.name = "streamRouting"; - } - - if (type === "function") { - getFunctions(); - editingFunctionName.value = name; - dialog.value.show = true; - dialog.value.name = "associateFunction"; - } -}; - onBeforeMount(() => { - getPipeline(); + const route = router.currentRoute.value; + if (route.name == "pipelineEditor") { + getPipeline(); + pipelineObj.isEditPipeline = true; + } else { + pipelineObj.isEditPipeline = false; + resetPipelineData(); - updateGraph(); + } + getFunctions(); }); +onMounted(()=>{ + +}) + + + + const getPipeline = () => { const route = router.currentRoute.value; pipelineService .getPipelines(store.state.selectedOrganization.identifier) .then(async (response) => { - await getFunctions(); - const _pipeline = response.data.list.find( - (pipeline: Pipeline) => - pipeline.name === route.query.name && - pipeline.stream_name === route.query.stream && - pipeline.stream_type === route.query.stream_type + (pipeline: Pipeline) => pipeline.pipeline_id === route.query.id, ); + _pipeline.edges.forEach((edge: any) => { + edge.markerEnd = { + type: MarkerType.ArrowClosed, + width: 20, // Increase arrow width + height: 20, // Increase arrow height + }; + edge.style = { + ...edge.style, // Preserve existing styles + strokeWidth: 2, + }; + edge.type = 'button'; + edge.animated = true; + }); + + + + _pipeline.nodes.forEach((node : any) => { + node.type = node.io_type; + }); + + _pipeline.nodes.forEach((node : any) => { + node.type = node.io_type; + }); + if (!_pipeline) { q.notify({ message: "Pipeline not found", @@ -374,392 +450,15 @@ const getPipeline = () => { return; } - pipeline.value = { - ..._pipeline, - functions: _pipeline?.functions?.list || [], - }; - - setupNodes(); - }); -}; - -const setupNodes = () => { - nodeRows.value = [null, pipeline.value.stream_name, null]; - - // set the base stream node - const node: Node = { - name: pipeline.value.stream_name, - x: 0, - y: 300, - type: "stream", - fixed: true, - }; - - nodes.value.push(node); - - nodeLinks.value[pipeline.value.stream_name] = { - from: [], - to: [], - }; - - // set functions nodes - pipeline.value?.functions - ?.sort((a, b) => a.order - b.order) - .forEach((_function) => { - createFunctionNode( - _function.name, - pipeline.value.stream_name, - _function.order - ); - - functions.value[_function.name] = _function; - - associatedFunctions.value.push(_function.name); - }); - - updateFunctionNodesOrder(); - - if (pipeline.value?.routing) - Object.keys(pipeline.value?.routing).forEach((stream: string) => { - createStreamRouteNode({ - name: stream, - }); - streamRoutes.value[stream] = { - name: stream, - conditions: pipeline.value?.routing[stream], - is_real_time: true, - }; - }); - - (pipeline.value.derived_streams || []).forEach((stream: any) => { - createStreamRouteNode({ - name: stream.name, - }); - streamRoutes.value[stream.name] = { ...stream }; - }); - - updateGraph(); -}; - -const onMouseMove = (params: any) => {}; - -const onMouseOut = (params: any) => {}; - -const addFunction = () => { - getFunctions(); - resetDialog(); - dialog.value.show = true; - dialog.value.name = "associateFunction"; -}; - -const addStream = () => { - resetDialog(); - dialog.value.show = true; - dialog.value.name = "streamRouting"; -}; - -const getFunctionFrom = (stream: string) => { - let fromNodeName = stream; - - const functions = nodes.value.filter( - (node) => node.type === "function" && node.stream === stream - ); - - if (functions.length) { - fromNodeName = functions[functions.length - 1].name; - } - - return fromNodeName; -}; - -// TODO OK : create separate functions for each type node i.e. function, streamRoute, stream -const getNodePosition = (type: string, node?: any) => { - let lastNode = nodes.value.filter((node) => node.type === type).pop(); - - // If this if first function node, adding offset to left from the main stream node - const nodeGap = !lastNode ? 130 : 50; - - if (type === "function" && !lastNode) lastNode = nodes.value[0]; - - if (type === "function") { - if (lastNode?.x?.toString() && lastNode?.y?.toString()) - return { - x: lastNode.x + nodeGap, - y: lastNode.y, - }; - else console.log("Error while getting last node position"); - } - - const isAnyNull = nodeRows.value.every((row) => row !== null); - if (isAnyNull) { - nodeRows.value.push(null); - nodeRows.value.unshift(null); - } - - if (type === "streamRoute") { - const centerIndex = Math.ceil(nodeRows.value.length / 2) - 1; - - for (let i = centerIndex - 1; i >= 0; i--) { - if (nodeRows.value[i] === null) { - const refRow = nodeRows.value[i + 1]; - const refNode = nodes.value.find((node) => node.name === refRow); - - nodeRows.value[i] = node.name; - - if (refNode?.y?.toString()) - return { - x: 50, - y: refNode.y - 50, - }; - else - console.log("Error while getting last node position for streamRoute"); - } - } - - for (let i = centerIndex + 1; i < nodeRows.value.length; i++) { - if (nodeRows.value[i] === null) { - const refRow = nodeRows.value[i - 1]; - const refNode = nodes.value.find((node) => node.name === refRow); - - nodeRows.value[i] = node.name; - - if (refNode?.y?.toString()) - return { - x: 50, - y: refNode.y + 50, - }; - else - console.log("Error while getting last node position for streamRoute"); - } - } - } -}; - -const addFunctionNode = (data: { data: Function }) => { - const nodeName = data.data.name; - - // TODO OK: Optimize this function node creation - // Get all function nodes - // Get the editing node - // Remove function nodes from the nodes state - // Create functions nodes again and sort function nodes by order - - const functionNodes = nodes.value.filter((node) => node.type === "function"); - - if (editingFunctionName.value) { - const editedNode = nodes.value.find( - (node) => node.name === editingFunctionName.value - ); - - if (editedNode) editedNode.order = data.data.order; - - editingFunctionName.value = ""; - } else { - functionNodes.push( - createFunctionNode( - nodeName, - pipeline.value.stream_name, - data.data.order - ) as Node - ); - } - - nodes.value = nodes.value.filter((node) => node.type !== "function"); - - functionNodes - .sort((a, b) => - a.order?.toString() && b.order?.toString() - ? Number(a.order) - Number(b.order) - : 0 - ) - .forEach((node) => { - createFunctionNode( - node.name, - node.stream as string, - node.order as number + pipelineObj.currentSelectedPipeline = _pipeline; + pipelineObj.pipelineWithoutChange = JSON.parse( + JSON.stringify(_pipeline), ); }); - - if (!functions.value[nodeName]) { - functions.value[nodeName] = data.data; - functionOptions.value.push(nodeName); - } - - functions.value[nodeName].order = data.data.order; - - associatedFunctions.value.push(nodeName); - - // reorder function nodes with order key and its links too - updateFunctionNodesOrder(); - - updateGraph(); -}; - -const createFunctionNode = ( - nodeName: string, - streamName: string, - order: number -) => { - const position = getNodePosition("function"); - - if (position?.x === undefined || position?.y === undefined) { - console.log("Error in getting node position"); - return; - } - - const node = { - name: nodeName, - x: position.x, - y: position.y, - type: "function", - fixed: true, - order: order || 1, - stream: streamName, - }; - - nodes.value.push(node); - - nodeLinks.value[nodeName] = { from: [], to: [] }; - - return node; -}; - -const updateFunctionNodesOrder = () => { - const sortedFunctionNodes = nodes.value - .filter((node) => node.type === "function") - .sort((a, b) => - a.order?.toString() && b.order?.toString() - ? Number(a.order) - Number(b.order) - : 0 - ); - - sortedFunctionNodes.forEach((node, index) => { - nodeLinks.value[node.name] = { from: [], to: [] }; - - if (index === 0) { - nodeLinks.value[node.name].from = [nodes.value[0].name]; - nodeLinks.value[nodes.value[0].name].to = [node.name]; - } else { - nodeLinks.value[node.name].from = [sortedFunctionNodes[index - 1].name]; - nodeLinks.value[sortedFunctionNodes[index - 1].name].to = [node.name]; - } - }); -}; - -const addStreamNode = (data: { data: Node }) => { - const nodeName = data.data.name; - - streamRoutes.value[nodeName] = data.data; - - if (editingStreamRouteName.value) { - editingStreamRouteName.value = ""; - } else { - createStreamRouteNode(data.data); - } - - updateGraph(); -}; - -const createStreamRouteNode = (streamRouteData: { name: string }) => { - const position = getNodePosition("streamRoute", streamRouteData); - - const nodeName = streamRouteData.name; - - if (position?.x === undefined || position?.y === undefined) { - console.log("Error in getting node position"); - return; - } - - const condition = nodes.value.push({ - name: nodeName + ":condition", - x: position.x, - y: position.y, - type: "condition", - fixed: true, - }); - - const stream = nodes.value.push({ - name: nodeName, - x: position.x + 50, - y: position.y, - type: "streamRoute", - fixed: true, - }); - - nodeLinks.value[nodeName + ":condition"] = { - from: [nodes.value[0].name], - to: [nodeName], - }; - - nodeLinks.value[nodeName] = { from: [nodeName + ":condition"], to: [] }; - - return { - condition, - stream, - }; -}; - -const symbolMapping: { [key: string]: string } = { - stream: streamImage, - function: functionImage, - streamRoute: streamRouteImage, - condition: conditionImage, -}; - -const getNodeSymbol = (type: string) => { - return ( - "image://" + - window.location.origin + - (symbolMapping[type].startsWith("/") - ? symbolMapping[type] - : "/" + symbolMapping[type]) - ); -}; - -const updateGraph = () => { - const data = nodes.value.map((node) => ({ - name: node.name, - x: node.x, // You may want to adjust these manually if needed (like x: 100 for 'k8s_event') - y: node.y, - symbol: getNodeSymbol(node.type), - fixed: node.fixed, - type: node.type, - label: { - show: true, - position: "bottom", - fontSize: 12, - formatter: (params: any) => { - if (params.data.type === "condition") return "Condition"; - else return params.data.name; - }, - }, - })); - - // Prepare links from 'nodeLinks' - const links: any[] = []; - for (const nodeName in nodeLinks.value) { - const { to, from } = nodeLinks.value[nodeName]; - from.forEach((source) => { - links.push({ - source: source, - target: nodeName, - }); - }); - - to.forEach((target) => { - links.push({ - source: nodeName, - target: target, - }); - }); - } - - plotChart.value.options.series[0].data = data; - plotChart.value.options.series[0].links = links; }; const getFunctions = () => { - if (Object.keys(functions.value).length) return; + // if (Object.keys(functions.value).length) return; isFetchingFunctions.value = true; return functionsService .list( @@ -783,110 +482,87 @@ const getFunctions = () => { }); }; -const deleteNode = (node: { data: Node; type: string }) => { - nodes.value = nodes.value.filter((n) => n.name !== node.data.name); - - delete nodeLinks.value[node.data.name]; - - if (node.type === "function") { - associatedFunctions.value = associatedFunctions.value.filter( - (func) => func !== node.data.name - ); - editingFunctionName.value = ""; - } - - if (node.type === "streamRoute") { - delete streamRoutes.value[node.data.name]; - editingStreamRouteName.value = ""; - } - - nodeRows.value.forEach((row, index) => { - if (row === node.data.name) nodeRows.value[index] = null; - }); - - updateFunctionNodesOrder(); - - updateGraph(); -}; - const resetDialog = () => { - dialog.value.show = false; - dialog.value.name = ""; + + pipelineObj.dialog.show = false; + pipelineObj.dialog.name = ""; editingFunctionName.value = ""; editingStreamRouteName.value = ""; }; -const getPipelinePayload = () => { - const payload = { - name: pipeline.value.name, - description: pipeline.value.description, - stream_name: pipeline.value.stream_name, - stream_type: pipeline.value.stream_type, - routing: {} as Routing, - derived_streams: [] as any[], - }; - - Object.values(streamRoutes.value).forEach((route: any) => { - if (!route.is_real_time) { - payload.derived_streams.push(route); - return; - } else { - payload.routing[route.name] = route.conditions; - } - }); - - return payload; -}; - -const associateFunctions = async () => { - const associateFunctionPromises = []; - - const previousAssociatedFunctions = pipeline.value.functions.map( - (func) => func.name - ); - - for (let i = 0; i < associatedFunctions.value.length; i++) { - const functionIndex = previousAssociatedFunctions.indexOf( - associatedFunctions.value[i] - ); - - // If function is already associated with the pipeline and order is same, skip. - // If there is no change in associated function name and order, skip. - if (functionIndex > -1) { - if ( - pipeline.value.functions[i].name === associatedFunctions.value[i] && - pipeline.value.functions[i].order === - functions.value[associatedFunctions.value[i]].order - ) - continue; - } - - associateFunctionPromises.push( - jstransform.apply_stream_function( - store.state.selectedOrganization.identifier, - pipeline.value.stream_name, - pipeline.value.stream_type, - associatedFunctions.value[i], - { - order: Number(functions.value[associatedFunctions.value[i]].order), - } - ) - ); - } - return Promise.all(associateFunctionPromises); -}; - -const deleteFunctionAssociation = (name: string) => { - return jstransform.remove_stream_function( - store.state.selectedOrganization.identifier, - pipeline.value.stream_name, - pipeline.value.stream_type, - name - ); -}; - const savePipeline = async () => { - const payload = getPipelinePayload(); + if (pipelineObj.currentSelectedPipeline.name === "") { + q.notify({ + message: "Pipeline name is required", + color: "negative", + position: "bottom", + timeout: 3000, + }); + return; + } + + // Find the input node + const inputNodeIndex = pipelineObj.currentSelectedPipeline.nodes.findIndex( + (node:any) => + node?.io_type === "input" && + (node.data?.node_type === "stream" || node.data?.node_type === "query") + ); + + const outputNodeIndex = pipelineObj.currentSelectedPipeline.nodes.findIndex( + (node:any) => + node?.io_type === "output" + ); + + if (inputNodeIndex === -1) { + q.notify({ + message: "Source node is required", + color: "negative", + position: "bottom", + timeout: 3000, + }); + return; + } + + + + else if(outputNodeIndex === -1){ + q.notify({ + message: "Destination node is required", + color: "negative", + position: "bottom", + timeout: 3000, + }); + return; + } + + else { + pipelineObj.currentSelectedPipeline.nodes.map((node : any) => { + if (node.data.node_type === "stream" && node.data.stream_name && node.data.stream_name.hasOwnProperty("value")) { + node.data.stream_name = node.data.stream_name.value; + } + }); + const nodes = pipelineObj.currentSelectedPipeline.nodes as any[]; + + const inputNode : any = nodes.splice(inputNodeIndex, 1)[0]; + nodes.unshift(inputNode); + if (inputNode.data.node_type === "stream") { + pipelineObj.currentSelectedPipeline.source.source_type = "realtime"; + } else { + pipelineObj.currentSelectedPipeline.source.source_type = "scheduled"; + } + } + + pipelineObj.currentSelectedPipeline.org = + store.state.selectedOrganization.identifier; + if(findMissingEdges()){ + q.notify({ + message: "Please connect all nodes before saving", + color: "negative", + position: "bottom", + timeout: 3000, + }); + return; + } const dismiss = q.notify({ message: "Saving pipeline...", @@ -894,46 +570,68 @@ const savePipeline = async () => { spinner: true, }); - if (pipeline.value.functions.length) { - const deleteFunctionPromises = []; - for (let i = 0; i < pipeline.value.functions.length; i++) { - if (!associatedFunctions.value.includes(pipeline.value.functions[i].name)) - deleteFunctionPromises.push( - deleteFunctionAssociation(pipeline.value.functions[i].name) - ); - } - await Promise.all(deleteFunctionPromises); - } + const saveOperation = pipelineObj.isEditPipeline + ? pipelineService.updatePipeline({ + data: pipelineObj.currentSelectedPipeline, + org_identifier: store.state.selectedOrganization.identifier, + }) + : pipelineService.createPipeline({ + data: pipelineObj.currentSelectedPipeline, + org_identifier: store.state.selectedOrganization.identifier, + }); + - await associateFunctions(); - - pipelineService - .updatePipeline({ - data: payload, - org_identifier: store.state.selectedOrganization.identifier, - }) + saveOperation .then(() => { + if(pipelineObj.isEditPipeline){ + pipelineObj.isEditPipeline = false; + } q.notify({ message: "Pipeline saved successfully", color: "positive", position: "bottom", timeout: 3000, }); + router.push({ + name: "pipelines", + query: { + org_identifier: store.state.selectedOrganization.identifier, + }, + }); }) .catch((error) => { - q.notify({ - message: error.response?.data?.message || "Error while saving pipeline", + if(pipelineObj.isEditPipeline){ + pipelineObj.isEditPipeline = true; + } + + + + if(error.response?.data?.message === "Invalid Pipeline: empty edges list"){ + q.notify({ + message: "Please connect all nodes", + color: "negative", + position: "bottom", + timeout: 3000, + }); + } + else{ + q.notify({ + message: + error.response?.data?.message || "Error while saving pipeline", color: "negative", position: "bottom", timeout: 3000, }); + } }) .finally(() => { dismiss(); }); }; + + const openCancelDialog = () => { confirmDialogMeta.value.show = true; confirmDialogMeta.value.title = t("common.cancelChanges"); @@ -949,6 +647,35 @@ const resetConfirmDialog = () => { confirmDialogMeta.value.data = null; }; +const findMissingEdges = () => { + const nodes = pipelineObj.currentSelectedPipeline.nodes; + const edges = pipelineObj.currentSelectedPipeline.edges; + + // Collect node IDs that are part of edges (either source or target) + const outgoingConnections = new Set(edges.map((edge:any) => edge.source)); + const incomingConnections = new Set(edges.map((edge:any) => edge.target)); + + // Find nodes that are not connected properly + const unconnectedNodes = nodes.filter((node:any) => { + if (node.type === 'default') { + // Check for both incoming and outgoing edges + return !incomingConnections.has(node.id) || !outgoingConnections.has(node.id); + } else { + // Check for at least one connection (incoming or outgoing) + return !incomingConnections.has(node.id) && !outgoingConnections.has(node.id); + } + }); + + if (unconnectedNodes.length > 0) { + console.log(unconnectedNodes, "unconnectedNodes"); + return true; // There are unconnected nodes + } + + return false; // All nodes are properly connected +}; + + + // Drag n Drop methods const onNodeDragStart = (event: any, data: any) => { @@ -958,14 +685,6 @@ const onNodeDragStart = (event: any, data: any) => { const onNodeDrop = (event: any) => { event.preventDefault(); const nodeType = event.dataTransfer.getData("text"); - - if (nodeType === "function") { - addFunction(); - } - - if (nodeType === "streamRoute") { - addStream(); - } }; const onNodeDragOver = (event: any) => { @@ -990,6 +709,7 @@ const updateNewFunction = (_function: Function) => { } .pipeline-chart-container { +height: 80vh; border-radius: 12px; width: calc(100% - 200px); } @@ -1011,4 +731,34 @@ const updateNewFunction = (_function: Function) => { .stream-routing-dialog-container { min-width: 540px !important; } + +.o2vf_node { + width: auto; + + .vue-flow__node { + padding: 0px; + width: auto; + } + + .o2vf_node_input, + .vue-flow__node-input { + background-color: #c8d6f5; + border-color: 1px solid #2c6b2f; + color: black; + } + + .o2vf_node_output, + .vue-flow__node-output { + background-color: #8fd4b8; + border-color: 1px solid #3b6f3f; + color: black; + } + + .o2vf_node_default, + .vue-flow__node-default { + background-color: #efefef; + border-color: 1px solid #171e25; + color: black; + } +} diff --git a/web/src/components/pipeline/PipelineView.vue b/web/src/components/pipeline/PipelineView.vue new file mode 100644 index 0000000000..82190df3a3 --- /dev/null +++ b/web/src/components/pipeline/PipelineView.vue @@ -0,0 +1,198 @@ + + + + + + + + \ No newline at end of file diff --git a/web/src/components/pipeline/PipelinesList.vue b/web/src/components/pipeline/PipelinesList.vue index 6ea0ad5695..83e4c4e8e9 100644 --- a/web/src/components/pipeline/PipelinesList.vue +++ b/web/src/components/pipeline/PipelinesList.vue @@ -16,11 +16,13 @@ along with this program. If not, see .