diff --git a/src/api/management/src/request/notifications.rs b/src/api/management/src/request/notifications.rs index 1fdcb9ced4..c51915b87d 100644 --- a/src/api/management/src/request/notifications.rs +++ b/src/api/management/src/request/notifications.rs @@ -173,9 +173,15 @@ fn validate(query: &DeliveriesQuery) -> Result<(NotificationKind, String), Strin ) })?; // Only the kinds whose producers write rows today; the others land with their stages. - if !matches!(kind, NotificationKind::Anomaly | NotificationKind::Incident) { + if !matches!( + kind, + NotificationKind::Anomaly + | NotificationKind::Incident + | NotificationKind::Synthetics + | NotificationKind::Downtime + ) { return Err(format!( - "kind: '{}' has no delivery rows yet; allowed: anomaly, incident", + "kind: '{}' has no delivery rows yet; allowed: anomaly, incident, synthetics, downtime", query.kind )); } @@ -200,6 +206,8 @@ fn owner_of_episode(episode_id: &str) -> String { async fn can_read(org_id: &str, user_id: &str, kind: NotificationKind, owner: &str) -> bool { let object_type = match kind { NotificationKind::Incident => "incidents", + NotificationKind::Synthetics => "synthetics", + NotificationKind::Downtime => "downtimes", _ => "alerts", }; openobserve_core::auth::check_permissions( @@ -250,9 +258,15 @@ mod tests { #[test] fn kinds_without_rows_yet_are_refused() { - for kind in ["alert", "synthetics", "downtime", "on_call"] { + for kind in ["alert", "on_call"] { let err = validate(&q(kind, Some("x1"), None)).unwrap_err(); - assert!(err.contains("allowed: anomaly, incident"), "{kind}: {err}"); + assert!( + err.contains("allowed: anomaly, incident, synthetics, downtime"), + "{kind}: {err}" + ); + } + for kind in ["synthetics", "downtime"] { + assert!(validate(&q(kind, Some("x1"), None)).is_ok(), "{kind}"); } } diff --git a/src/api/management/src/request/synthetics/mod.rs b/src/api/management/src/request/synthetics/mod.rs index 562e672678..d6273f7a2b 100644 --- a/src/api/management/src/request/synthetics/mod.rs +++ b/src/api/management/src/request/synthetics/mod.rs @@ -713,6 +713,28 @@ async fn composition_error_response( }) } +/// A template override must name a user template that applies to synthetics; "" clears it. +async fn checked_template( + org_id: &str, + body: &mut config::meta::synthetics::Synthetic, +) -> Result<(), Response> { + body.template = body + .template + .take() + .map(|t| t.trim().to_string()) + .filter(|t| !t.is_empty()); + if let Some(name) = body.template.as_deref() { + openobserve_core::alerts::templates::check_binding( + org_id, + name, + config::meta::alerts::notification_kind::NotificationKind::Synthetics, + ) + .await + .map_err(|e| MetaHttpResponse::bad_request(e.as_str()))?; + } + Ok(()) +} + #[utoipa::path( post, path = "/{org_id}/synthetics", @@ -775,6 +797,9 @@ pub async fn create_synthetic( } } + if let Err(response) = checked_template(&org_id, &mut body).await { + return response; + } let created_by = user_email.user_id.as_str(); match openobserve_synthetics::service::create_synthetic(&org_id, body, created_by).await { Ok(check) => MetaHttpResponse::json(check), @@ -945,6 +970,10 @@ pub async fn update_synthetic( #[cfg(not(feature = "enterprise"))] let user_id = ""; + let mut body = body; + if let Err(response) = checked_template(&org_id, &mut body).await { + return response; + } match openobserve_synthetics::service::update_synthetic(&org_id, &id, body).await { Ok(check) => MetaHttpResponse::json(check), Err(e) => { @@ -1718,7 +1747,7 @@ async fn process_ack( let should_notify = !matches!(resp.alert, AlertDecision::Silent) && resp.suppressed_by.is_none(); // Enterprise-gated because alert *destinations* are: the dispatch it ends in - // (`alerts::alert::dispatch_notification`) is not built in OSS. An OSS build + // (the shared notification path) is not built in OSS. An OSS build // runs the check, records the run and serves the result — it just has // nowhere to send a notification, exactly as it has for alerts. #[cfg(feature = "enterprise")] @@ -1744,6 +1773,8 @@ async fn process_ack( failing_locations: resp.failing_locations.clone(), failing_environments: resp.failing_environments.clone(), passing_locations: resp.passing_locations.clone(), + template: resp.template.clone(), + alerting_since: resp.alerting_since, }; tokio::spawn(async move { openobserve_core::synthetics::notify_check_result(notification).await; @@ -2448,6 +2479,8 @@ mod tests { failing_environments: Vec::new(), usage_events, suppressed_by: None, + alerting_since: 0, + template: None, } } diff --git a/src/config/src/config.rs b/src/config/src/config.rs index b3d32a7092..d8ea4d6864 100644 --- a/src/config/src/config.rs +++ b/src/config/src/config.rs @@ -136,7 +136,8 @@ pub type RwBTreeMap = tokio::sync::RwLock>; // 136: add applies_to to templates. // 137: rename user templates under the reserved o2_default_ prefix. // 138: add template to anomaly_detection_config. -pub const DB_SCHEMA_VERSION: u64 = 138; +// 139: add template and alerting_since to synthetics. +pub const DB_SCHEMA_VERSION: u64 = 139; pub const DB_SCHEMA_KEY: &str = "/db_schema_version/"; // global version variables diff --git a/src/config/src/meta/synthetics.rs b/src/config/src/meta/synthetics.rs index d0df50ff5a..15f02e8ba0 100644 --- a/src/config/src/meta/synthetics.rs +++ b/src/config/src/meta/synthetics.rs @@ -140,6 +140,9 @@ pub struct Synthetic { /// Alert destination names to notify on check failure. #[serde(default)] pub destinations: Vec, + /// Template override for this check's messages; null or "" uses the destination's. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub template: Option, /// Number of retries before marking a check failed (0 = no retry). #[serde(default)] pub retries: i32, diff --git a/src/core/src/alerts/alert.rs b/src/core/src/alerts/alert.rs index 12fb29ffea..f837f18979 100644 --- a/src/core/src/alerts/alert.rs +++ b/src/core/src/alerts/alert.rs @@ -3418,42 +3418,6 @@ async fn send_to_destination_type( } } -/// Send a pre-built message string to a single destination type. -/// -/// Used by synthetics and downtime notifications, which build their own -/// payload rather than going through the alert template system. -#[cfg(feature = "enterprise")] -pub(crate) async fn dispatch_notification( - org_id: &str, - dest_type: &DestinationType, - subject: &str, - msg: String, -) -> Result { - match dest_type { - DestinationType::Http(endpoint) => send_http_notification(endpoint, msg).await, - // Callers build one payload with no HTML/plaintext split, so both - // parts carry the same string — byte-identical to the MIME this path - // emitted before the signature took two bodies. - DestinationType::Email(email) => { - let (email, skipped) = org_member_email(org_id, email).await?; - send_email_notification(subject, &email, Some(msg.clone()), msg) - .await - .map(|resp| note_skipped(resp, &skipped)) - } - DestinationType::Sns(aws_sns) => send_sns_notification(subject, aws_sns, msg).await, - DestinationType::SlackApp(target) => { - crate::alerts::slack_publish::send_post( - &crate::alerts::slack_publish::QueueSink, - org_id, - target, - subject, - &msg, - ) - .await - } - } -} - /// Dispatch a rendered test-send message to one destination. /// /// `title` is the already-`[TEST] `-marked title `build_test_message` stamped @@ -3464,14 +3428,6 @@ pub(crate) async fn dispatch_notification( /// of re-deriving it from `rendered`, or hardcoding a literal) keeps this /// function agnostic to the rendered payload's shape and guarantees the SNS /// subject is never anything other than the actual marked title. -/// -/// Unlike [`dispatch_notification`], this is NOT enterprise-gated: test-send -/// (Task 15) is an OSS-visible feature, and it needs the same private -/// `send_http_notification` / `send_email_notification` / `send_sns_notification` -/// transports this module already owns. Kept as a distinct, narrowly-scoped -/// function rather than removing the `#[cfg]` from `dispatch_notification` -/// itself, so the enterprise-only synthetics and downtime call sites are -/// untouched. pub(crate) async fn dispatch_test_message( org_id: &str, dest_type: &DestinationType, diff --git a/src/core/src/alerts/notifications/context.rs b/src/core/src/alerts/notifications/context.rs index e97f7d38de..ad8ad5fc4a 100644 --- a/src/core/src/alerts/notifications/context.rs +++ b/src/core/src/alerts/notifications/context.rs @@ -107,6 +107,16 @@ pub struct NotificationContext { pub cluster_name: String, /// Today's body for a generic webhook while no user template applies (D1). pub legacy_payload: Option, + pub render_hints: RenderHints, +} + +/// Producer choices the renderer honors; the defaults keep the alert output unchanged. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RenderHints { + /// The email or SNS subject when a Custom template has no title of its own. + pub fallback_subject: Option, + /// HTML-escape context attributes in email bodies, as the producer's own sender did. + pub escape_email_attributes: bool, } /// Builds a producer's pre-template JSON body only when the legacy rule selects it. diff --git a/src/core/src/alerts/notifications/custom.rs b/src/core/src/alerts/notifications/custom.rs index 0ea38a70a9..8f72f0e0c9 100644 --- a/src/core/src/alerts/notifications/custom.rs +++ b/src/core/src/alerts/notifications/custom.rs @@ -232,7 +232,12 @@ pub(crate) fn apply_custom_template_plain( } } for (key, value) in ctx.context_attributes.iter() { - process_variable_replace(&mut resp, key, &VarValue::Str(value), is_email); + if is_email && ctx.render_hints.escape_email_attributes { + let escaped = html_escape(value); + process_variable_replace(&mut resp, key, &VarValue::Str(&escaped), is_email); + } else { + process_variable_replace(&mut resp, key, &VarValue::Str(value), is_email); + } } // Substitute endpoint metadata variables (e.g., credential_assignmentGroup, @@ -494,6 +499,15 @@ impl VarValue<'_> { } } +fn html_escape(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + #[cfg(test)] mod golden { //! Byte-identity corpus for the custom-kind renderer. diff --git a/src/core/src/alerts/notifications/default_template.rs b/src/core/src/alerts/notifications/default_template.rs index d606c906a7..3fc127bd06 100644 --- a/src/core/src/alerts/notifications/default_template.rs +++ b/src/core/src/alerts/notifications/default_template.rs @@ -81,6 +81,7 @@ pub fn compiled_content_for(kind: NotificationKind) -> ContentSpec { match kind { NotificationKind::Incident => compiled_incident_content(), NotificationKind::Anomaly => compiled_anomaly_content(), + NotificationKind::Synthetics => compiled_synthetics_content(), _ => compiled_default_content(), } } @@ -88,7 +89,9 @@ pub fn compiled_content_for(kind: NotificationKind) -> ContentSpec { /// The reserved name that `compiled_content_for(kind)` is seeded under. pub fn compiled_name_for(kind: NotificationKind) -> &'static str { match kind { - NotificationKind::Incident | NotificationKind::Anomaly => kind.compiled_default_name(), + NotificationKind::Incident | NotificationKind::Anomaly | NotificationKind::Synthetics => { + kind.compiled_default_name() + } _ => DEFAULT_CONTENT_TEMPLATE_NAME, } } @@ -115,6 +118,29 @@ pub fn compiled_anomaly_content() -> ContentSpec { } } +/// The title is today's subject line, so a mail or SNS subject reads exactly as before. +pub fn compiled_synthetics_content() -> ContentSpec { + ContentSpec { + title: "{check_title}".into(), + title_overrides: Default::default(), + body: "{alert_description}".into(), + fields: vec![ + field("Check", "{check_name} ({check_type})"), + field("Target", "{check_target}"), + field("Locations", "{check_locations_line}"), + field("Error", "{check_error}"), + ], + rows: RowsSpec { + enabled: false, + max: 5, + columns: None, + format: None, + }, + links: vec![], + chart: ChartSpec { enabled: false }, + } +} + pub fn compiled_incident_content() -> ContentSpec { ContentSpec { title: "Incident {incident_severity}: {incident_title}".into(), @@ -165,6 +191,7 @@ pub(crate) async fn ensure_default_content_template( NotificationKind::Alert, NotificationKind::Incident, NotificationKind::Anomaly, + NotificationKind::Synthetics, ] { ensure_compiled_row( compiled_name_for(kind), diff --git a/src/core/src/alerts/notifications/delivery/tests/flow.rs b/src/core/src/alerts/notifications/delivery/tests/flow.rs index e17520804c..61e965023b 100644 --- a/src/core/src/alerts/notifications/delivery/tests/flow.rs +++ b/src/core/src/alerts/notifications/delivery/tests/flow.rs @@ -915,3 +915,142 @@ fn an_email_with_an_inline_chart_replays_and_filters_recipients_again() { }, ); } + +#[cfg(feature = "enterprise")] +async fn kind_rows( + kind: &str, + object: &str, +) -> Vec { + let filter = ListFilter { + org: "acme".to_string(), + kind: kind.into(), + object_id: Some(object.to_string()), + size: 50, + ..Default::default() + }; + ledger::list(infra::db::get_orm_client_rw().await, &filter) + .await + .unwrap() + .1 +} + +#[cfg(feature = "enterprise")] +fn synthetics_close( + check: &str, + since: i64, + send_resolves: bool, + won: bool, +) -> openobserve_synthetics::alerting::EpisodeClose { + openobserve_synthetics::alerting::EpisodeClose { + org_id: "acme".into(), + check_id: check.to_string(), + check_name: "checkout".into(), + check_type: "http".into(), + target: "https://shop.acme.com".into(), + episode_id: format!("{check}:{since}"), + run_id: "r2".into(), + destinations: vec!["pd-syn".into(), "slack-syn".into()], + template: None, + send_resolves, + swap: Box::new(move |_| Box::pin(async move { Ok(won) })), + } +} + +#[cfg(feature = "enterprise")] +#[test] +fn a_synthetics_recovery_resolves_the_firing_record_and_a_muted_one_writes_no_row() { + run( + concat!( + module_path!(), + "::a_synthetics_recovery_resolves_the_firing_record_and_a_muted_one_writes_no_row" + ), + &[], + || async { + store().await; + let (base, seen) = mock().await; + register("pd-syn", http(format!("{base}/pd"), "pagerduty")); + register("slack-syn", http(format!("{base}/ok"), "slack")); + let check = svix_ksuid::Ksuid::new(None, None).to_string(); + crate::synthetics::notify_check_result(crate::synthetics::CheckNotification { + org_id: "acme".into(), + check_name: "checkout".into(), + check_id: check.clone(), + check_type: "http".into(), + target: "https://shop.acme.com".into(), + destinations: vec!["pd-syn".into(), "slack-syn".into()], + run_id: "r1".into(), + status: "failed".into(), + job_count: 1, + error: Some("timeout".into()), + checked_at: 1, + recovery: false, + consecutive_failures: 1, + flaky: false, + status_reason: None, + error_source: String::new(), + degraded: false, + failing_locations: vec!["us-east-1".into()], + passing_locations: vec![], + failing_environments: vec![], + template: None, + alerting_since: 100, + }) + .await; + let triggers = calls(&seen, "/pd"); + assert_eq!(triggers.len(), 1, "the firing reached PagerDuty once"); + + // A lost swap closes nothing and writes no row. + assert!( + !crate::synthetics::close_alert_episode(synthetics_close(&check, 100, true, false)) + .await + .unwrap() + ); + assert!( + kind_rows("synthetics", &check) + .await + .iter() + .all(|r| r.event != "resolved") + ); + + assert!( + crate::synthetics::close_alert_episode(synthetics_close(&check, 100, true, true)) + .await + .unwrap() + ); + let pd = calls(&seen, "/pd"); + assert_eq!(pd.len(), 2, "one trigger, then one resolve"); + let resolve = String::from_utf8_lossy(&pd[1].1).to_string(); + let trigger = String::from_utf8_lossy(&pd[0].1).to_string(); + assert!(resolve.contains("resolve"), "{resolve}"); + assert!( + trigger.contains(&format!("{check}:100")) + && resolve.contains(&format!("{check}:100")) + ); + let resolved: Vec<_> = kind_rows("synthetics", &check) + .await + .into_iter() + .filter(|r| r.event == "resolved") + .collect(); + assert_eq!(resolved.len(), 2); + assert!(resolved.iter().all(|r| r.status == "sent"), "{resolved:?}"); + + // A muted recovery closes its episode without a resolve row. + assert!( + crate::synthetics::close_alert_episode(synthetics_close(&check, 200, false, true)) + .await + .unwrap() + ); + assert!( + kind_rows("synthetics", &check) + .await + .iter() + .all(|r| r.episode_id != format!("{check}:200")) + ); + assert_eq!( + calls(&seen, "/pd").len(), + 2, + "nothing more reached PagerDuty" + ); + }, + ); +} diff --git a/src/core/src/alerts/notifications/examples.rs b/src/core/src/alerts/notifications/examples.rs index 96fb2b08b5..3afa853fcc 100644 --- a/src/core/src/alerts/notifications/examples.rs +++ b/src/core/src/alerts/notifications/examples.rs @@ -47,6 +47,36 @@ pub const INCIDENT_VARIABLES: [&str; 6] = [ "incident_url", ]; +pub const SYNTHETICS_VARIABLES: [&str; 16] = [ + "check_id", + "check_name", + "check_type", + "check_target", + "check_status", + "check_status_label", + "check_status_reason", + "check_locations_line", + "check_location_count", + "check_consecutive_failures", + "check_flaky", + "check_degraded", + "check_run_id", + "check_url", + "check_error", + "check_title", +]; + +pub const DOWNTIME_VARIABLES: [&str; 8] = [ + "downtime_name", + "downtime_event", + "downtime_reason", + "downtime_starts_at", + "downtime_ends_at", + "downtime_targets", + "downtime_counts", + "downtime_url", +]; + const COMMON_VARIABLES: [&str; 27] = [ "org_name", "stream_type", @@ -82,6 +112,8 @@ pub fn variables_for(kind: NotificationKind) -> Vec { let kind_vars: &[&str] = match kind { NotificationKind::Anomaly => &ANOMALY_VARIABLES, NotificationKind::Incident => &INCIDENT_VARIABLES, + NotificationKind::Synthetics => &SYNTHETICS_VARIABLES, + NotificationKind::Downtime => &DOWNTIME_VARIABLES, _ => &[], }; COMMON_VARIABLES @@ -100,10 +132,12 @@ pub struct VariableGroup { } /// The kinds whose producer sends through templates in this release; the UI hides the others. -pub const SHIPPED_KINDS: [NotificationKind; 3] = [ +pub const SHIPPED_KINDS: [NotificationKind; 5] = [ NotificationKind::Alert, NotificationKind::Incident, NotificationKind::Anomaly, + NotificationKind::Synthetics, + NotificationKind::Downtime, ]; /// `common`, then one group per shipped kind with its own variables. @@ -122,6 +156,8 @@ pub fn variable_groups() -> Vec { let vars: &[&str] = match kind { NotificationKind::Anomaly => &ANOMALY_VARIABLES, NotificationKind::Incident => &INCIDENT_VARIABLES, + NotificationKind::Synthetics => &SYNTHETICS_VARIABLES, + NotificationKind::Downtime => &DOWNTIME_VARIABLES, _ => &[], }; groups.push(group(kind.as_str(), vars)); @@ -134,6 +170,7 @@ pub fn example_context(kind: NotificationKind, resolved: bool) -> NotificationCo match kind { NotificationKind::Anomaly => anomaly_example(resolved), NotificationKind::Incident => incident_example(resolved), + NotificationKind::Synthetics => synthetics_example(resolved), _ => { let level = if resolved { AlertLevel::Ok @@ -233,6 +270,69 @@ fn anomaly_example(resolved: bool) -> NotificationContext { ctx } +fn synthetics_example(resolved: bool) -> NotificationContext { + let url = "https://o2.acme.com/web/synthetic/chk_9d/results?org_identifier=acme"; + let level = if resolved { + AlertLevel::Ok + } else { + AlertLevel::Critical + }; + let mut ctx = base(NotificationKind::Synthetics, resolved, level); + ctx.stream_type = "synthetics".into(); + ctx.stream_name = "https://shop.acme.com/api/health".into(); + ctx.alert_name = "checkout API".into(); + ctx.alert_url = url.into(); + ctx.episode_id = Some("chk_9d:1762077900000000".into()); + ctx.event = if resolved { + STATUS_RESOLVED.into() + } else { + "firing:run_42".into() + }; + ctx.alert_trigger_time_str = "2026-11-02 10:05:00".into(); + let (title, description, label, locations, error) = if resolved { + ( + "[OpenObserve Synthetics] ✅ checkout API has RECOVERED", + "checkout API has recovered", + "recovered", + "us-east-1, eu-west-1 recovered", + "", + ) + } else { + ( + "[OpenObserve Synthetics] 🔴 checkout API is FAILED", + "checkout API is failing\nconnection timed out after 10s", + "failed", + "1 of 2 failed: us-east-1", + "connection timed out after 10s", + ) + }; + ctx.alert_description = description.into(); + ctx.context_attributes = [ + ("check_id", "chk_9d"), + ("check_name", "checkout API"), + ("check_type", "http"), + ("check_target", "https://shop.acme.com/api/health"), + ("check_status", if resolved { "passed" } else { "failed" }), + ("check_status_label", label), + ("check_status_reason", ""), + ("check_locations_line", locations), + ("check_location_count", "2"), + ( + "check_consecutive_failures", + if resolved { "0" } else { "3" }, + ), + ("check_flaky", "false"), + ("check_degraded", "false"), + ("check_run_id", "run_42"), + ("check_url", url), + ("check_error", error), + ("check_title", title), + ] + .map(|(k, v)| (k.to_string(), v.to_string())) + .to_vec(); + ctx +} + fn incident_example(resolved: bool) -> NotificationContext { let url = "https://o2.acme.com/web/alerts/incidents/inc_7f3a?org_identifier=acme"; let level = if resolved { @@ -304,6 +404,7 @@ mod tests { const EXAMPLE_ANOMALY: &str = include_str!("render/testdata/examples/anomaly.json"); const EXAMPLE_INCIDENT: &str = include_str!("render/testdata/examples/incident.json"); + const EXAMPLE_SYNTHETICS: &str = include_str!("render/testdata/examples/synthetics.json"); fn rendered( kind: NotificationKind, @@ -349,6 +450,19 @@ mod tests { ); } + #[test] + fn golden_example_synthetics_keeps_the_subject_line_as_its_title() { + let want: serde_json::Value = serde_json::from_str(EXAMPLE_SYNTHETICS).unwrap(); + assert_eq!(example(NotificationKind::Synthetics), want); + let title = &want["slack_firing"]["attachments"][0]["blocks"][0]["text"]["text"]; + assert!( + title + .as_str() + .unwrap() + .contains("[OpenObserve Synthetics] 🔴 checkout API is FAILED") + ); + } + #[test] fn anomaly_recovery_hides_empty_fields() { let ctx = example_context(NotificationKind::Anomaly, true); @@ -382,6 +496,7 @@ mod tests { for (kind, file) in [ (NotificationKind::Anomaly, "anomaly.json"), (NotificationKind::Incident, "incident.json"), + (NotificationKind::Synthetics, "synthetics.json"), ] { let body = serde_json::to_string_pretty(&example(kind)).unwrap() + "\n"; std::fs::write(format!("{dir}/{file}"), body).unwrap(); diff --git a/src/core/src/alerts/notifications/org_default.rs b/src/core/src/alerts/notifications/org_default.rs index 6939342959..31ffc4de5d 100644 --- a/src/core/src/alerts/notifications/org_default.rs +++ b/src/core/src/alerts/notifications/org_default.rs @@ -92,6 +92,20 @@ pub async fn set_org_default_template( Ok(()) } +/// A producer's level 1 override by name; a missing one falls through to the next level. +pub async fn load_override(org_id: &str, name: Option<&str>) -> Vec