From 928053ec7cf17aabac8eaea42e96cb19af0cbccf Mon Sep 17 00:00:00 2001
From: Subhra264 <chakrabortysubhradeep556@gmail.com>
Date: Wed, 28 Aug 2024 13:37:57 +0530
Subject: [PATCH] fix: when updating alert first check, if the scheduled_job
 exist or not (#4359)

Fixes #4360

In case some `scheduled_job` records (required for alerts to get
scheduled properly) are missing for some alerts, they can be recreated
by simply disabling and enabling it.

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit


- **Improvements**
- Enhanced error logging for trigger updates and creations, providing
clearer context with the inclusion of the `schedule_key`.
- Improved robustness of trigger management logic, facilitating easier
diagnosis of issues related to alert triggers.
- Increased flexibility in alert handling by dynamically setting the
`is_realtime` property based on the alert's characteristics.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
---
 src/service/alerts/scheduler.rs |  2 +-
 src/service/db/alerts/alert.rs  | 24 +++++++++++++++++++-----
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/src/service/alerts/scheduler.rs b/src/service/alerts/scheduler.rs
index ec4c594435..51ef714a73 100644
--- a/src/service/alerts/scheduler.rs
+++ b/src/service/alerts/scheduler.rs
@@ -127,7 +127,7 @@ async fn handle_alert_triggers(trigger: db::scheduler::Trigger) -> Result<(), an
 
     let mut new_trigger = db::scheduler::Trigger {
         next_run_at: Utc::now().timestamp_micros(),
-        is_realtime: false,
+        is_realtime: alert.is_real_time,
         is_silenced: false,
         status: db::scheduler::TriggerStatus::Waiting,
         retries: 0,
diff --git a/src/service/db/alerts/alert.rs b/src/service/db/alerts/alert.rs
index d137d34e8e..6cfbdb7183 100644
--- a/src/service/db/alerts/alert.rs
+++ b/src/service/db/alerts/alert.rs
@@ -61,7 +61,7 @@ pub async fn set(
         Ok(_) => {
             let trigger = db::scheduler::Trigger {
                 org: org_id.to_string(),
-                module_key: schedule_key,
+                module_key: schedule_key.clone(),
                 next_run_at: chrono::Utc::now().timestamp_micros(),
                 is_realtime: alert.is_real_time,
                 is_silenced: false,
@@ -71,21 +71,35 @@ pub async fn set(
                 match db::scheduler::push(trigger).await {
                     Ok(_) => Ok(()),
                     Err(e) => {
-                        log::error!("Failed to save trigger: {}", e);
+                        log::error!("Failed to save trigger for alert {schedule_key}: {}", e);
+                        Ok(())
+                    }
+                }
+            } else if db::scheduler::exists(
+                org_id,
+                db::scheduler::TriggerModule::Alert,
+                &schedule_key,
+            )
+            .await
+            {
+                match db::scheduler::update_trigger(trigger).await {
+                    Ok(_) => Ok(()),
+                    Err(e) => {
+                        log::error!("Failed to update trigger for alert {schedule_key}: {}", e);
                         Ok(())
                     }
                 }
             } else {
-                match db::scheduler::update_trigger(trigger).await {
+                match db::scheduler::push(trigger).await {
                     Ok(_) => Ok(()),
                     Err(e) => {
-                        log::error!("Failed to update trigger: {}", e);
+                        log::error!("Failed to save trigger for alert {schedule_key}: {}", e);
                         Ok(())
                     }
                 }
             }
         }
-        Err(e) => Err(anyhow::anyhow!("Error save alert: {}", e)),
+        Err(e) => Err(anyhow::anyhow!("Error save alert {schedule_key}: {}", e)),
     }
 }