mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-06 00:09:22 +00:00
fix: compute alert_count from actual triggers to prevent inconsistencies
The denormalized alert_count field can become out of sync due to race conditions in concurrent updates. Now both list and detail endpoints compute the count from actual triggers (source of truth) and log mismatches for debugging. - Add get_alert_counts() for efficient batch counting - Self-healing: overrides stored count with actual count - Affects both list and detail endpoints
This commit is contained in:
@@ -275,6 +275,46 @@ pub async fn get_incident_alerts(
|
||||
.map_err(|e| Error::DbError(DbError::SeaORMError(e.to_string())))
|
||||
}
|
||||
|
||||
/// Get actual alert counts for multiple incidents (source of truth)
|
||||
///
|
||||
/// Returns a HashMap of incident_id -> actual_count from junction table.
|
||||
/// Use this to fix denormalized alert_count fields that may be out of sync.
|
||||
pub async fn get_alert_counts(
|
||||
incident_ids: &[String],
|
||||
) -> Result<std::collections::HashMap<String, i32>, errors::Error> {
|
||||
use sea_orm::FromQueryResult;
|
||||
|
||||
if incident_ids.is_empty() {
|
||||
return Ok(std::collections::HashMap::new());
|
||||
}
|
||||
|
||||
let client = ORM_CLIENT.get_or_init(connect_to_orm).await;
|
||||
|
||||
#[derive(Debug, FromQueryResult)]
|
||||
struct CountResult {
|
||||
incident_id: String,
|
||||
count: i64,
|
||||
}
|
||||
|
||||
let results = alert_incident_alerts::Entity::find()
|
||||
.filter(
|
||||
alert_incident_alerts::Column::IncidentId
|
||||
.is_in(incident_ids.iter().map(|s| s.as_str())),
|
||||
)
|
||||
.group_by(alert_incident_alerts::Column::IncidentId)
|
||||
.column_as(alert_incident_alerts::Column::IncidentId.count(), "count")
|
||||
.column(alert_incident_alerts::Column::IncidentId)
|
||||
.into_model::<CountResult>()
|
||||
.all(client)
|
||||
.await
|
||||
.map_err(|e| Error::DbError(DbError::SeaORMError(e.to_string())))?;
|
||||
|
||||
Ok(results
|
||||
.into_iter()
|
||||
.map(|r| (r.incident_id, r.count as i32))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Count open incidents for an org
|
||||
pub async fn count_open(org_id: &str) -> Result<u64, errors::Error> {
|
||||
let client = ORM_CLIENT.get_or_init(connect_to_orm).await;
|
||||
|
||||
@@ -524,7 +524,20 @@ pub async fn get_incident_with_alerts(
|
||||
let incident_org_id = incident.org_id.clone();
|
||||
|
||||
// Convert to config types
|
||||
let incident_data = model_to_incident(incident).await?;
|
||||
let mut incident_data = model_to_incident(incident).await?;
|
||||
|
||||
// Fix alert_count from actual triggers (source of truth)
|
||||
// This heals any inconsistencies from race conditions or historical bugs
|
||||
let actual_count = incident_alerts.len() as i32;
|
||||
if incident_data.alert_count != actual_count {
|
||||
log::warn!(
|
||||
"[incidents] Incident {} alert_count mismatch: stored={}, actual={}. Using actual count.",
|
||||
incident_id,
|
||||
incident_data.alert_count,
|
||||
actual_count
|
||||
);
|
||||
incident_data.alert_count = actual_count;
|
||||
}
|
||||
|
||||
// Convert incident_alerts to triggers
|
||||
let triggers: Vec<IncidentAlert> = incident_alerts
|
||||
@@ -591,11 +604,29 @@ pub async fn list_incidents(
|
||||
|
||||
let incidents = infra::table::alert_incidents::list(org_id, status, limit, offset).await?;
|
||||
|
||||
let incidents_list = incidents
|
||||
// Get actual alert counts from junction table (source of truth)
|
||||
let incident_ids: Vec<String> = incidents.iter().map(|i| i.id.clone()).collect();
|
||||
let actual_counts = infra::table::alert_incidents::get_alert_counts(&incident_ids).await?;
|
||||
|
||||
let incidents_list: Vec<Incident> = incidents
|
||||
.into_iter()
|
||||
.map(|i| {
|
||||
// topology_context is not required while listing all incidents
|
||||
model_to_incident_with_topology(i, None)
|
||||
let mut incident = model_to_incident_with_topology(i.clone(), None);
|
||||
|
||||
// Fix alert_count from actual triggers (self-healing for corrupted data)
|
||||
if let Some(&actual_count) = actual_counts.get(&i.id)
|
||||
&& incident.alert_count != actual_count
|
||||
{
|
||||
log::debug!(
|
||||
"[incidents] List: Incident {} alert_count mismatch: stored={}, actual={}",
|
||||
i.id,
|
||||
incident.alert_count,
|
||||
actual_count
|
||||
);
|
||||
incident.alert_count = actual_count;
|
||||
}
|
||||
|
||||
incident
|
||||
})
|
||||
.collect();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user