From 7df19cb00a023b38591826c5f191ed2ebc703146 Mon Sep 17 00:00:00 2001 From: ByteBaker <42913098+ByteBaker@users.noreply.github.com> Date: Mon, 2 Feb 2026 20:01:34 +0530 Subject: [PATCH] fix: incident graphs and service graphs improvements (#10318) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### **User description** ## Changes ### 1. Reduce alert graph edge explosion - Modified temporal edge logic to prevent O(n²) growth - Changed same-service edges to only connect to most recent previous alert - Changed cross-service temporal edges to connect to chronologically closest alert - **Impact**: 88% reduction in edges (16 instead of 136 for 17 alerts) ### 2. Improve correlation error handling - Added fallback mechanism when service registry is empty - Better error messages for correlation failures - Enhanced incident detail view with clearer feedback **Files modified:** - `src/service/alerts/incidents.rs` - Incident correlation handling ___ ### **PR Type** Bug fix, Enhancement ___ ### **Description** - Prevent alert topology edge explosion - Add frontend fallback incident correlation - Improve correlation tab empty/error UX - Share time/stream state across tabs ___ ### Diagram Walkthrough ```mermaid flowchart LR A["Incident alerts ingested"] B["Topology enrichment"] C["Same-service temporal chain"] D["Cross-service edges"] E["ServiceGraph tab shared state"] F["Incident detail correlation"] G["FQN-priority drift detection"] H["Frontend fallback correlation"] A -- "build graph" --> B B -- "temporal edges" --> C B -- "dependency edges" --> D E -- "share datetime/stream" --> B F -- "fetch correlated streams" --> G G -- "if not drift" --> H ```

File Walkthrough
Relevant files
Bug fix
convertTraceData.ts
Simplify service-graph tree request metrics                           
web/src/utils/traces/convertTraceData.ts
  • Use incoming edge metrics for non-root
  • Root sums outgoing edge request counts
  • Default root error rate to zero
  • Keep connection count as in+out
 +5/-6     
incidents.rs
Limit temporal topology edges to nearest alerts                   
src/service/alerts/incidents.rs
  • Same-service temporal edge targets most recent prior
  • Keep full ServiceDependency edges when SG edge exists
  • For non-dependent services, link to closest later alert
  • Avoid duplicate edges with type-specific checks
 +80/-47 
Enhancement
IncidentDetailDrawer.vue
Add fallback correlation and improve tab states                   
web/src/components/alerts/IncidentDetailDrawer.vue
  • Show refresh header only when streams exist
  • Unify error/no-data states with clearer messaging
  • Detect FQN priority changes post-incident creation
  • Build frontend fallback correlation from schema mapping
 +217/-72
ServiceGraph.vue
Share Service Graph time/stream with traces state               
web/src/plugins/traces/ServiceGraph.vue
  • Bind datetime picker to searchObj.data.datetime
  • Use shared searchObj.data.stream.* for stream lists
  • Remove local stream/time state and duplicate stream loading
  • Simplify empty-state guidance and code styling
 +50/-181
___ --------- Co-authored-by: sai nikhil kethe --- src/service/alerts/destinations.rs | 18 +- src/service/alerts/incidents.rs | 124 +++++--- src/service/promql/search/cache/mod.rs | 18 +- .../alerts/IncidentDetailDrawer.vue | 289 +++++++++++++----- .../alerts/IncidentServiceGraph.spec.ts | 13 +- .../alerts/IncidentServiceGraph.vue | 219 ++++++++++--- .../useMetricsCorrelationDashboard.spec.ts | 33 +- .../useMetricsCorrelationDashboard.ts | 55 +--- web/src/plugins/traces/ServiceGraph.vue | 231 +++----------- web/src/utils/traces/convertTraceData.ts | 23 +- 10 files changed, 596 insertions(+), 427 deletions(-) diff --git a/src/service/alerts/destinations.rs b/src/service/alerts/destinations.rs index 98d4405171..c8476f5dca 100644 --- a/src/service/alerts/destinations.rs +++ b/src/service/alerts/destinations.rs @@ -481,19 +481,17 @@ mod tests { template: template1, .. } = dest1_retrieved.module - { - if let Module::Alert { + && let Module::Alert { template: template2, .. } = dest2_retrieved.module - { - assert_eq!(template1, Some("prebuilt_slack".to_string())); - assert_eq!(template2, Some("prebuilt_slack".to_string())); - assert_eq!( - template1, template2, - "Both destinations should use the same template" - ); - } + { + assert_eq!(template1, Some("prebuilt_slack".to_string())); + assert_eq!(template2, Some("prebuilt_slack".to_string())); + assert_eq!( + template1, template2, + "Both destinations should use the same template" + ); } // Verify only ONE template exists diff --git a/src/service/alerts/incidents.rs b/src/service/alerts/incidents.rs index 2ebd26e288..146712d2c5 100644 --- a/src/service/alerts/incidents.rs +++ b/src/service/alerts/incidents.rs @@ -723,32 +723,37 @@ pub async fn enrich_with_topology( if is_new_node { let current_node_first_fired = topology.nodes[current_node_index].first_fired_at; - for (idx, node) in topology.nodes.iter().enumerate() { - if idx == current_node_index { - continue; // Skip self - } + // Find the most recent previous node from same service + let most_recent_previous = topology + .nodes + .iter() + .enumerate() + .filter(|(idx, node)| { + *idx != current_node_index + && node.service_name == service_name + && node.first_fired_at < current_node_first_fired + }) + .max_by_key(|(_, node)| node.first_fired_at); - // Only connect nodes from same service - if node.service_name == service_name && node.first_fired_at < current_node_first_fired { - // Check if edge already exists - let edge_exists = topology.edges.iter().any(|e| { - e.from_node_index == idx - && e.to_node_index == current_node_index - && matches!(e.edge_type, EdgeType::Temporal) + if let Some((prev_idx, _)) = most_recent_previous { + // Check if edge already exists + let edge_exists = topology.edges.iter().any(|e| { + e.from_node_index == prev_idx + && e.to_node_index == current_node_index + && matches!(e.edge_type, EdgeType::Temporal) + }); + + if !edge_exists { + topology.edges.push(AlertEdge { + from_node_index: prev_idx, + to_node_index: current_node_index, + edge_type: EdgeType::Temporal, }); - - if !edge_exists { - topology.edges.push(AlertEdge { - from_node_index: idx, - to_node_index: current_node_index, - edge_type: EdgeType::Temporal, - }); - log::debug!( - "[incidents] Added temporal edge: {} -> {} (same service)", - idx, - current_node_index - ); - } + log::debug!( + "[incidents] Added temporal edge: {} -> {} (most recent from same service)", + prev_idx, + current_node_index + ); } } } @@ -796,42 +801,71 @@ pub async fn enrich_with_topology( .iter() .any(|e| &e.from == from_service && &e.to == to_service); - // Add edge from each node in from_service to each node in to_service - // that occurred chronologically after - for &from_idx in from_indices { - for &to_idx in to_indices { + if has_sg_edge { + // Service dependency: connect ALL nodes (full causality) + for &from_idx in from_indices { + for &to_idx in to_indices { + let from_node = &topology.nodes[from_idx]; + let to_node = &topology.nodes[to_idx]; + + // Only add if from happened before to (chronological) + if from_node.first_fired_at < to_node.first_fired_at { + // Check if edge already exists + let edge_exists = topology.edges.iter().any(|e| { + e.from_node_index == from_idx + && e.to_node_index == to_idx + && matches!(e.edge_type, EdgeType::ServiceDependency) + }); + + if !edge_exists { + topology.edges.push(AlertEdge { + from_node_index: from_idx, + to_node_index: to_idx, + edge_type: EdgeType::ServiceDependency, + }); + log::debug!( + "[incidents] Added ServiceDependency edge: {from_idx} ({from_service}) -> {to_idx} ({to_service})" + ); + } + } + } + } + } else { + // Temporal only: connect to most recent node in target service + // For each node in from_service, find the most recent + // chronologically-after node in to_service + for &from_idx in from_indices { let from_node = &topology.nodes[from_idx]; - let to_node = &topology.nodes[to_idx]; - // Only add if from happened before to (chronological) - if from_node.first_fired_at < to_node.first_fired_at { - // Determine edge type: ServiceDependency if SG edge exists, otherwise - // Temporal - let edge_type = if has_sg_edge { - EdgeType::ServiceDependency - } else { - EdgeType::Temporal - }; + // Find most recent node in to_service that fired after from_node + let most_recent_to = to_indices + .iter() + .filter_map(|&to_idx| { + let to_node = &topology.nodes[to_idx]; + if to_node.first_fired_at > from_node.first_fired_at { + Some((to_idx, to_node.first_fired_at)) + } else { + None + } + }) + .min_by_key(|(_, fired_at)| *fired_at); // Closest in time (earliest after from_node) + if let Some((to_idx, _)) = most_recent_to { // Check if edge already exists let edge_exists = topology.edges.iter().any(|e| { e.from_node_index == from_idx && e.to_node_index == to_idx - && matches!( - (&e.edge_type, &edge_type), - (EdgeType::ServiceDependency, EdgeType::ServiceDependency) - | (EdgeType::Temporal, EdgeType::Temporal) - ) + && matches!(e.edge_type, EdgeType::Temporal) }); if !edge_exists { topology.edges.push(AlertEdge { from_node_index: from_idx, to_node_index: to_idx, - edge_type, + edge_type: EdgeType::Temporal, }); log::debug!( - "[incidents] Added {edge_type:?} edge: {from_idx} ({from_service}) -> {to_idx} ({to_service})" + "[incidents] Added Temporal edge: {from_idx} ({from_service}) -> {to_idx} ({to_service}, most recent)" ); } } diff --git a/src/service/promql/search/cache/mod.rs b/src/service/promql/search/cache/mod.rs index 8a596e46a2..14c2f78062 100644 --- a/src/service/promql/search/cache/mod.rs +++ b/src/service/promql/search/cache/mod.rs @@ -681,21 +681,9 @@ mod tests { assert!(cap1 >= len1); // Values may change due to concurrent tests, but should not vary wildly - let len_diff = if len2 > len1 { - len2 - len1 - } else { - len1 - len2 - }; - let cap_diff = if cap2 > cap1 { - cap2 - cap1 - } else { - cap1 - cap2 - }; - let mem_diff = if mem2 > mem1 { - mem2 - mem1 - } else { - mem1 - mem2 - }; + let len_diff = len2.abs_diff(len1); + let cap_diff = cap2.abs_diff(cap1); + let mem_diff = mem2.abs_diff(mem1); assert!(len_diff < 1000 || len1 == 0 || len2 == 0); assert!(cap_diff < 10000 || cap1 == 0 || cap2 == 0); assert!(mem_diff < 1000000 || mem1 == 0 || mem2 == 0); diff --git a/web/src/components/alerts/IncidentDetailDrawer.vue b/web/src/components/alerts/IncidentDetailDrawer.vue index 348eef9d99..978fe2e4ae 100644 --- a/web/src/components/alerts/IncidentDetailDrawer.vue +++ b/web/src/components/alerts/IncidentDetailDrawer.vue @@ -1010,8 +1010,8 @@ along with this program. If not, see .
- -
+ +
Showing correlated logs from incident timeframe .
Loading correlated logs...
- -
- -
Failed to load correlated logs
-
{{ correlationError }}
+ +
+ +
+ {{ correlationError || 'No correlated logs found' }} +
+
+ The service discovery configuration (FQN priority dimensions) was changed after this incident was created. +
- -
- -
No correlated logs found
-
- Try searching manually using these dimensions: -
-
-
- {{ key }}: - {{ value }} - -
-
-
-
.
- -
+ +
Showing correlated metrics from incident timeframe .
Loading correlated metrics...
- -
- -
Failed to load correlated metrics
-
{{ correlationError }}
+ +
+ +
+ {{ correlationError || 'No correlated metrics found' }} +
+
+ The service discovery configuration (FQN priority dimensions) was changed after this incident was created. +
- -
- -
No correlated metrics found
-
- Try searching manually using these dimensions: -
-
-
- {{ key }}: - {{ value }} - -
-
-
-
.
- -
+ +
Showing correlated traces from incident timeframe .
Loading correlated traces...
- -
- -
Failed to load correlated traces
-
{{ correlationError }}
+ +
+ +
+ {{ correlationError || 'No correlated traces found' }} +
+
+ The service discovery configuration (FQN priority dimensions) was changed after this incident was created. +
- -
- -
No correlated traces found
-
- Try searching manually using these dimensions: -
-
-
- {{ key }}: - {{ value }} - -
-
-
-
incidentDetails.value.created_at) { + correlationError.value = + "FQN priority was modified since this incident was created, cannot correlate"; + console.warn( + `[Incident Correlation] FQN config updated at ${fqnConfig.updated_at}, incident created at ${incidentDetails.value.created_at}` + ); + } else { + // Not a config issue - try building fallback correlation + await buildFallbackCorrelation(org, incidentDetails.value); + } + } catch (configError) { + // If can't fetch config, try fallback anyway + console.error("Failed to fetch FQN config:", configError); + await buildFallbackCorrelation(org, incidentDetails.value); + } + } } catch (error: any) { console.error("Failed to load correlated streams:", error); correlationError.value = @@ -1529,6 +1544,136 @@ export default defineComponent({ } }; + // Build fallback correlation using first alert's stream schema + const buildFallbackCorrelation = async (org: string, incident: Incident) => { + try { + console.log("[Fallback Correlation] Building local correlation from first alert's stream"); + + // Get first alert to determine source stream + const firstAlert = alerts.value?.[0]; + if (!firstAlert) { + console.warn("[Fallback Correlation] No alerts found in incident"); + return; + } + + // Use actual stream type and name from the alert + const streamType = firstAlert.stream_type || "logs"; + const streamName = firstAlert.stream_name || "default"; + + // Get stream schema + const schemaResponse = await streamService.schema(org, streamName, streamType); + const schema = schemaResponse.data; + + // Get semantic groups + const semanticGroupsResponse = await serviceStreamsApi.getSemanticGroups(org); + const semanticGroups = semanticGroupsResponse.data; + + // Map incident dimensions to field names in schema + // Schema response has .schema array with {name, type} objects + const schemaFields = new Set( + (schema.schema || schema.fields || []).map((f: any) => f.name) + ); + + console.log("[Fallback Correlation] Incident dimensions:", incident.stable_dimensions); + console.log("[Fallback Correlation] Total schema fields:", schemaFields.size); + console.log("[Fallback Correlation] Sample schema fields:", Array.from(schemaFields).slice(0, 10)); + + const filters: Record = {}; + + for (const [dimId, dimValue] of Object.entries(incident.stable_dimensions)) { + // Find semantic group + const group = semanticGroups.find((g: any) => g.id === dimId); + + console.log(`[Fallback Correlation] Processing ${dimId} = ${dimValue}`); + + if (!group) { + console.warn(`[Fallback Correlation] No semantic group found for: ${dimId}`); + continue; + } + + if (!group.fields || group.fields.length === 0) { + console.warn(`[Fallback Correlation] Semantic group ${dimId} has empty fields array`); + continue; + } + + console.log(`[Fallback Correlation] Trying field variants for ${dimId}:`, group.fields.slice(0, 5)); + + // Collect ALL matching field names and pick the best one + const matchingFields = []; + for (const fieldName of group.fields) { + if (schemaFields.has(fieldName)) { + matchingFields.push(fieldName); + } + } + + if (matchingFields.length > 0) { + // Prefer shorter field names without prefixes + const bestField = matchingFields.sort((a, b) => { + // Penalize long prefixes + const aPenalty = a.startsWith('service_') ? 1000 : + a.startsWith('resource_') ? 500 : + a.startsWith('attributes_') ? 300 : 0; + const bPenalty = b.startsWith('service_') ? 1000 : + b.startsWith('resource_') ? 500 : + b.startsWith('attributes_') ? 300 : 0; + + // Then by length (shorter = better) + return (a.length + aPenalty) - (b.length + bPenalty); + })[0]; + + filters[bestField] = dimValue; + console.log(`[Fallback Correlation] ✅ Mapped ${dimId} → ${bestField} = ${dimValue} (from ${matchingFields.length} options: ${matchingFields.slice(0, 3).join(', ')})`); + } else { + console.warn(`[Fallback Correlation] ❌ No matching field in schema for ${dimId}`); + } + } + + if (Object.keys(filters).length === 0) { + console.warn("[Fallback Correlation] No dimensions could be mapped to stream fields"); + return; + } + + console.log("[Fallback Correlation] Mapped filters:", filters); + console.log("[Fallback Correlation] Schema fields:", Array.from(schemaFields)); + console.log("[Fallback Correlation] Stream type:", streamType); + + // Build StreamInfo object + const streamInfo = { + stream_name: streamName, + stream_type: streamType === 'logs' ? 'Logs' : streamType === 'metrics' ? 'Metrics' : 'Traces', + filters + }; + + // Build correlation response with only the source stream type + correlationData.value = { + serviceName: `dimension-match-${incident.stable_dimensions.service || 'unknown'}`, + matchedDimensions: incident.stable_dimensions, + additionalDimensions: {}, + logStreams: streamType === 'logs' ? [streamInfo] : [], + metricStreams: streamType === 'metrics' ? [streamInfo] : [], + traceStreams: streamType === 'traces' ? [streamInfo] : [], + correlationData: { + service_name: `dimension-match-${incident.stable_dimensions.service || 'unknown'}`, + matched_dimensions: incident.stable_dimensions, + additional_dimensions: {}, + related_streams: { + logs: streamType === 'logs' ? [streamInfo] : [], + metrics: streamType === 'metrics' ? [streamInfo] : [], + traces: streamType === 'traces' ? [streamInfo] : [], + }, + correlation_method: "frontend-fallback" + } + }; + + console.log("[Fallback Correlation] Built correlation data:", correlationData.value); + console.log("[Fallback Correlation] Log streams:", correlationData.value.logStreams); + console.log("[Fallback Correlation] Filters being passed:", correlationData.value.logStreams[0]?.filters); + } catch (fallbackError) { + console.error("[Fallback Correlation] Failed to build fallback:", fallbackError); + // Don't set error - let tabs show "No correlated X found" + } + }; + // Refresh correlation data const refreshCorrelation = () => { fetchCorrelatedStreams(true); diff --git a/web/src/components/alerts/IncidentServiceGraph.spec.ts b/web/src/components/alerts/IncidentServiceGraph.spec.ts index 024552eb52..6a70b6d12f 100644 --- a/web/src/components/alerts/IncidentServiceGraph.spec.ts +++ b/web/src/components/alerts/IncidentServiceGraph.spec.ts @@ -432,9 +432,10 @@ describe("IncidentServiceGraph.vue", () => { const node2 = chartData.options.series[0].data[1]; // 7 alerts const node3 = chartData.options.series[0].data[2]; // 12 alerts - expect(node1.symbolSize).toBe(45); // 30 + 3*5 - expect(node2.symbolSize).toBe(65); // 30 + 7*5 - expect(node3.symbolSize).toBe(90); // 30 + 12*5 + // All nodes have fixed size of 60 + expect(node1.symbolSize).toBe(60); + expect(node2.symbolSize).toBe(60); + expect(node3.symbolSize).toBe(60); }); it("should cap node size at 100", async () => { @@ -463,7 +464,7 @@ describe("IncidentServiceGraph.vue", () => { const chartData = wrapper.vm.chartData; const largeNode = chartData.options.series[0].data[0]; - expect(largeNode.symbolSize).toBe(100); // Capped at 100 + expect(largeNode.symbolSize).toBe(60); // Fixed size for all nodes }); it("should add border to primary service nodes", async () => { @@ -839,7 +840,7 @@ describe("IncidentServiceGraph.vue", () => { const chartData = wrapper.vm.chartData; // Second node has 0 alerts const zeroAlertNode = chartData.options.series[0].data[1]; - expect(zeroAlertNode.symbolSize).toBe(30); // Base size + expect(zeroAlertNode.symbolSize).toBe(60); // Fixed size for all nodes }); it("should handle very large alert counts", async () => { @@ -867,7 +868,7 @@ describe("IncidentServiceGraph.vue", () => { const chartData = wrapper.vm.chartData; const largeNode = chartData.options.series[0].data[0]; - expect(largeNode.symbolSize).toBe(100); // Capped at 100 + expect(largeNode.symbolSize).toBe(60); // Fixed size for all nodes }); it("should handle null orgId gracefully", () => { diff --git a/web/src/components/alerts/IncidentServiceGraph.vue b/web/src/components/alerts/IncidentServiceGraph.vue index 78e9e8692d..c5601f75b6 100644 --- a/web/src/components/alerts/IncidentServiceGraph.vue +++ b/web/src/components/alerts/IncidentServiceGraph.vue @@ -15,7 +15,43 @@ along with this program. If not, see . -->