mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-04 00:08:55 +00:00
fix: alert graph edges not rendering due to timestamp equality (#10347)
### **User description** ## Problem Fixes openobserve/o2-enterprise#1229 Alert graphs were not rendering edges (arrows) when nodes had identical timestamps, causing the topology graph to show nodes without connections. ## Root Cause The edge creation logic used strict timestamp inequality (`<`, `>`), which failed when alerts fired at the exact same microsecond. This commonly occurs with: - High-frequency alerting - Bulk alert ingestion - Simulated/test data with fixed timestamps - Clock resolution limitations ## Solution Added timestamp tiebreaker logic using node index (insertion order) for all three edge creation scenarios: 1. **Same-service temporal edges**: Connect to most recent previous node, use index as tiebreaker 2. **Cross-service temporal edges**: Connect to chronologically closest node, use index as tiebreaker 3. **Service dependency edges**: Connect related services, use index as tiebreaker ## Changes - `src/service/alerts/incidents.rs`: Updated 3 edge creation locations with timestamp + index comparisons ## Impact - ✅ Edges now created even when alerts have identical timestamps - ✅ Deterministic behavior based on insertion order - ✅ Graph visualization shows proper alert flow with arrows - ✅ Root cause analysis now possible ## Testing - Verified code compiles successfully - Manual testing recommended with alerts at same timestamp ___ ### **PR Type** Bug fix ___ ### **Description** - Add tiebreaker for equal timestamps - Ensure deterministic topology edge creation - Fix missing incident graph arrows ___ ### Diagram Walkthrough ```mermaid flowchart LR A["New alert node added"] --> B["Find candidate prior/next nodes"] B --> C["Compare (first_fired_at, node_index)"] C --> D["Create temporal/dependency edges deterministically"] ``` <details> <summary><h3> File Walkthrough</h3></summary> <table><thead><tr><th></th><th align="left">Relevant files</th></tr></thead><tbody><tr><td><strong>Bug fix</strong></td><td><table> <tr> <td> <details> <summary><strong>incidents.rs</strong><dd><code>Deterministic edge creation for equal timestamps</code> </dd></summary> <hr> src/service/alerts/incidents.rs <ul><li>Add index-based tiebreaker for same-service edges<br> <li> Allow dependency edges when timestamps equal<br> <li> Refine cross-service temporal matching with index<br> <li> Update ordering keys to include node index</ul> </details> </td> <td><a href="https://github.com/openobserve/openobserve/pull/10347/files#diff-2411d921fd8b8783c049deea11da06726ed986550d405477d486234892353ca9">+17/-5</a> </td> </tr> </table></td></tr></tr></tbody></table> </details> ___
This commit is contained in:
@@ -729,6 +729,7 @@ pub async fn enrich_with_topology(
|
||||
let current_node_first_fired = topology.nodes[current_node_index].first_fired_at;
|
||||
|
||||
// Find the most recent previous node from same service
|
||||
// Use node index as tiebreaker when timestamps are equal (handles simultaneous alerts)
|
||||
let most_recent_previous = topology
|
||||
.nodes
|
||||
.iter()
|
||||
@@ -736,9 +737,11 @@ pub async fn enrich_with_topology(
|
||||
.filter(|(idx, node)| {
|
||||
*idx != current_node_index
|
||||
&& node.service_name == service_name
|
||||
&& node.first_fired_at < current_node_first_fired
|
||||
&& (node.first_fired_at < current_node_first_fired
|
||||
|| (node.first_fired_at == current_node_first_fired
|
||||
&& *idx < current_node_index))
|
||||
})
|
||||
.max_by_key(|(_, node)| node.first_fired_at);
|
||||
.max_by_key(|(idx, node)| (node.first_fired_at, *idx));
|
||||
|
||||
if let Some((prev_idx, _)) = most_recent_previous {
|
||||
// Check if edge already exists
|
||||
@@ -814,7 +817,11 @@ pub async fn enrich_with_topology(
|
||||
let to_node = &topology.nodes[to_idx];
|
||||
|
||||
// Only add if from happened before to (chronological)
|
||||
if from_node.first_fired_at < to_node.first_fired_at {
|
||||
// Use node index as tiebreaker when timestamps are equal (handles simultaneous
|
||||
// alerts)
|
||||
if from_node.first_fired_at < to_node.first_fired_at
|
||||
|| (from_node.first_fired_at == to_node.first_fired_at && from_idx < to_idx)
|
||||
{
|
||||
// Check if edge already exists
|
||||
let edge_exists = topology.edges.iter().any(|e| {
|
||||
e.from_node_index == from_idx
|
||||
@@ -843,17 +850,22 @@ pub async fn enrich_with_topology(
|
||||
let from_node = &topology.nodes[from_idx];
|
||||
|
||||
// Find most recent node in to_service that fired after from_node
|
||||
// Use node index as tiebreaker when timestamps are equal (handles simultaneous
|
||||
// alerts)
|
||||
let most_recent_to = to_indices
|
||||
.iter()
|
||||
.filter_map(|&to_idx| {
|
||||
let to_node = &topology.nodes[to_idx];
|
||||
if to_node.first_fired_at > from_node.first_fired_at {
|
||||
if to_node.first_fired_at > from_node.first_fired_at
|
||||
|| (to_node.first_fired_at == from_node.first_fired_at
|
||||
&& to_idx > from_idx)
|
||||
{
|
||||
Some((to_idx, to_node.first_fired_at))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.min_by_key(|(_, fired_at)| *fired_at); // Closest in time (earliest after from_node)
|
||||
.min_by_key(|(to_idx, fired_at)| (*fired_at, *to_idx)); // Closest in time, then by index
|
||||
|
||||
if let Some((to_idx, _)) = most_recent_to {
|
||||
// Check if edge already exists
|
||||
|
||||
@@ -88,6 +88,7 @@
|
||||
]"
|
||||
dense
|
||||
no-caps
|
||||
class="view-toggle-buttons"
|
||||
@update:model-value="setVisualizationType"
|
||||
/>
|
||||
|
||||
@@ -144,58 +145,9 @@
|
||||
<div class="text-h6 q-mt-md text-grey-7">
|
||||
No Service Graph Data
|
||||
</div>
|
||||
<div
|
||||
class="text-h6 text-grey-7 q-mt-lg q-mb-md"
|
||||
style="font-size: 1.1rem"
|
||||
>
|
||||
Possible causes:
|
||||
<div class="text-body2 text-grey-6 q-mt-sm">
|
||||
Try querying a longer duration
|
||||
</div>
|
||||
<div
|
||||
class="text-body1 text-grey-6"
|
||||
style="max-width: 800px; font-size: 0.95rem"
|
||||
>
|
||||
<div
|
||||
class="q-pa-md q-mb-md"
|
||||
style="
|
||||
background: rgba(var(--q-primary-rgb), 0.05);
|
||||
border-radius: 6px;
|
||||
"
|
||||
>
|
||||
<div
|
||||
class="text-weight-medium q-mb-sm"
|
||||
style="font-size: 1rem"
|
||||
>
|
||||
Queried time range has no traces, try selecting a longer duration.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
class="q-pa-md q-mb-md"
|
||||
style="
|
||||
background: rgba(var(--q-primary-rgb), 0.05);
|
||||
border-radius: 6px;
|
||||
"
|
||||
>
|
||||
<div
|
||||
class="text-weight-medium q-mb-sm"
|
||||
style="font-size: 1rem"
|
||||
>
|
||||
Maybe your traces have only internal spans (<code>span_kind = 1</code>)?
|
||||
</div>
|
||||
<div class="q-mt-sm">
|
||||
Make sure <code>O2_SERVICE_GRAPH_EXCLUDE_INTERNAL_SPANS=true</code>
|
||||
</div>
|
||||
<div>to process internal spans for service edges.</div>
|
||||
</div>
|
||||
</div>
|
||||
<q-btn
|
||||
outline
|
||||
color="primary"
|
||||
label="Refresh"
|
||||
icon="refresh"
|
||||
@click="loadServiceGraph"
|
||||
class="q-mt-lg"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div v-else class="tw:h-full">
|
||||
@@ -804,4 +756,24 @@ code {
|
||||
font-weight: 600;
|
||||
color: #333;
|
||||
}
|
||||
|
||||
.view-toggle-buttons {
|
||||
:deep(.q-btn) {
|
||||
padding-left: 1rem;
|
||||
padding-right: 1rem;
|
||||
}
|
||||
|
||||
:deep(.q-btn-item) {
|
||||
padding-left: 1rem;
|
||||
padding-right: 1rem;
|
||||
}
|
||||
|
||||
:deep(.q-btn__content) {
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
:deep(.q-separator) {
|
||||
margin: 0.25rem 0;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
Reference in New Issue
Block a user