fix(alerts): compare gate arrivals over an hour, not a day

The growth check asked whether a day's worth of jobs had arrived, so the
alert only tripped once roughly a day's backlog had accumulated behind
the gate. Compare against what the gate admits over an hour instead, and
drop the drain-horizon condition, which the growth check already implied.

Emitter changes live in windmill-ee-private (see ee-repo-ref.txt bump);
this commit carries the regression test and the query cache.
This commit is contained in:
Ruben Fiszel
2026-07-20 15:17:26 +00:00
parent 8cc55df839
commit 806aa6e1be
3 changed files with 29 additions and 7 deletions
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT concurrency_gate_id AS \"gate_id!\",\n COUNT(*) AS \"backlog!\",\n MIN(workspace_id) AS \"workspace_id!\",\n COUNT(*) FILTER (WHERE created_at >= NOW() - INTERVAL '24 hours')\n AS \"arrivals_24h!\"\n FROM v2_job_queue\n WHERE running = false\n AND concurrency_gate_id IS NOT NULL\n AND canceled_by IS NULL\n GROUP BY concurrency_gate_id\n HAVING MAX(concurrency_gated_at) >= NOW() - INTERVAL '1 hour'\n ",
"query": "\n SELECT concurrency_gate_id AS \"gate_id!\",\n COUNT(*) AS \"backlog!\",\n MIN(workspace_id) AS \"workspace_id!\",\n COUNT(*) FILTER (WHERE created_at >= NOW() - INTERVAL '1 hour' * $1)\n AS \"arrivals!\"\n FROM v2_job_queue\n WHERE running = false\n AND concurrency_gate_id IS NOT NULL\n AND canceled_by IS NULL\n GROUP BY concurrency_gate_id\n HAVING MAX(concurrency_gated_at) >= NOW() - INTERVAL '1 hour'\n ",
"describe": {
"columns": [
{
@@ -20,12 +20,14 @@
},
{
"ordinal": 3,
"name": "arrivals_24h!",
"name": "arrivals!",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
"Left": [
"Float8"
]
},
"nullable": [
true,
@@ -34,5 +36,5 @@
null
]
},
"hash": "a2278097381823098e1dff7cd91ffafc0262ff9e8440b562a0b5b32180f7d926"
"hash": "11ccd3d7a4fb565d66d330586c4fbb28567d9d538cefbc4f7b984114b8be6fb7"
}
+1 -1
View File
@@ -1 +1 @@
a31d6ae5e573d549e00e73f08b694a64a60ca119
f1f951359200bedfe31a75abd3c3f8962553f702
@@ -94,8 +94,9 @@ mod tests {
"a gate that cannot drain its backlog must alert, got {messages:?}"
);
assert!(
messages[0].contains("17280") && messages[0].contains("40000"),
"alert must name the backlog and the ceiling it exceeds: {}",
messages[0].contains("17280") && messages[0].contains("40000")
&& messages[0].contains("720"),
"alert must name the backlog and both ceilings it exceeds: {}",
messages[0]
);
}
@@ -195,4 +196,23 @@ mod tests {
.unwrap_or(0);
assert_eq!(unresolved, 0, "a fully drained gate must have its alert recovered");
}
/// The window governs how soon the alert fires, so a gate taking on less than
/// it admits over that window is keeping up and must stay silent even though
/// a long backlog sits in front of it.
#[ignore = "requires database setup - run with --ignored flag"]
#[sqlx::test(migrations = "../migrations")]
async fn arrivals_under_the_window_ceiling_do_not_alert(db: Pool<Postgres>) {
free_the_lock(&db).await;
queue_gated(&db, 40_000, SLOW_GATE, 48).await;
// 700 in the last hour against a 720/hour ceiling: still keeping up.
queue_gated(&db, 700, SLOW_GATE, 0).await;
concurrency_gate_alerts(&db).await;
assert!(
alert_messages(&db).await.is_empty(),
"arrivals below the gate ceiling for the window must not alert"
);
}
}