mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-09-10 00:05:27 +00:00
fix(alerts): compare gate arrivals over an hour, not a day
The growth check asked whether a day's worth of jobs had arrived, so the alert only tripped once roughly a day's backlog had accumulated behind the gate. Compare against what the gate admits over an hour instead, and drop the drain-horizon condition, which the growth check already implied. Emitter changes live in windmill-ee-private (see ee-repo-ref.txt bump); this commit carries the regression test and the query cache.
This commit is contained in:
+6
-4
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n SELECT concurrency_gate_id AS \"gate_id!\",\n COUNT(*) AS \"backlog!\",\n MIN(workspace_id) AS \"workspace_id!\",\n COUNT(*) FILTER (WHERE created_at >= NOW() - INTERVAL '24 hours')\n AS \"arrivals_24h!\"\n FROM v2_job_queue\n WHERE running = false\n AND concurrency_gate_id IS NOT NULL\n AND canceled_by IS NULL\n GROUP BY concurrency_gate_id\n HAVING MAX(concurrency_gated_at) >= NOW() - INTERVAL '1 hour'\n ",
|
||||
"query": "\n SELECT concurrency_gate_id AS \"gate_id!\",\n COUNT(*) AS \"backlog!\",\n MIN(workspace_id) AS \"workspace_id!\",\n COUNT(*) FILTER (WHERE created_at >= NOW() - INTERVAL '1 hour' * $1)\n AS \"arrivals!\"\n FROM v2_job_queue\n WHERE running = false\n AND concurrency_gate_id IS NOT NULL\n AND canceled_by IS NULL\n GROUP BY concurrency_gate_id\n HAVING MAX(concurrency_gated_at) >= NOW() - INTERVAL '1 hour'\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -20,12 +20,14 @@
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "arrivals_24h!",
|
||||
"name": "arrivals!",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
"Left": [
|
||||
"Float8"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true,
|
||||
@@ -34,5 +36,5 @@
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "a2278097381823098e1dff7cd91ffafc0262ff9e8440b562a0b5b32180f7d926"
|
||||
"hash": "11ccd3d7a4fb565d66d330586c4fbb28567d9d538cefbc4f7b984114b8be6fb7"
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
a31d6ae5e573d549e00e73f08b694a64a60ca119
|
||||
f1f951359200bedfe31a75abd3c3f8962553f702
|
||||
@@ -94,8 +94,9 @@ mod tests {
|
||||
"a gate that cannot drain its backlog must alert, got {messages:?}"
|
||||
);
|
||||
assert!(
|
||||
messages[0].contains("17280") && messages[0].contains("40000"),
|
||||
"alert must name the backlog and the ceiling it exceeds: {}",
|
||||
messages[0].contains("17280") && messages[0].contains("40000")
|
||||
&& messages[0].contains("720"),
|
||||
"alert must name the backlog and both ceilings it exceeds: {}",
|
||||
messages[0]
|
||||
);
|
||||
}
|
||||
@@ -195,4 +196,23 @@ mod tests {
|
||||
.unwrap_or(0);
|
||||
assert_eq!(unresolved, 0, "a fully drained gate must have its alert recovered");
|
||||
}
|
||||
|
||||
/// The window governs how soon the alert fires, so a gate taking on less than
|
||||
/// it admits over that window is keeping up and must stay silent even though
|
||||
/// a long backlog sits in front of it.
|
||||
#[ignore = "requires database setup - run with --ignored flag"]
|
||||
#[sqlx::test(migrations = "../migrations")]
|
||||
async fn arrivals_under_the_window_ceiling_do_not_alert(db: Pool<Postgres>) {
|
||||
free_the_lock(&db).await;
|
||||
queue_gated(&db, 40_000, SLOW_GATE, 48).await;
|
||||
// 700 in the last hour against a 720/hour ceiling: still keeping up.
|
||||
queue_gated(&db, 700, SLOW_GATE, 0).await;
|
||||
|
||||
concurrency_gate_alerts(&db).await;
|
||||
|
||||
assert!(
|
||||
alert_messages(&db).await.is_empty(),
|
||||
"arrivals below the gate ceiling for the window must not alert"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user