mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-17 16:20:25 +00:00
feat: fairness-aware queue alerts and a capped-workspace alert (#12432)
* feat: fairness-aware queue alerts and a capped-workspace alert Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PjsYvtgPD3rvKDd8155bTZ * fix: scope queue alert exclusion per tag, pin the EE companion Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PjsYvtgPD3rvKDd8155bTZ * fix: count only jobs holding a worker in the queue alert exclusion Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PjsYvtgPD3rvKDd8155bTZ * fix: drop fairness state when off, remove orphaned query cache entry Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PjsYvtgPD3rvKDd8155bTZ * chore: update ee-repo-ref to ff4169005ce8bf08f5dc8625c8e55ac70dd0db50 This commit updates the EE repository reference after PR #985 was merged in windmill-ee-private. Previous ee-repo-ref: ab0b2f2a1d898dabc6571fd81d3014f1ca4ad2f6 New ee-repo-ref: ff4169005ce8bf08f5dc8625c8e55ac70dd0db50 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
windmill-internal-app[bot]
parent
df56c111bb
commit
9724f3bdf0
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n WITH holding AS (\n -- Holding a worker, by the same test the fairness refresh counts with: a\n -- live runtime ping, and not a flow job, which stays running between steps\n -- without a worker.\n SELECT DISTINCT q.workspace_id FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n JOIN v2_job_runtime r ON r.id = q.id\n WHERE q.tag = $1 AND q.running = true AND q.suspend_until IS NULL\n AND q.workspace_id = ANY($3::text[])\n AND r.ping > NOW() - interval '60 seconds'\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n ), waiting AS (\n SELECT q.workspace_id, q.scheduled_for, h.workspace_id IS NOT NULL AS excluded\n FROM v2_job_queue q LEFT JOIN holding h USING (workspace_id)\n WHERE q.tag = $1\n AND q.scheduled_for <= NOW() - $2::interval\n AND q.running = false\n )\n SELECT COUNT(*) FILTER (WHERE NOT excluded) as count,\n COUNT(*) FILTER (WHERE excluded) as excluded_count,\n MIN(scheduled_for) FILTER (WHERE NOT excluded) as oldest_job,\n array_agg(DISTINCT workspace_id) FILTER (WHERE excluded) as excluded_workspaces\n FROM waiting\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "count",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "excluded_count",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "oldest_job",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "excluded_workspaces",
|
||||
"type_info": "VarcharArray"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Interval",
|
||||
"TextArray"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "7a14147a9d8d566e6ea627f32f63a6ac6f928103dd7725060712fa306ea0ccf1"
|
||||
}
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n SELECT COUNT(*) as count, \n MIN(scheduled_for) as oldest_job\n FROM v2_job_queue \n WHERE tag = $1 \n AND scheduled_for <= NOW() - $2::interval \n AND running = false\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "count",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "oldest_job",
|
||||
"type_info": "Timestamptz"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Interval"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "9e2810312302b36d3b4d761481c00296ee84c9536228496e19b4ec5df1781bc5"
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
07883025e535974b9a01b20913f904b8f798d311
|
||||
ff4169005ce8bf08f5dc8625c8e55ac70dd0db50
|
||||
|
||||
@@ -308,6 +308,25 @@ pub fn max_job_duration_secs(cloud_premium_workspace: bool) -> u64 {
|
||||
}
|
||||
}
|
||||
pub const MIN_PERIODIC_SCRIPT_INTERVAL_SECONDS: u64 = 60;
|
||||
|
||||
/// `background_task_state` row the workspace fairness refresh writes [`WorkspaceFairnessState`]
|
||||
/// to. Only pulls drive the refresh, so a row whose `updated_at` is old means no worker has
|
||||
/// pulled lately, not that nothing is overloaded.
|
||||
pub const WORKSPACE_FAIRNESS_STATE_NAME: &str = "workspace_fairness";
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug)]
|
||||
pub struct WorkspaceFairnessState {
|
||||
#[serde(default)]
|
||||
pub overloaded: Vec<String>,
|
||||
/// Admission probability for capped pulls in parts per 10_000. Absent in rows written
|
||||
/// before it existed, which leaves the admission atomic at its default.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub admission_ppm: Option<u32>,
|
||||
/// When each workspace in `overloaded` entered the list in its current episode of being
|
||||
/// capped, carried across refreshes until it leaves.
|
||||
#[serde(default)]
|
||||
pub capped_since: HashMap<String, chrono::DateTime<chrono::Utc>>,
|
||||
}
|
||||
/// Default for [`CONCURRENCY_KEY_MAX_QUEUED`]; also the value the setting loader restores when
|
||||
/// the setting is cleared or malformed.
|
||||
pub const CONCURRENCY_KEY_MAX_QUEUED_DEFAULT: u32 = 10_000;
|
||||
|
||||
Reference in New Issue
Block a user