diff --git a/backend/.sqlx/query-7a14147a9d8d566e6ea627f32f63a6ac6f928103dd7725060712fa306ea0ccf1.json b/backend/.sqlx/query-7a14147a9d8d566e6ea627f32f63a6ac6f928103dd7725060712fa306ea0ccf1.json new file mode 100644 index 0000000000..b6b76b39bc --- /dev/null +++ b/backend/.sqlx/query-7a14147a9d8d566e6ea627f32f63a6ac6f928103dd7725060712fa306ea0ccf1.json @@ -0,0 +1,42 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH holding AS (\n -- Holding a worker, by the same test the fairness refresh counts with: a\n -- live runtime ping, and not a flow job, which stays running between steps\n -- without a worker.\n SELECT DISTINCT q.workspace_id FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n JOIN v2_job_runtime r ON r.id = q.id\n WHERE q.tag = $1 AND q.running = true AND q.suspend_until IS NULL\n AND q.workspace_id = ANY($3::text[])\n AND r.ping > NOW() - interval '60 seconds'\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n ), waiting AS (\n SELECT q.workspace_id, q.scheduled_for, h.workspace_id IS NOT NULL AS excluded\n FROM v2_job_queue q LEFT JOIN holding h USING (workspace_id)\n WHERE q.tag = $1\n AND q.scheduled_for <= NOW() - $2::interval\n AND q.running = false\n )\n SELECT COUNT(*) FILTER (WHERE NOT excluded) as count,\n COUNT(*) FILTER (WHERE excluded) as excluded_count,\n MIN(scheduled_for) FILTER (WHERE NOT excluded) as oldest_job,\n array_agg(DISTINCT workspace_id) FILTER (WHERE excluded) as excluded_workspaces\n FROM waiting\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "excluded_count", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "oldest_job", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "excluded_workspaces", + "type_info": "VarcharArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Interval", + "TextArray" + ] + }, + "nullable": [ + null, + null, + null, + null + ] + }, + "hash": "7a14147a9d8d566e6ea627f32f63a6ac6f928103dd7725060712fa306ea0ccf1" +} diff --git a/backend/.sqlx/query-9e2810312302b36d3b4d761481c00296ee84c9536228496e19b4ec5df1781bc5.json b/backend/.sqlx/query-9e2810312302b36d3b4d761481c00296ee84c9536228496e19b4ec5df1781bc5.json deleted file mode 100644 index 70df593afe..0000000000 --- a/backend/.sqlx/query-9e2810312302b36d3b4d761481c00296ee84c9536228496e19b4ec5df1781bc5.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT COUNT(*) as count, \n MIN(scheduled_for) as oldest_job\n FROM v2_job_queue \n WHERE tag = $1 \n AND scheduled_for <= NOW() - $2::interval \n AND running = false\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "oldest_job", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - "Interval" - ] - }, - "nullable": [ - null, - null - ] - }, - "hash": "9e2810312302b36d3b4d761481c00296ee84c9536228496e19b4ec5df1781bc5" -} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 303cd2c59d..ca9ee26708 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -07883025e535974b9a01b20913f904b8f798d311 +ff4169005ce8bf08f5dc8625c8e55ac70dd0db50 diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index bf0492f47b..a2783389c0 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -308,6 +308,25 @@ pub fn max_job_duration_secs(cloud_premium_workspace: bool) -> u64 { } } pub const MIN_PERIODIC_SCRIPT_INTERVAL_SECONDS: u64 = 60; + +/// `background_task_state` row the workspace fairness refresh writes [`WorkspaceFairnessState`] +/// to. Only pulls drive the refresh, so a row whose `updated_at` is old means no worker has +/// pulled lately, not that nothing is overloaded. +pub const WORKSPACE_FAIRNESS_STATE_NAME: &str = "workspace_fairness"; + +#[derive(Serialize, Deserialize, Default, Debug)] +pub struct WorkspaceFairnessState { + #[serde(default)] + pub overloaded: Vec, + /// Admission probability for capped pulls in parts per 10_000. Absent in rows written + /// before it existed, which leaves the admission atomic at its default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub admission_ppm: Option, + /// When each workspace in `overloaded` entered the list in its current episode of being + /// capped, carried across refreshes until it leaves. + #[serde(default)] + pub capped_since: HashMap>, +} /// Default for [`CONCURRENCY_KEY_MAX_QUEUED`]; also the value the setting loader restores when /// the setting is cleared or malformed. pub const CONCURRENCY_KEY_MAX_QUEUED_DEFAULT: u32 = 10_000;