feat: fairness reserve workers that only serve non-overloaded workspaces (#12431)

* feat: fairness reserve workers that only serve non-overloaded workspaces

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: refuse the fairness reserve flag on agent workers

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: let reserve workers resume suspended jobs of non-overloaded workspaces

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: refresh overload state before reserve resumes, share suspended query

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: refuse reserve workers on CE builds, warn while fairness is off

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: gate the reserve flag on the enterprise feature, tidy comments

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* docs: fairness setting comment names the enterprise gate

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
2026-11-14 07:55:44 +01:00
committed by GitHub
co-authored by Claude Opus 5.5
parent a9d9e42f27
commit df56c111bb
9 changed files with 119 additions and 21 deletions
+16
View File
@@ -689,9 +689,25 @@ async fn windmill_main() -> anyhow::Result<()> {
_ => {}
}
// Fairness needs an Enterprise license, which a CE build (`private` included) never has, so
// no workspace would ever be overloaded and the worker would idle forever.
if *windmill_common::worker::WORKSPACE_FAIRNESS_RESERVE_WORKER && !cfg!(feature = "enterprise")
{
eprintln!("WORKSPACE_FAIRNESS_RESERVE_WORKER requires an Enterprise build");
std::process::exit(1);
}
// Before `num_workers`: it reads `WORKER_GROUP`, which an agent authenticating with a
// workload identity token only learns from the server.
let agent_setup = if mode == Mode::Agent {
// An agent pulls through the server with its own query, which the reserve gate never sees.
if *windmill_common::worker::WORKSPACE_FAIRNESS_RESERVE_WORKER {
eprintln!(
"WORKSPACE_FAIRNESS_RESERVE_WORKER is not supported on agent workers: run reserve \
workers with a database connection"
);
std::process::exit(1);
}
let agent_config = match AgentConfig::from_env() {
Ok(config) => config,
Err(e) => {
+6 -1
View File
@@ -361,7 +361,7 @@ pub async fn initial_load(
}
});
// Workspace fairness (cloud-only). The percentage/duration/min knobs apply
// Workspace fairness (Enterprise). The percentage/duration/min knobs apply
// *before* the enabled flag so that `apply_workspace_fairness_enabled` reads
// current values when re-storing the pull queries, which declaration order gives us.
pass.setting(
@@ -982,6 +982,11 @@ pub async fn apply_workspace_fairness_enabled(value: Option<serde_json::Value>)
_ => false,
};
let prev = WORKSPACE_FAIRNESS_ENABLED.swap(new_enabled, Ordering::Relaxed);
if !new_enabled && *windmill_common::worker::WORKSPACE_FAIRNESS_RESERVE_WORKER {
tracing::warn!(
"fairness reserve worker stays idle: {WORKSPACE_FAIRNESS_ENABLED_SETTING} is off"
);
}
// Re-store the pull queries so the fairness variants appear/disappear in
// lockstep with the toggle.
if prev != new_enabled {
+19 -4
View File
@@ -5,7 +5,7 @@
use serde_json::Value;
use sqlx::{Pool, Postgres};
use windmill_common::worker::make_suspended_pull_query;
use windmill_common::worker::{make_suspended_pull_query, make_suspended_pull_query_fairness};
/// Depth-first walk of an `EXPLAIN (FORMAT JSON)` plan tree.
fn nodes(plan: &Value, out: &mut Vec<Value>) {
@@ -39,19 +39,34 @@ async fn suspended_pull_tests_resume_time_inside_the_index(
let version: String = sqlx::query_scalar("SELECT version()")
.fetch_one(&mut *conn)
.await?;
let tags = ["flow".to_string()];
// FORMAT JSON rather than the default: `Index Cond` and `Filter` are separate keys on the
// node, so this does not ride on EXPLAIN's line layout staying put across a major bump.
let explained: Value = sqlx::query_scalar(&format!(
let plain: Value = sqlx::query_scalar(&format!(
"EXPLAIN (FORMAT JSON) {}",
make_suspended_pull_query(&["flow".to_string()])
make_suspended_pull_query(&tags)
))
.bind("test-worker")
.fetch_one(&mut *conn)
.await?;
// The variant fairness reserve workers resume through.
let fairness: Value = sqlx::query_scalar(&format!(
"EXPLAIN (FORMAT JSON) {}",
make_suspended_pull_query_fairness(&tags)
))
.bind("test-worker")
.bind(vec!["overloaded-workspace".to_string()])
.fetch_one(&mut *conn)
.await?;
assert_resume_test_in_index(&plain, &version)?;
assert_resume_test_in_index(&fairness, &version)?;
Ok(())
}
fn assert_resume_test_in_index(explained: &Value, version: &str) -> anyhow::Result<()> {
let mut all = vec![];
nodes(&explained[0]["Plan"], &mut all);
let pretty = serde_json::to_string_pretty(&explained)?;
let pretty = serde_json::to_string_pretty(explained)?;
let scan = all
.iter()
.find(|n| n["Index Name"] == "queue_suspended_v2")
+1 -1
View File
@@ -824,7 +824,7 @@ pub async fn delete_global_setting(db: &DB, key: &str) -> error::Result<()> {
Ok(())
}
/// Returns true when `key` is one of the workspace-fairness settings whose
/// writes must be gated to cloud only.
/// writes require an Enterprise license.
fn is_workspace_fairness_setting(key: &str) -> bool {
matches!(
key,
@@ -471,7 +471,7 @@ pub const ALERT_CONFIG_SETTING: &str = "alert_job_queue_waiting";
/// The instance's workspace roles (EE), a list of `crate::roles::RoleDefinition`.
pub const WORKSPACE_ROLES_SETTING: &str = "workspace_roles";
// Workspace fairness: cloud-only mechanism that caps any single workspace at
// Workspace fairness (Enterprise): caps any single workspace at
// `workspace_fairness_max_percent`% of the shared worker pool once it has been
// occupying it for more than `workspace_fairness_duration_secs` seconds. See
// `windmill-queue/src/workspace_fairness.rs`.
+26 -5
View File
@@ -494,8 +494,9 @@ lazy_static::lazy_static! {
pub static ref WORKER_PULL_QUERIES: arc_swap::ArcSwap<Vec<String>> = arc_swap::ArcSwap::from_pointee(vec![]);
pub static ref WORKER_PULL_QUERIES_FAIRNESS: arc_swap::ArcSwap<Vec<String>> = arc_swap::ArcSwap::from_pointee(vec![]);
pub static ref WORKER_SUSPENDED_PULL_QUERY: arc_swap::ArcSwap<String> = arc_swap::ArcSwap::from_pointee("".to_string());
pub static ref WORKER_SUSPENDED_PULL_QUERY_FAIRNESS: arc_swap::ArcSwap<String> = arc_swap::ArcSwap::from_pointee("".to_string());
// Workspace fairness (cloud-only). When enabled, a workspace whose footprint over the rolling
// Workspace fairness (Enterprise). When enabled, a workspace whose footprint over the rolling
// `WORKSPACE_FAIRNESS_DURATION_SECS` window represents >= `WORKSPACE_FAIRNESS_MAX_PERCENT`% of
// all worker activity gets excluded from the pull query, freeing slots for other workspaces.
// The list of overloaded workspaces is computed cluster-wide via a single coordinated UPDATE
@@ -518,6 +519,12 @@ lazy_static::lazy_static! {
/// `should_admit_capped` is moot and "admit all" is the correct no-op.
pub static ref WORKSPACE_FAIRNESS_ADMISSION_PPM: AtomicU32 = AtomicU32::new(10_000);
/// Pulls nothing while no workspace is overloaded, and otherwise only jobs of the workspaces
/// that are not. See "Reserve workers" in `windmill_queue::workspace_fairness`.
pub static ref WORKSPACE_FAIRNESS_RESERVE_WORKER: bool =
std::env::var("WORKSPACE_FAIRNESS_RESERVE_WORKER")
.is_ok_and(|x| x == "1" || x.eq_ignore_ascii_case("true"));
/// Cloud-only ceiling on the number of jobs queued behind a single concurrency key.
/// A concurrency-limited key drains at most `concurrent_limit` jobs per window, so a
/// producer pushing faster than that grows an unbounded backlog that no amount of
@@ -867,29 +874,43 @@ fn format_pull_query(peek: String) -> String {
// sync with `queue_suspended_v2` (migration 20260826202939): if it no longer matches, the
// test silently reverts to a heap filter over every suspended row on every worker poll.
pub fn make_suspended_pull_query(tags: &[String]) -> String {
suspended_pull_query(tags, "")
}
/// [`make_suspended_pull_query`] leaving out the workspaces bound as `$2::text[]`.
pub fn make_suspended_pull_query_fairness(tags: &[String]) -> String {
suspended_pull_query(tags, "AND workspace_id <> ALL($2::text[])")
}
fn suspended_pull_query(tags: &[String], extra_filter: &str) -> String {
format_pull_query(format!(
"SELECT id
FROM v2_job_queue
WHERE suspend_until IS NOT NULL
AND (CASE WHEN suspend <= 0 THEN '-infinity'::timestamptz ELSE suspend_until END) <= now()
AND tag IN ({})
{extra_filter}
ORDER BY priority DESC NULLS LAST, created_at
FOR UPDATE SKIP LOCKED
LIMIT 1",
tags.iter().map(|x| format!("'{x}'")).join(", ")
))
}
// pub async fn make_suspended
pub async fn store_suspended_pull_query(wc: &WorkerConfig) {
// An empty tag list must clear the query: keeping the previous one would leave a reloaded
// worker pulling suspended jobs for tags it no longer listens to.
let query = if wc.worker_tags.is_empty() {
let (query, fairness_query) = if wc.worker_tags.is_empty() {
tracing::error!("Empty tags in worker tags, pulling no suspended jobs");
String::new()
(String::new(), String::new())
} else {
make_suspended_pull_query(&wc.worker_tags)
(
make_suspended_pull_query(&wc.worker_tags),
make_suspended_pull_query_fairness(&wc.worker_tags),
)
};
WORKER_SUSPENDED_PULL_QUERY.store(std::sync::Arc::new(query));
WORKER_SUSPENDED_PULL_QUERY_FAIRNESS.store(std::sync::Arc::new(fairness_query));
}
pub fn make_pull_query(tags: &[String]) -> String {
+30 -6
View File
@@ -85,7 +85,8 @@ use windmill_common::{
worker::{
to_raw_value, CLOUD_HOSTED, DEPENDENCY_JOB_TAG, DISABLE_FLOW_SCRIPT, NO_LOGS,
PREVIEW_TAGS_OVERRIDE, WORKER_PULL_QUERIES, WORKER_PULL_QUERIES_FAIRNESS,
WORKER_SUSPENDED_PULL_QUERY, WORKSPACE_FAIRNESS_OVERLOADED,
WORKER_SUSPENDED_PULL_QUERY, WORKER_SUSPENDED_PULL_QUERY_FAIRNESS,
WORKSPACE_FAIRNESS_OVERLOADED, WORKSPACE_FAIRNESS_RESERVE_WORKER,
},
DB, METRICS_ENABLED,
};
@@ -5167,14 +5168,30 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>(
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
return Ok((None, false));
}
let r = if suspend_first {
let r = if !suspend_first {
None
} else if *WORKSPACE_FAIRNESS_RESERVE_WORKER {
// Parked WAC parents and parallelism-capped flow branches resume only through this
// pull, so a reserve must resume the ones it started, but never the overloaded
// workspace's. Refreshed here too: after a successful resume the worker keeps
// resuming first, and would never reach the refresh below.
crate::workspace_fairness::maybe_refresh_overloaded(db);
let overloaded = WORKSPACE_FAIRNESS_OVERLOADED.load_full();
if overloaded.is_empty() {
None
} else {
sqlx::query_as::<_, PulledJob>(&WORKER_SUSPENDED_PULL_QUERY_FAIRNESS.load())
.bind(worker_name)
.bind(overloaded.as_slice())
.fetch_optional(db)
.await?
}
} else {
// tracing::info!("Pulling job with query: {}", query);
sqlx::query_as::<_, PulledJob>(&query)
.bind(worker_name)
.fetch_optional(db)
.await?
} else {
None
};
if r.is_none() {
@@ -5206,8 +5223,13 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>(
crate::workspace_fairness::maybe_refresh_overloaded(db);
let overloaded = WORKSPACE_FAIRNESS_OVERLOADED.load_full();
let fairness_active = !overloaded.is_empty();
let reserve = *WORKSPACE_FAIRNESS_RESERVE_WORKER;
if fairness_active && !crate::workspace_fairness::should_admit_capped() {
if reserve && !fairness_active {
return Ok((None, false));
}
if fairness_active && (reserve || !crate::workspace_fairness::should_admit_capped()) {
let fairness_queries = WORKER_PULL_QUERIES_FAIRNESS.load();
let overloaded_slice: &[String] = overloaded.as_slice();
for query in fairness_queries.iter() {
@@ -5230,7 +5252,9 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>(
}
}
if highest_priority_job.is_none() {
// A reserve worker never falls back: that fallback is how a capped workspace gets
// back onto a worker.
if highest_priority_job.is_none() && !reserve {
// Standard pull path. Also acts as the fallback when fairness
// filtered out every candidate: prefer running a capped
// workspace's job over leaving a worker idle. (The cap is
@@ -143,9 +143,8 @@
//! the first iteration: with `p_c` correctly tightened, victims do get the
//! slot they need *when they exist*, and absent victims, falling back to
//! the capped pool is the right behaviour (otherwise the cluster
//! under-utilises itself for no benefit). Adding a reserve-capacity skip
//! is a fine-tuning lever for bursty victim arrival patterns and is left
//! as a follow-up.
//! under-utilises itself for no benefit). Reserve capacity is opt-in per
//! worker instead, see "Reserve workers" below.
//!
//! ### Degenerate cases
//!
@@ -248,6 +247,18 @@
//! cached state in one pull cycle (resetting `WORKSPACE_FAIRNESS_OVERLOADED`
//! to empty and `WORKSPACE_FAIRNESS_ADMISSION_PPM` to 10_000 = "admit all"),
//! so toggling the feature off without restarting workers is safe.
//!
//! ## 7. Reserve workers
//!
//! Admission only steers what a worker pulls *next*. A workspace whose jobs
//! already hold every worker (sync parents polling for children that need a
//! worker themselves) leaves no pull to steer, for as long as those jobs run.
//! A worker started with `WORKSPACE_FAIRNESS_RESERVE_WORKER=true` pulls
//! nothing while the overloaded set is empty, and otherwise only the fairness
//! queries (ordinary and suspended-resume), with no fallback. It stays idle rather
//! than serving everyone while nothing is capped: a burst landing before the
//! refresh flags its workspace would otherwise take it too. Fairness must be
//! enabled, or the set stays empty and a reserve worker never pulls.
#[cfg(feature = "private")]
#[allow(unused)]
+6
View File
@@ -3462,6 +3462,12 @@ pub async fn run_worker(
WORKER_CONFIG.load()
);
}
if *windmill_common::worker::WORKSPACE_FAIRNESS_RESERVE_WORKER {
tracing::info!(
worker = %worker_name, hostname = %hostname,
"fairness reserve worker: pulls only while a workspace is overloaded, and only other workspaces' jobs"
);
}
// Dedicated workers wait for the init script before installing their dependencies, so it has to
// be queued before they are spawned.