mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-17 00:20:51 +00:00
feat: fairness reserve workers that only serve non-overloaded workspaces (#12431)
* feat: fairness reserve workers that only serve non-overloaded workspaces Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: refuse the fairness reserve flag on agent workers Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: let reserve workers resume suspended jobs of non-overloaded workspaces Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: refresh overload state before reserve resumes, share suspended query Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: refuse reserve workers on CE builds, warn while fairness is off Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: gate the reserve flag on the enterprise feature, tidy comments Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * docs: fairness setting comment names the enterprise gate Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a9d9e42f27
commit
df56c111bb
@@ -689,9 +689,25 @@ async fn windmill_main() -> anyhow::Result<()> {
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// Fairness needs an Enterprise license, which a CE build (`private` included) never has, so
|
||||
// no workspace would ever be overloaded and the worker would idle forever.
|
||||
if *windmill_common::worker::WORKSPACE_FAIRNESS_RESERVE_WORKER && !cfg!(feature = "enterprise")
|
||||
{
|
||||
eprintln!("WORKSPACE_FAIRNESS_RESERVE_WORKER requires an Enterprise build");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
// Before `num_workers`: it reads `WORKER_GROUP`, which an agent authenticating with a
|
||||
// workload identity token only learns from the server.
|
||||
let agent_setup = if mode == Mode::Agent {
|
||||
// An agent pulls through the server with its own query, which the reserve gate never sees.
|
||||
if *windmill_common::worker::WORKSPACE_FAIRNESS_RESERVE_WORKER {
|
||||
eprintln!(
|
||||
"WORKSPACE_FAIRNESS_RESERVE_WORKER is not supported on agent workers: run reserve \
|
||||
workers with a database connection"
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
let agent_config = match AgentConfig::from_env() {
|
||||
Ok(config) => config,
|
||||
Err(e) => {
|
||||
|
||||
@@ -361,7 +361,7 @@ pub async fn initial_load(
|
||||
}
|
||||
});
|
||||
|
||||
// Workspace fairness (cloud-only). The percentage/duration/min knobs apply
|
||||
// Workspace fairness (Enterprise). The percentage/duration/min knobs apply
|
||||
// *before* the enabled flag so that `apply_workspace_fairness_enabled` reads
|
||||
// current values when re-storing the pull queries, which declaration order gives us.
|
||||
pass.setting(
|
||||
@@ -982,6 +982,11 @@ pub async fn apply_workspace_fairness_enabled(value: Option<serde_json::Value>)
|
||||
_ => false,
|
||||
};
|
||||
let prev = WORKSPACE_FAIRNESS_ENABLED.swap(new_enabled, Ordering::Relaxed);
|
||||
if !new_enabled && *windmill_common::worker::WORKSPACE_FAIRNESS_RESERVE_WORKER {
|
||||
tracing::warn!(
|
||||
"fairness reserve worker stays idle: {WORKSPACE_FAIRNESS_ENABLED_SETTING} is off"
|
||||
);
|
||||
}
|
||||
// Re-store the pull queries so the fairness variants appear/disappear in
|
||||
// lockstep with the toggle.
|
||||
if prev != new_enabled {
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
use serde_json::Value;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use windmill_common::worker::make_suspended_pull_query;
|
||||
use windmill_common::worker::{make_suspended_pull_query, make_suspended_pull_query_fairness};
|
||||
|
||||
/// Depth-first walk of an `EXPLAIN (FORMAT JSON)` plan tree.
|
||||
fn nodes(plan: &Value, out: &mut Vec<Value>) {
|
||||
@@ -39,19 +39,34 @@ async fn suspended_pull_tests_resume_time_inside_the_index(
|
||||
let version: String = sqlx::query_scalar("SELECT version()")
|
||||
.fetch_one(&mut *conn)
|
||||
.await?;
|
||||
let tags = ["flow".to_string()];
|
||||
// FORMAT JSON rather than the default: `Index Cond` and `Filter` are separate keys on the
|
||||
// node, so this does not ride on EXPLAIN's line layout staying put across a major bump.
|
||||
let explained: Value = sqlx::query_scalar(&format!(
|
||||
let plain: Value = sqlx::query_scalar(&format!(
|
||||
"EXPLAIN (FORMAT JSON) {}",
|
||||
make_suspended_pull_query(&["flow".to_string()])
|
||||
make_suspended_pull_query(&tags)
|
||||
))
|
||||
.bind("test-worker")
|
||||
.fetch_one(&mut *conn)
|
||||
.await?;
|
||||
// The variant fairness reserve workers resume through.
|
||||
let fairness: Value = sqlx::query_scalar(&format!(
|
||||
"EXPLAIN (FORMAT JSON) {}",
|
||||
make_suspended_pull_query_fairness(&tags)
|
||||
))
|
||||
.bind("test-worker")
|
||||
.bind(vec!["overloaded-workspace".to_string()])
|
||||
.fetch_one(&mut *conn)
|
||||
.await?;
|
||||
assert_resume_test_in_index(&plain, &version)?;
|
||||
assert_resume_test_in_index(&fairness, &version)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn assert_resume_test_in_index(explained: &Value, version: &str) -> anyhow::Result<()> {
|
||||
let mut all = vec![];
|
||||
nodes(&explained[0]["Plan"], &mut all);
|
||||
let pretty = serde_json::to_string_pretty(&explained)?;
|
||||
let pretty = serde_json::to_string_pretty(explained)?;
|
||||
let scan = all
|
||||
.iter()
|
||||
.find(|n| n["Index Name"] == "queue_suspended_v2")
|
||||
|
||||
@@ -824,7 +824,7 @@ pub async fn delete_global_setting(db: &DB, key: &str) -> error::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
/// Returns true when `key` is one of the workspace-fairness settings whose
|
||||
/// writes must be gated to cloud only.
|
||||
/// writes require an Enterprise license.
|
||||
fn is_workspace_fairness_setting(key: &str) -> bool {
|
||||
matches!(
|
||||
key,
|
||||
|
||||
@@ -471,7 +471,7 @@ pub const ALERT_CONFIG_SETTING: &str = "alert_job_queue_waiting";
|
||||
/// The instance's workspace roles (EE), a list of `crate::roles::RoleDefinition`.
|
||||
pub const WORKSPACE_ROLES_SETTING: &str = "workspace_roles";
|
||||
|
||||
// Workspace fairness: cloud-only mechanism that caps any single workspace at
|
||||
// Workspace fairness (Enterprise): caps any single workspace at
|
||||
// `workspace_fairness_max_percent`% of the shared worker pool once it has been
|
||||
// occupying it for more than `workspace_fairness_duration_secs` seconds. See
|
||||
// `windmill-queue/src/workspace_fairness.rs`.
|
||||
|
||||
@@ -494,8 +494,9 @@ lazy_static::lazy_static! {
|
||||
pub static ref WORKER_PULL_QUERIES: arc_swap::ArcSwap<Vec<String>> = arc_swap::ArcSwap::from_pointee(vec![]);
|
||||
pub static ref WORKER_PULL_QUERIES_FAIRNESS: arc_swap::ArcSwap<Vec<String>> = arc_swap::ArcSwap::from_pointee(vec![]);
|
||||
pub static ref WORKER_SUSPENDED_PULL_QUERY: arc_swap::ArcSwap<String> = arc_swap::ArcSwap::from_pointee("".to_string());
|
||||
pub static ref WORKER_SUSPENDED_PULL_QUERY_FAIRNESS: arc_swap::ArcSwap<String> = arc_swap::ArcSwap::from_pointee("".to_string());
|
||||
|
||||
// Workspace fairness (cloud-only). When enabled, a workspace whose footprint over the rolling
|
||||
// Workspace fairness (Enterprise). When enabled, a workspace whose footprint over the rolling
|
||||
// `WORKSPACE_FAIRNESS_DURATION_SECS` window represents >= `WORKSPACE_FAIRNESS_MAX_PERCENT`% of
|
||||
// all worker activity gets excluded from the pull query, freeing slots for other workspaces.
|
||||
// The list of overloaded workspaces is computed cluster-wide via a single coordinated UPDATE
|
||||
@@ -518,6 +519,12 @@ lazy_static::lazy_static! {
|
||||
/// `should_admit_capped` is moot and "admit all" is the correct no-op.
|
||||
pub static ref WORKSPACE_FAIRNESS_ADMISSION_PPM: AtomicU32 = AtomicU32::new(10_000);
|
||||
|
||||
/// Pulls nothing while no workspace is overloaded, and otherwise only jobs of the workspaces
|
||||
/// that are not. See "Reserve workers" in `windmill_queue::workspace_fairness`.
|
||||
pub static ref WORKSPACE_FAIRNESS_RESERVE_WORKER: bool =
|
||||
std::env::var("WORKSPACE_FAIRNESS_RESERVE_WORKER")
|
||||
.is_ok_and(|x| x == "1" || x.eq_ignore_ascii_case("true"));
|
||||
|
||||
/// Cloud-only ceiling on the number of jobs queued behind a single concurrency key.
|
||||
/// A concurrency-limited key drains at most `concurrent_limit` jobs per window, so a
|
||||
/// producer pushing faster than that grows an unbounded backlog that no amount of
|
||||
@@ -867,29 +874,43 @@ fn format_pull_query(peek: String) -> String {
|
||||
// sync with `queue_suspended_v2` (migration 20260826202939): if it no longer matches, the
|
||||
// test silently reverts to a heap filter over every suspended row on every worker poll.
|
||||
pub fn make_suspended_pull_query(tags: &[String]) -> String {
|
||||
suspended_pull_query(tags, "")
|
||||
}
|
||||
|
||||
/// [`make_suspended_pull_query`] leaving out the workspaces bound as `$2::text[]`.
|
||||
pub fn make_suspended_pull_query_fairness(tags: &[String]) -> String {
|
||||
suspended_pull_query(tags, "AND workspace_id <> ALL($2::text[])")
|
||||
}
|
||||
|
||||
fn suspended_pull_query(tags: &[String], extra_filter: &str) -> String {
|
||||
format_pull_query(format!(
|
||||
"SELECT id
|
||||
FROM v2_job_queue
|
||||
WHERE suspend_until IS NOT NULL
|
||||
AND (CASE WHEN suspend <= 0 THEN '-infinity'::timestamptz ELSE suspend_until END) <= now()
|
||||
AND tag IN ({})
|
||||
{extra_filter}
|
||||
ORDER BY priority DESC NULLS LAST, created_at
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1",
|
||||
tags.iter().map(|x| format!("'{x}'")).join(", ")
|
||||
))
|
||||
}
|
||||
// pub async fn make_suspended
|
||||
|
||||
pub async fn store_suspended_pull_query(wc: &WorkerConfig) {
|
||||
// An empty tag list must clear the query: keeping the previous one would leave a reloaded
|
||||
// worker pulling suspended jobs for tags it no longer listens to.
|
||||
let query = if wc.worker_tags.is_empty() {
|
||||
let (query, fairness_query) = if wc.worker_tags.is_empty() {
|
||||
tracing::error!("Empty tags in worker tags, pulling no suspended jobs");
|
||||
String::new()
|
||||
(String::new(), String::new())
|
||||
} else {
|
||||
make_suspended_pull_query(&wc.worker_tags)
|
||||
(
|
||||
make_suspended_pull_query(&wc.worker_tags),
|
||||
make_suspended_pull_query_fairness(&wc.worker_tags),
|
||||
)
|
||||
};
|
||||
WORKER_SUSPENDED_PULL_QUERY.store(std::sync::Arc::new(query));
|
||||
WORKER_SUSPENDED_PULL_QUERY_FAIRNESS.store(std::sync::Arc::new(fairness_query));
|
||||
}
|
||||
|
||||
pub fn make_pull_query(tags: &[String]) -> String {
|
||||
|
||||
@@ -85,7 +85,8 @@ use windmill_common::{
|
||||
worker::{
|
||||
to_raw_value, CLOUD_HOSTED, DEPENDENCY_JOB_TAG, DISABLE_FLOW_SCRIPT, NO_LOGS,
|
||||
PREVIEW_TAGS_OVERRIDE, WORKER_PULL_QUERIES, WORKER_PULL_QUERIES_FAIRNESS,
|
||||
WORKER_SUSPENDED_PULL_QUERY, WORKSPACE_FAIRNESS_OVERLOADED,
|
||||
WORKER_SUSPENDED_PULL_QUERY, WORKER_SUSPENDED_PULL_QUERY_FAIRNESS,
|
||||
WORKSPACE_FAIRNESS_OVERLOADED, WORKSPACE_FAIRNESS_RESERVE_WORKER,
|
||||
},
|
||||
DB, METRICS_ENABLED,
|
||||
};
|
||||
@@ -5167,14 +5168,30 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>(
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
|
||||
return Ok((None, false));
|
||||
}
|
||||
let r = if suspend_first {
|
||||
let r = if !suspend_first {
|
||||
None
|
||||
} else if *WORKSPACE_FAIRNESS_RESERVE_WORKER {
|
||||
// Parked WAC parents and parallelism-capped flow branches resume only through this
|
||||
// pull, so a reserve must resume the ones it started, but never the overloaded
|
||||
// workspace's. Refreshed here too: after a successful resume the worker keeps
|
||||
// resuming first, and would never reach the refresh below.
|
||||
crate::workspace_fairness::maybe_refresh_overloaded(db);
|
||||
let overloaded = WORKSPACE_FAIRNESS_OVERLOADED.load_full();
|
||||
if overloaded.is_empty() {
|
||||
None
|
||||
} else {
|
||||
sqlx::query_as::<_, PulledJob>(&WORKER_SUSPENDED_PULL_QUERY_FAIRNESS.load())
|
||||
.bind(worker_name)
|
||||
.bind(overloaded.as_slice())
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
}
|
||||
} else {
|
||||
// tracing::info!("Pulling job with query: {}", query);
|
||||
sqlx::query_as::<_, PulledJob>(&query)
|
||||
.bind(worker_name)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if r.is_none() {
|
||||
@@ -5206,8 +5223,13 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>(
|
||||
crate::workspace_fairness::maybe_refresh_overloaded(db);
|
||||
let overloaded = WORKSPACE_FAIRNESS_OVERLOADED.load_full();
|
||||
let fairness_active = !overloaded.is_empty();
|
||||
let reserve = *WORKSPACE_FAIRNESS_RESERVE_WORKER;
|
||||
|
||||
if fairness_active && !crate::workspace_fairness::should_admit_capped() {
|
||||
if reserve && !fairness_active {
|
||||
return Ok((None, false));
|
||||
}
|
||||
|
||||
if fairness_active && (reserve || !crate::workspace_fairness::should_admit_capped()) {
|
||||
let fairness_queries = WORKER_PULL_QUERIES_FAIRNESS.load();
|
||||
let overloaded_slice: &[String] = overloaded.as_slice();
|
||||
for query in fairness_queries.iter() {
|
||||
@@ -5230,7 +5252,9 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>(
|
||||
}
|
||||
}
|
||||
|
||||
if highest_priority_job.is_none() {
|
||||
// A reserve worker never falls back: that fallback is how a capped workspace gets
|
||||
// back onto a worker.
|
||||
if highest_priority_job.is_none() && !reserve {
|
||||
// Standard pull path. Also acts as the fallback when fairness
|
||||
// filtered out every candidate: prefer running a capped
|
||||
// workspace's job over leaving a worker idle. (The cap is
|
||||
|
||||
@@ -143,9 +143,8 @@
|
||||
//! the first iteration: with `p_c` correctly tightened, victims do get the
|
||||
//! slot they need *when they exist*, and absent victims, falling back to
|
||||
//! the capped pool is the right behaviour (otherwise the cluster
|
||||
//! under-utilises itself for no benefit). Adding a reserve-capacity skip
|
||||
//! is a fine-tuning lever for bursty victim arrival patterns and is left
|
||||
//! as a follow-up.
|
||||
//! under-utilises itself for no benefit). Reserve capacity is opt-in per
|
||||
//! worker instead, see "Reserve workers" below.
|
||||
//!
|
||||
//! ### Degenerate cases
|
||||
//!
|
||||
@@ -248,6 +247,18 @@
|
||||
//! cached state in one pull cycle (resetting `WORKSPACE_FAIRNESS_OVERLOADED`
|
||||
//! to empty and `WORKSPACE_FAIRNESS_ADMISSION_PPM` to 10_000 = "admit all"),
|
||||
//! so toggling the feature off without restarting workers is safe.
|
||||
//!
|
||||
//! ## 7. Reserve workers
|
||||
//!
|
||||
//! Admission only steers what a worker pulls *next*. A workspace whose jobs
|
||||
//! already hold every worker (sync parents polling for children that need a
|
||||
//! worker themselves) leaves no pull to steer, for as long as those jobs run.
|
||||
//! A worker started with `WORKSPACE_FAIRNESS_RESERVE_WORKER=true` pulls
|
||||
//! nothing while the overloaded set is empty, and otherwise only the fairness
|
||||
//! queries (ordinary and suspended-resume), with no fallback. It stays idle rather
|
||||
//! than serving everyone while nothing is capped: a burst landing before the
|
||||
//! refresh flags its workspace would otherwise take it too. Fairness must be
|
||||
//! enabled, or the set stays empty and a reserve worker never pulls.
|
||||
|
||||
#[cfg(feature = "private")]
|
||||
#[allow(unused)]
|
||||
|
||||
@@ -3462,6 +3462,12 @@ pub async fn run_worker(
|
||||
WORKER_CONFIG.load()
|
||||
);
|
||||
}
|
||||
if *windmill_common::worker::WORKSPACE_FAIRNESS_RESERVE_WORKER {
|
||||
tracing::info!(
|
||||
worker = %worker_name, hostname = %hostname,
|
||||
"fairness reserve worker: pulls only while a workspace is overloaded, and only other workspaces' jobs"
|
||||
);
|
||||
}
|
||||
|
||||
// Dedicated workers wait for the init script before installing their dependencies, so it has to
|
||||
// be queued before they are spawned.
|
||||
|
||||
Reference in New Issue
Block a user