diff --git a/backend/src/main.rs b/backend/src/main.rs index 8d42a8e031..6148ab5569 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -689,9 +689,25 @@ async fn windmill_main() -> anyhow::Result<()> { _ => {} } + // Fairness needs an Enterprise license, which a CE build (`private` included) never has, so + // no workspace would ever be overloaded and the worker would idle forever. + if *windmill_common::worker::WORKSPACE_FAIRNESS_RESERVE_WORKER && !cfg!(feature = "enterprise") + { + eprintln!("WORKSPACE_FAIRNESS_RESERVE_WORKER requires an Enterprise build"); + std::process::exit(1); + } + // Before `num_workers`: it reads `WORKER_GROUP`, which an agent authenticating with a // workload identity token only learns from the server. let agent_setup = if mode == Mode::Agent { + // An agent pulls through the server with its own query, which the reserve gate never sees. + if *windmill_common::worker::WORKSPACE_FAIRNESS_RESERVE_WORKER { + eprintln!( + "WORKSPACE_FAIRNESS_RESERVE_WORKER is not supported on agent workers: run reserve \ + workers with a database connection" + ); + std::process::exit(1); + } let agent_config = match AgentConfig::from_env() { Ok(config) => config, Err(e) => { diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index f69d484b36..82d98df7d6 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -361,7 +361,7 @@ pub async fn initial_load( } }); - // Workspace fairness (cloud-only). The percentage/duration/min knobs apply + // Workspace fairness (Enterprise). The percentage/duration/min knobs apply // *before* the enabled flag so that `apply_workspace_fairness_enabled` reads // current values when re-storing the pull queries, which declaration order gives us. pass.setting( @@ -982,6 +982,11 @@ pub async fn apply_workspace_fairness_enabled(value: Option) _ => false, }; let prev = WORKSPACE_FAIRNESS_ENABLED.swap(new_enabled, Ordering::Relaxed); + if !new_enabled && *windmill_common::worker::WORKSPACE_FAIRNESS_RESERVE_WORKER { + tracing::warn!( + "fairness reserve worker stays idle: {WORKSPACE_FAIRNESS_ENABLED_SETTING} is off" + ); + } // Re-store the pull queries so the fairness variants appear/disappear in // lockstep with the toggle. if prev != new_enabled { diff --git a/backend/tests/suspended_pull_index.rs b/backend/tests/suspended_pull_index.rs index 4b31f4d45e..c1e81009fe 100644 --- a/backend/tests/suspended_pull_index.rs +++ b/backend/tests/suspended_pull_index.rs @@ -5,7 +5,7 @@ use serde_json::Value; use sqlx::{Pool, Postgres}; -use windmill_common::worker::make_suspended_pull_query; +use windmill_common::worker::{make_suspended_pull_query, make_suspended_pull_query_fairness}; /// Depth-first walk of an `EXPLAIN (FORMAT JSON)` plan tree. fn nodes(plan: &Value, out: &mut Vec) { @@ -39,19 +39,34 @@ async fn suspended_pull_tests_resume_time_inside_the_index( let version: String = sqlx::query_scalar("SELECT version()") .fetch_one(&mut *conn) .await?; + let tags = ["flow".to_string()]; // FORMAT JSON rather than the default: `Index Cond` and `Filter` are separate keys on the // node, so this does not ride on EXPLAIN's line layout staying put across a major bump. - let explained: Value = sqlx::query_scalar(&format!( + let plain: Value = sqlx::query_scalar(&format!( "EXPLAIN (FORMAT JSON) {}", - make_suspended_pull_query(&["flow".to_string()]) + make_suspended_pull_query(&tags) )) .bind("test-worker") .fetch_one(&mut *conn) .await?; + // The variant fairness reserve workers resume through. + let fairness: Value = sqlx::query_scalar(&format!( + "EXPLAIN (FORMAT JSON) {}", + make_suspended_pull_query_fairness(&tags) + )) + .bind("test-worker") + .bind(vec!["overloaded-workspace".to_string()]) + .fetch_one(&mut *conn) + .await?; + assert_resume_test_in_index(&plain, &version)?; + assert_resume_test_in_index(&fairness, &version)?; + Ok(()) +} +fn assert_resume_test_in_index(explained: &Value, version: &str) -> anyhow::Result<()> { let mut all = vec![]; nodes(&explained[0]["Plan"], &mut all); - let pretty = serde_json::to_string_pretty(&explained)?; + let pretty = serde_json::to_string_pretty(explained)?; let scan = all .iter() .find(|n| n["Index Name"] == "queue_suspended_v2") diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 463eae1e5c..44701f936d 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -824,7 +824,7 @@ pub async fn delete_global_setting(db: &DB, key: &str) -> error::Result<()> { Ok(()) } /// Returns true when `key` is one of the workspace-fairness settings whose -/// writes must be gated to cloud only. +/// writes require an Enterprise license. fn is_workspace_fairness_setting(key: &str) -> bool { matches!( key, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 8985ef3fde..03d0edf8d6 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -471,7 +471,7 @@ pub const ALERT_CONFIG_SETTING: &str = "alert_job_queue_waiting"; /// The instance's workspace roles (EE), a list of `crate::roles::RoleDefinition`. pub const WORKSPACE_ROLES_SETTING: &str = "workspace_roles"; -// Workspace fairness: cloud-only mechanism that caps any single workspace at +// Workspace fairness (Enterprise): caps any single workspace at // `workspace_fairness_max_percent`% of the shared worker pool once it has been // occupying it for more than `workspace_fairness_duration_secs` seconds. See // `windmill-queue/src/workspace_fairness.rs`. diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 86bd367614..bf0492f47b 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -494,8 +494,9 @@ lazy_static::lazy_static! { pub static ref WORKER_PULL_QUERIES: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(vec![]); pub static ref WORKER_PULL_QUERIES_FAIRNESS: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(vec![]); pub static ref WORKER_SUSPENDED_PULL_QUERY: arc_swap::ArcSwap = arc_swap::ArcSwap::from_pointee("".to_string()); + pub static ref WORKER_SUSPENDED_PULL_QUERY_FAIRNESS: arc_swap::ArcSwap = arc_swap::ArcSwap::from_pointee("".to_string()); - // Workspace fairness (cloud-only). When enabled, a workspace whose footprint over the rolling + // Workspace fairness (Enterprise). When enabled, a workspace whose footprint over the rolling // `WORKSPACE_FAIRNESS_DURATION_SECS` window represents >= `WORKSPACE_FAIRNESS_MAX_PERCENT`% of // all worker activity gets excluded from the pull query, freeing slots for other workspaces. // The list of overloaded workspaces is computed cluster-wide via a single coordinated UPDATE @@ -518,6 +519,12 @@ lazy_static::lazy_static! { /// `should_admit_capped` is moot and "admit all" is the correct no-op. pub static ref WORKSPACE_FAIRNESS_ADMISSION_PPM: AtomicU32 = AtomicU32::new(10_000); + /// Pulls nothing while no workspace is overloaded, and otherwise only jobs of the workspaces + /// that are not. See "Reserve workers" in `windmill_queue::workspace_fairness`. + pub static ref WORKSPACE_FAIRNESS_RESERVE_WORKER: bool = + std::env::var("WORKSPACE_FAIRNESS_RESERVE_WORKER") + .is_ok_and(|x| x == "1" || x.eq_ignore_ascii_case("true")); + /// Cloud-only ceiling on the number of jobs queued behind a single concurrency key. /// A concurrency-limited key drains at most `concurrent_limit` jobs per window, so a /// producer pushing faster than that grows an unbounded backlog that no amount of @@ -867,29 +874,43 @@ fn format_pull_query(peek: String) -> String { // sync with `queue_suspended_v2` (migration 20260826202939): if it no longer matches, the // test silently reverts to a heap filter over every suspended row on every worker poll. pub fn make_suspended_pull_query(tags: &[String]) -> String { + suspended_pull_query(tags, "") +} + +/// [`make_suspended_pull_query`] leaving out the workspaces bound as `$2::text[]`. +pub fn make_suspended_pull_query_fairness(tags: &[String]) -> String { + suspended_pull_query(tags, "AND workspace_id <> ALL($2::text[])") +} + +fn suspended_pull_query(tags: &[String], extra_filter: &str) -> String { format_pull_query(format!( "SELECT id FROM v2_job_queue WHERE suspend_until IS NOT NULL AND (CASE WHEN suspend <= 0 THEN '-infinity'::timestamptz ELSE suspend_until END) <= now() AND tag IN ({}) + {extra_filter} ORDER BY priority DESC NULLS LAST, created_at FOR UPDATE SKIP LOCKED LIMIT 1", tags.iter().map(|x| format!("'{x}'")).join(", ") )) } -// pub async fn make_suspended + pub async fn store_suspended_pull_query(wc: &WorkerConfig) { // An empty tag list must clear the query: keeping the previous one would leave a reloaded // worker pulling suspended jobs for tags it no longer listens to. - let query = if wc.worker_tags.is_empty() { + let (query, fairness_query) = if wc.worker_tags.is_empty() { tracing::error!("Empty tags in worker tags, pulling no suspended jobs"); - String::new() + (String::new(), String::new()) } else { - make_suspended_pull_query(&wc.worker_tags) + ( + make_suspended_pull_query(&wc.worker_tags), + make_suspended_pull_query_fairness(&wc.worker_tags), + ) }; WORKER_SUSPENDED_PULL_QUERY.store(std::sync::Arc::new(query)); + WORKER_SUSPENDED_PULL_QUERY_FAIRNESS.store(std::sync::Arc::new(fairness_query)); } pub fn make_pull_query(tags: &[String]) -> String { diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 873f9b58ff..bf8c39bd76 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -85,7 +85,8 @@ use windmill_common::{ worker::{ to_raw_value, CLOUD_HOSTED, DEPENDENCY_JOB_TAG, DISABLE_FLOW_SCRIPT, NO_LOGS, PREVIEW_TAGS_OVERRIDE, WORKER_PULL_QUERIES, WORKER_PULL_QUERIES_FAIRNESS, - WORKER_SUSPENDED_PULL_QUERY, WORKSPACE_FAIRNESS_OVERLOADED, + WORKER_SUSPENDED_PULL_QUERY, WORKER_SUSPENDED_PULL_QUERY_FAIRNESS, + WORKSPACE_FAIRNESS_OVERLOADED, WORKSPACE_FAIRNESS_RESERVE_WORKER, }, DB, METRICS_ENABLED, }; @@ -5167,14 +5168,30 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; return Ok((None, false)); } - let r = if suspend_first { + let r = if !suspend_first { + None + } else if *WORKSPACE_FAIRNESS_RESERVE_WORKER { + // Parked WAC parents and parallelism-capped flow branches resume only through this + // pull, so a reserve must resume the ones it started, but never the overloaded + // workspace's. Refreshed here too: after a successful resume the worker keeps + // resuming first, and would never reach the refresh below. + crate::workspace_fairness::maybe_refresh_overloaded(db); + let overloaded = WORKSPACE_FAIRNESS_OVERLOADED.load_full(); + if overloaded.is_empty() { + None + } else { + sqlx::query_as::<_, PulledJob>(&WORKER_SUSPENDED_PULL_QUERY_FAIRNESS.load()) + .bind(worker_name) + .bind(overloaded.as_slice()) + .fetch_optional(db) + .await? + } + } else { // tracing::info!("Pulling job with query: {}", query); sqlx::query_as::<_, PulledJob>(&query) .bind(worker_name) .fetch_optional(db) .await? - } else { - None }; if r.is_none() { @@ -5206,8 +5223,13 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( crate::workspace_fairness::maybe_refresh_overloaded(db); let overloaded = WORKSPACE_FAIRNESS_OVERLOADED.load_full(); let fairness_active = !overloaded.is_empty(); + let reserve = *WORKSPACE_FAIRNESS_RESERVE_WORKER; - if fairness_active && !crate::workspace_fairness::should_admit_capped() { + if reserve && !fairness_active { + return Ok((None, false)); + } + + if fairness_active && (reserve || !crate::workspace_fairness::should_admit_capped()) { let fairness_queries = WORKER_PULL_QUERIES_FAIRNESS.load(); let overloaded_slice: &[String] = overloaded.as_slice(); for query in fairness_queries.iter() { @@ -5230,7 +5252,9 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( } } - if highest_priority_job.is_none() { + // A reserve worker never falls back: that fallback is how a capped workspace gets + // back onto a worker. + if highest_priority_job.is_none() && !reserve { // Standard pull path. Also acts as the fallback when fairness // filtered out every candidate: prefer running a capped // workspace's job over leaving a worker idle. (The cap is diff --git a/backend/windmill-queue/src/workspace_fairness.rs b/backend/windmill-queue/src/workspace_fairness.rs index 8eb3a69394..0ab6a1d113 100644 --- a/backend/windmill-queue/src/workspace_fairness.rs +++ b/backend/windmill-queue/src/workspace_fairness.rs @@ -143,9 +143,8 @@ //! the first iteration: with `p_c` correctly tightened, victims do get the //! slot they need *when they exist*, and absent victims, falling back to //! the capped pool is the right behaviour (otherwise the cluster -//! under-utilises itself for no benefit). Adding a reserve-capacity skip -//! is a fine-tuning lever for bursty victim arrival patterns and is left -//! as a follow-up. +//! under-utilises itself for no benefit). Reserve capacity is opt-in per +//! worker instead, see "Reserve workers" below. //! //! ### Degenerate cases //! @@ -248,6 +247,18 @@ //! cached state in one pull cycle (resetting `WORKSPACE_FAIRNESS_OVERLOADED` //! to empty and `WORKSPACE_FAIRNESS_ADMISSION_PPM` to 10_000 = "admit all"), //! so toggling the feature off without restarting workers is safe. +//! +//! ## 7. Reserve workers +//! +//! Admission only steers what a worker pulls *next*. A workspace whose jobs +//! already hold every worker (sync parents polling for children that need a +//! worker themselves) leaves no pull to steer, for as long as those jobs run. +//! A worker started with `WORKSPACE_FAIRNESS_RESERVE_WORKER=true` pulls +//! nothing while the overloaded set is empty, and otherwise only the fairness +//! queries (ordinary and suspended-resume), with no fallback. It stays idle rather +//! than serving everyone while nothing is capped: a burst landing before the +//! refresh flags its workspace would otherwise take it too. Fairness must be +//! enabled, or the set stays empty and a reserve worker never pulls. #[cfg(feature = "private")] #[allow(unused)] diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index b00db47ad4..070325f1e4 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -3462,6 +3462,12 @@ pub async fn run_worker( WORKER_CONFIG.load() ); } + if *windmill_common::worker::WORKSPACE_FAIRNESS_RESERVE_WORKER { + tracing::info!( + worker = %worker_name, hostname = %hostname, + "fairness reserve worker: pulls only while a workspace is overloaded, and only other workspaces' jobs" + ); + } // Dedicated workers wait for the init script before installing their dependencies, so it has to // be queued before they are spawned.