starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-12 08:05:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: cap queued jobs per concurrency key on cloud (#10197)

Browse Source 
* feat: cap queued jobs per concurrency key on cloud

* fix: close preprocessed-flow bypass and bound concurrency cap scan

* fix: only cap concurrency keys with an active concurrent_limit

* chore: only load concurrency key cap setting when cloud hosted

* fix: reject queued-job import on cloud
This commit is contained in:
Ruben Fiszel
2026-07-20 12:33:40 +02:00
committed by GitHub
parent 1abfe49f7e
commit 71f2d47cb4
17 changed files with 446 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/main.rs
+28 -22
View File
@@ -40,14 +40,14 @@ use windmill_common::{
global_settings::{
AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, AUDIT_LOG_RETENTION_DAYS_SETTING,
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, BUN_INSTALL_MIN_RELEASE_AGE_SETTING,
CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING,
CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING,
DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING,
HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
CONCURRENCY_KEY_MAX_QUEUED_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING,
CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING,
ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
EXTRA_PIP_INDEX_URL_SETTING, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING,
HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING,
INDEXER_SETTING, INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING,
KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING,
@@ -122,20 +122,21 @@ use windmill_worker::{
};
use crate::monitor::{
initial_load, load_disable_password_login, load_fork_workspace_tag_append_fork_suffix,
load_keep_job_dir, load_metrics_debug_enabled, load_preview_tags_override,
load_require_preexisting_user, load_retention_period_overrides, load_tag_per_workspace_enabled,
load_tag_per_workspace_workspaces, load_workspace_fairness_duration_secs,
load_workspace_fairness_enabled, load_workspace_fairness_max_percent,
load_workspace_fairness_min_total, monitor_db, reload_app_workspaced_route_setting,
reload_audit_log_retention_days_setting, reload_base_url_setting,
reload_bun_install_min_release_age_setting, reload_bunfig_install_scopes_setting,
reload_critical_alert_mute_ui_setting, reload_critical_alerts_on_token_expiry_setting,
reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting,
reload_hub_base_url_setting, reload_instance_events_webhook_setting,
reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting,
reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting,
initial_load, load_concurrency_key_max_queued, load_disable_password_login,
load_fork_workspace_tag_append_fork_suffix, load_keep_job_dir, load_metrics_debug_enabled,
load_preview_tags_override, load_require_preexisting_user, load_retention_period_overrides,
load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces,
load_workspace_fairness_duration_secs, load_workspace_fairness_enabled,
load_workspace_fairness_max_percent, load_workspace_fairness_min_total, monitor_db,
reload_app_workspaced_route_setting, reload_audit_log_retention_days_setting,
reload_base_url_setting, reload_bun_install_min_release_age_setting,
reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting,
reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting,
reload_extra_pip_index_url_setting, reload_http_route_workspaced_route_setting,
reload_hub_api_secret_setting, reload_hub_base_url_setting,
reload_instance_events_webhook_setting, reload_job_default_timeout_setting,
reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key,
reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting,
reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting,
reload_pip_index_url_setting, reload_retention_period_setting,
reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting,
@@ -1879,6 +1880,11 @@ async fn process_notify_event(
tracing::error!("Error loading workspace fairness min total: {e:#}");
}
}
CONCURRENCY_KEY_MAX_QUEUED_SETTING => {
if let Err(e) = load_concurrency_key_max_queued(db).await {
tracing::error!("Error loading concurrency key max queued: {e:#}");
}
}
SMTP_SETTING => {
reload_smtp_config(db).await;
}
backend/src/monitor.rs
+52 -6
View File
@@ -53,11 +53,12 @@ use windmill_common::{
flow_status::{FlowStatus, FlowStatusModule},
global_settings::{
AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING,
BUN_INSTALL_MIN_RELEASE_AGE_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING,
CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN, DISABLE_PASSWORD_LOGIN_SETTING,
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
BUN_INSTALL_MIN_RELEASE_AGE_SETTING, CONCURRENCY_KEY_MAX_QUEUED_SETTING,
CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING,
CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING,
DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
DISABLE_PASSWORD_LOGIN, DISABLE_PASSWORD_LOGIN_SETTING, EXPOSE_DEBUG_METRICS_SETTING,
EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HUB_API_SECRET_SETTING,
HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
@@ -86,7 +87,8 @@ use windmill_common::{
load_env_vars, load_init_bash_from_env, load_periodic_bash_script_from_env,
load_periodic_bash_script_interval_from_env, load_whitelist_env_vars_from_env,
load_worker_config, reload_custom_tags_setting, store_pull_query,
store_suspended_pull_query, Connection, WorkerConfig, DEFAULT_TAGS_PER_WORKSPACE,
store_suspended_pull_query, Connection, WorkerConfig, CLOUD_HOSTED,
CONCURRENCY_KEY_MAX_QUEUED, CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, DEFAULT_TAGS_PER_WORKSPACE,
DEFAULT_TAGS_WORKSPACES, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, INDEXER_CONFIG,
PREVIEW_TAGS_OVERRIDE, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG,
WORKER_GROUP, WORKSPACE_FAIRNESS_DURATION_SECS, WORKSPACE_FAIRNESS_ENABLED,
@@ -291,6 +293,13 @@ pub async fn initial_load(
if let Err(e) = load_workspace_fairness_enabled(db).await {
tracing::error!("Error loading workspace fairness enabled: {e:#}");
}
// Only the cloud reads this cap, so don't spend a query loading it anywhere else.
if *CLOUD_HOSTED {
if let Err(e) = load_concurrency_key_max_queued(db).await {
tracing::error!("Error loading concurrency key max queued: {e:#}");
}
}
}
if server_mode {
@@ -610,6 +619,10 @@ const WORKSPACE_FAIRNESS_MAX_PERCENT_DEFAULT: u32 = 50;
const WORKSPACE_FAIRNESS_DURATION_SECS_DEFAULT: u32 = 10;
const WORKSPACE_FAIRNESS_MIN_TOTAL_DEFAULT: u32 = 4;
/// The cap is used as a SQL `LIMIT`, so it must survive the `u32 -> i64` widening without
/// becoming absurd; `u32::MAX` is already far beyond any queue depth worth allowing.
const CONCURRENCY_KEY_MAX_QUEUED_MAX: u64 = u32::MAX as u64;
pub async fn load_workspace_fairness_enabled(db: &DB) -> error::Result<()> {
// Match the convention used by `load_preview_tags_override` /
// `load_fork_workspace_tag_append_fork_suffix`: on transient DB errors, leave the in-memory
@@ -697,6 +710,39 @@ pub async fn load_workspace_fairness_min_total(db: &DB) -> error::Result<()> {
Ok(())
}
pub async fn load_concurrency_key_max_queued(db: &DB) -> error::Result<()> {
// See `load_workspace_fairness_max_percent` for the Err / None / invalid policy.
match load_value_from_global_settings(db, CONCURRENCY_KEY_MAX_QUEUED_SETTING).await? {
Some(serde_json::Value::Number(n)) => {
// `0` is a meaningful value here (disable the cap), so unlike the fairness knobs
// the lower bound is 0 rather than 1.
let v = n
.as_u64()
.map(|u| u.min(CONCURRENCY_KEY_MAX_QUEUED_MAX) as u32)
.unwrap_or_else(|| {
// Warn rather than silently defaulting: `-1` and `"0"` are plausible
// attempts to disable the cap, and both would otherwise land on 10000.
tracing::warn!(
"{CONCURRENCY_KEY_MAX_QUEUED_SETTING}={n} is not a non-negative integer, \
falling back to {CONCURRENCY_KEY_MAX_QUEUED_DEFAULT}. Set 0 to disable."
);
CONCURRENCY_KEY_MAX_QUEUED_DEFAULT
});
CONCURRENCY_KEY_MAX_QUEUED.store(v, Ordering::Relaxed);
}
other => {
if let Some(v) = other {
tracing::warn!(
"{CONCURRENCY_KEY_MAX_QUEUED_SETTING}={v} is not a number, falling back to \
{CONCURRENCY_KEY_MAX_QUEUED_DEFAULT}. Set 0 to disable."
);
}
CONCURRENCY_KEY_MAX_QUEUED.store(CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, Ordering::Relaxed);
}
}
Ok(())
}
pub async fn load_fork_workspace_tag_append_fork_suffix(db: &DB) -> error::Result<()> {
let value =
load_value_from_global_settings(db, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING).await;