starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-25 16:02:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add instance setting to mute zombie job restart alerts (#10813)

Browse Source 
* feat: add instance setting to opt out of zombie job restart alerts

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* fix: preserve explicit false for default-on boolean instance settings

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* refactor: invert zombie restart alert setting to a mute flag

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
hugocasa
2026-08-24 22:36:21 +02:00
committed by GitHub
parent 93081e255f
commit 2906504125
5 changed files with 90 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/main.rs
+21 -12
View File
@@ -42,12 +42,13 @@ use windmill_common::{
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, BUN_INSTALL_MIN_RELEASE_AGE_SETTING,
CONCURRENCY_KEY_MAX_QUEUED_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING,
CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING,
ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
EXTRA_PIP_INDEX_URL_SETTING, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING,
HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING,
INDEXER_SETTING, INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING, CRITICAL_ERROR_CHANNELS_SETTING,
CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING,
HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING,
KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING,
@@ -131,12 +132,13 @@ use crate::monitor::{
load_workspace_max_queued_jobs, monitor_db, reload_app_workspaced_route_setting,
reload_audit_log_retention_days_setting, reload_base_url_setting,
reload_bun_install_min_release_age_setting, reload_bunfig_install_scopes_setting,
reload_critical_alert_mute_ui_setting, reload_critical_alerts_on_token_expiry_setting,
reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting,
reload_hub_base_url_setting, reload_instance_events_webhook_setting,
reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting,
reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting,
reload_critical_alert_mute_ui_setting, reload_critical_alert_mute_zombie_job_restart_setting,
reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting,
reload_extra_pip_index_url_setting, reload_http_route_workspaced_route_setting,
reload_hub_api_secret_setting, reload_hub_base_url_setting,
reload_instance_events_webhook_setting, reload_job_default_timeout_setting,
reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key,
reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting,
reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting,
reload_pip_index_url_setting, reload_retention_period_setting,
reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting,
@@ -2127,6 +2129,13 @@ async fn process_notify_event(
tracing::error!(error = %e, "Could not reload critical alerts on token expiry setting");
}
}
CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING => {
if let Err(e) =
reload_critical_alert_mute_zombie_job_restart_setting(conn).await
{
tracing::error!(error = %e, "Could not reload zombie job restart alert mute setting");
}
}
INSTANCE_EVENTS_WEBHOOK_SETTING => {
reload_instance_events_webhook_setting(db).await;
}
backend/src/monitor.rs
+57 -19
View File
@@ -52,10 +52,10 @@ use windmill_common::{
flow_status::{FlowStatus, FlowStatusModule},
global_settings::{
get_or_create_jwt_secret, load_value_from_global_settings,
AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING,
BUNFIG_INSTALL_SCOPES_SETTING, BUN_INSTALL_MIN_RELEASE_AGE_SETTING,
CONCURRENCY_KEY_MAX_QUEUED_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING,
CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING,
BUN_INSTALL_MIN_RELEASE_AGE_SETTING, CONCURRENCY_KEY_MAX_QUEUED_SETTING,
CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING,
CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING,
CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN, DISABLE_PASSWORD_LOGIN_SETTING,
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
@@ -87,18 +87,18 @@ use windmill_common::{
worker::{
load_env_vars, load_init_bash_from_env, load_periodic_bash_script_from_env,
load_periodic_bash_script_interval_from_env, load_whitelist_env_vars_from_env,
load_worker_config, store_pull_query,
store_suspended_pull_query, Connection, WorkerConfig, CLOUD_HOSTED,
CONCURRENCY_KEY_MAX_QUEUED, CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, DEFAULT_TAGS_PER_WORKSPACE,
DEFAULT_TAGS_WORKSPACES, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, INDEXER_CONFIG,
PREVIEW_TAGS_OVERRIDE, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG,
load_worker_config, store_pull_query, store_suspended_pull_query, Connection, WorkerConfig,
CLOUD_HOSTED, CONCURRENCY_KEY_MAX_QUEUED, CONCURRENCY_KEY_MAX_QUEUED_DEFAULT,
DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX,
INDEXER_CONFIG, PREVIEW_TAGS_OVERRIDE, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG,
WORKER_GROUP, WORKSPACE_FAIRNESS_DURATION_SECS, WORKSPACE_FAIRNESS_ENABLED,
WORKSPACE_FAIRNESS_MAX_PERCENT, WORKSPACE_FAIRNESS_MIN_TOTAL, WORKSPACE_MAX_QUEUED_JOBS,
WORKSPACE_MAX_QUEUED_JOBS_DEFAULT,
},
KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE,
CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB,
DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES,
CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED,
CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL,
HUB_BASE_URL, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES,
JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED,
SERVICE_LOG_RETENTION_SECS, STORE_AUDIT_LOGS_S3,
@@ -273,6 +273,11 @@ pub async fn initial_load(
true,
|v| async move { apply_critical_alerts_on_token_expiry_setting(v) },
);
pass.setting(
CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING,
true,
|v| async move { apply_critical_alert_mute_zombie_job_restart_setting(v) },
);
if let Some(db) = conn.as_sql() {
pass.setting(DEFAULT_TAGS_PER_WORKSPACE_SETTING, false, |v| async move {
@@ -1070,6 +1075,31 @@ pub fn apply_critical_alerts_on_token_expiry_setting(value: Option<serde_json::V
}
}
pub async fn reload_critical_alert_mute_zombie_job_restart_setting(
conn: &Connection,
) -> error::Result<()> {
let v = load_value_from_global_settings_with_conn(
conn,
CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING,
true,
)
.await?;
apply_critical_alert_mute_zombie_job_restart_setting(v);
Ok(())
}
pub fn apply_critical_alert_mute_zombie_job_restart_setting(value: Option<serde_json::Value>) {
match value {
Some(serde_json::Value::Bool(t)) => {
CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART.store(t, Ordering::Relaxed)
}
// Deleting the row must un-mute: keeping the last value would leave an instance
// silently muted until the next restart.
None => CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART.store(false, Ordering::Relaxed),
_ => (),
};
}
pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> {
let v =
load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await?;
@@ -5184,13 +5214,18 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, node_n
.execute(db)
.await;
tracing::error!(critical_error_message);
report_critical_error(
critical_error_message,
db.clone(),
Some(&r.workspace_id),
None,
)
.await;
// A restart that still has attempts left is self-healing, so an operator can mute it on
// an instance with flaky workers. Exhausting the attempts is a real failure and always
// alerts.
if !restart || !CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART.load(Ordering::Relaxed) {
report_critical_error(
critical_error_message,
db.clone(),
Some(&r.workspace_id),
None,
)
.await;
}
if !restart {
zombie_jobs_uuid_restart_limit_reached.push(r.id);
@@ -5661,7 +5696,10 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
flow.id, flow.workspace_id
);
tracing::error!(error_message);
report_critical_error(error_message, db.clone(), Some(&flow.workspace_id), None).await;
if !CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART.load(Ordering::Relaxed) {
report_critical_error(error_message, db.clone(), Some(&flow.workspace_id), None)
.await;
}
// if the flow hasn't started and is a zombie, we can simply restart it
let mut tx = db.begin().await?;
backend/windmill-common/src/global_settings.rs
+2
View File
@@ -100,6 +100,8 @@ pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels";
pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui";
pub const CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING: &str = "critical_alerts_on_db_oversize";
pub const CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING: &str = "critical_alerts_on_token_expiry";
pub const CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING: &str =
"critical_alert_mute_zombie_job_restart";
pub const DEV_INSTANCE_SETTING: &str = "dev_instance";
pub const JWT_SECRET_SETTING: &str = "jwt_secret";
pub const EMAIL_DOMAIN_SETTING: &str = "email_domain";
backend/windmill-common/src/lib.rs
+1
View File
@@ -354,6 +354,7 @@ lazy_static::lazy_static! {
pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false);
pub static ref CRITICAL_ALERTS_ON_TOKEN_EXPIRY: AtomicBool = AtomicBool::new(false);
pub static ref CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART: AtomicBool = AtomicBool::new(false);
pub static ref BASE_URL: arc_swap::ArcSwap<String> = arc_swap::ArcSwap::from_pointee("".to_string());
pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
frontend/src/lib/components/instanceSettings.ts
+9
View File
@@ -914,6 +914,15 @@ export const settings: Record<string, Setting[]> = {
storage: 'setting',
ee_only: ''
},
{
label: 'Mute zombie job restart alerts',
description:
'Stop sending critical alerts when a zombie job or flow is detected and automatically restarted. Jobs that exhaust all their restart attempts, and flows cancelled after hanging between steps, keep alerting.',
key: 'critical_alert_mute_zombie_job_restart',
fieldType: 'boolean',
storage: 'setting',
ee_only: ''
},
{
label: 'Slack',
key: 'slack',