diff --git a/backend/src/main.rs b/backend/src/main.rs index 1819146cbe..10d1434f98 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -42,12 +42,13 @@ use windmill_common::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, BUN_INSTALL_MIN_RELEASE_AGE_SETTING, CONCURRENCY_KEY_MAX_QUEUED_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, - CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, - ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, - EXTRA_PIP_INDEX_URL_SETTING, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, - HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, - INDEXER_SETTING, INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING, + CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, + CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, + DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, + EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, + FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, + HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, + INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, @@ -131,12 +132,13 @@ use crate::monitor::{ load_workspace_max_queued_jobs, monitor_db, reload_app_workspaced_route_setting, reload_audit_log_retention_days_setting, reload_base_url_setting, reload_bun_install_min_release_age_setting, reload_bunfig_install_scopes_setting, - reload_critical_alert_mute_ui_setting, reload_critical_alerts_on_token_expiry_setting, - reload_critical_error_channels_setting, reload_extra_pip_index_url_setting, - reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting, - reload_hub_base_url_setting, reload_instance_events_webhook_setting, - reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, - reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, + reload_critical_alert_mute_ui_setting, reload_critical_alert_mute_zombie_job_restart_setting, + reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting, + reload_extra_pip_index_url_setting, reload_http_route_workspaced_route_setting, + reload_hub_api_secret_setting, reload_hub_base_url_setting, + reload_instance_events_webhook_setting, reload_job_default_timeout_setting, + reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, + reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, reload_retention_period_setting, reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting, @@ -2127,6 +2129,13 @@ async fn process_notify_event( tracing::error!(error = %e, "Could not reload critical alerts on token expiry setting"); } } + CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING => { + if let Err(e) = + reload_critical_alert_mute_zombie_job_restart_setting(conn).await + { + tracing::error!(error = %e, "Could not reload zombie job restart alert mute setting"); + } + } INSTANCE_EVENTS_WEBHOOK_SETTING => { reload_instance_events_webhook_setting(db).await; } diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 94dcbf0d46..430f3659c3 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -52,10 +52,10 @@ use windmill_common::{ flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ get_or_create_jwt_secret, load_value_from_global_settings, - AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, - BUNFIG_INSTALL_SCOPES_SETTING, BUN_INSTALL_MIN_RELEASE_AGE_SETTING, - CONCURRENCY_KEY_MAX_QUEUED_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, - CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, + AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, + BUN_INSTALL_MIN_RELEASE_AGE_SETTING, CONCURRENCY_KEY_MAX_QUEUED_SETTING, + CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN, DISABLE_PASSWORD_LOGIN_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, @@ -87,18 +87,18 @@ use windmill_common::{ worker::{ load_env_vars, load_init_bash_from_env, load_periodic_bash_script_from_env, load_periodic_bash_script_interval_from_env, load_whitelist_env_vars_from_env, - load_worker_config, store_pull_query, - store_suspended_pull_query, Connection, WorkerConfig, CLOUD_HOSTED, - CONCURRENCY_KEY_MAX_QUEUED, CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, DEFAULT_TAGS_PER_WORKSPACE, - DEFAULT_TAGS_WORKSPACES, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, INDEXER_CONFIG, - PREVIEW_TAGS_OVERRIDE, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG, + load_worker_config, store_pull_query, store_suspended_pull_query, Connection, WorkerConfig, + CLOUD_HOSTED, CONCURRENCY_KEY_MAX_QUEUED, CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, + DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, + INDEXER_CONFIG, PREVIEW_TAGS_OVERRIDE, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG, WORKER_GROUP, WORKSPACE_FAIRNESS_DURATION_SECS, WORKSPACE_FAIRNESS_ENABLED, WORKSPACE_FAIRNESS_MAX_PERCENT, WORKSPACE_FAIRNESS_MIN_TOTAL, WORKSPACE_MAX_QUEUED_JOBS, WORKSPACE_MAX_QUEUED_JOBS_DEFAULT, }, KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, - CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, - DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, + CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, + CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, + HUB_BASE_URL, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS, STORE_AUDIT_LOGS_S3, @@ -273,6 +273,11 @@ pub async fn initial_load( true, |v| async move { apply_critical_alerts_on_token_expiry_setting(v) }, ); + pass.setting( + CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING, + true, + |v| async move { apply_critical_alert_mute_zombie_job_restart_setting(v) }, + ); if let Some(db) = conn.as_sql() { pass.setting(DEFAULT_TAGS_PER_WORKSPACE_SETTING, false, |v| async move { @@ -1070,6 +1075,31 @@ pub fn apply_critical_alerts_on_token_expiry_setting(value: Option error::Result<()> { + let v = load_value_from_global_settings_with_conn( + conn, + CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING, + true, + ) + .await?; + apply_critical_alert_mute_zombie_job_restart_setting(v); + Ok(()) +} + +pub fn apply_critical_alert_mute_zombie_job_restart_setting(value: Option) { + match value { + Some(serde_json::Value::Bool(t)) => { + CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART.store(t, Ordering::Relaxed) + } + // Deleting the row must un-mute: keeping the last value would leave an instance + // silently muted until the next restart. + None => CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART.store(false, Ordering::Relaxed), + _ => (), + }; +} + pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> { let v = load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await?; @@ -5184,13 +5214,18 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n .execute(db) .await; tracing::error!(critical_error_message); - report_critical_error( - critical_error_message, - db.clone(), - Some(&r.workspace_id), - None, - ) - .await; + // A restart that still has attempts left is self-healing, so an operator can mute it on + // an instance with flaky workers. Exhausting the attempts is a real failure and always + // alerts. + if !restart || !CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART.load(Ordering::Relaxed) { + report_critical_error( + critical_error_message, + db.clone(), + Some(&r.workspace_id), + None, + ) + .await; + } if !restart { zombie_jobs_uuid_restart_limit_reached.push(r.id); @@ -5661,7 +5696,10 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { flow.id, flow.workspace_id ); tracing::error!(error_message); - report_critical_error(error_message, db.clone(), Some(&flow.workspace_id), None).await; + if !CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART.load(Ordering::Relaxed) { + report_critical_error(error_message, db.clone(), Some(&flow.workspace_id), None) + .await; + } // if the flow hasn't started and is a zombie, we can simply restart it let mut tx = db.begin().await?; diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 37b4b60da4..b539c1c6b3 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -100,6 +100,8 @@ pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; pub const CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING: &str = "critical_alerts_on_db_oversize"; pub const CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING: &str = "critical_alerts_on_token_expiry"; +pub const CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING: &str = + "critical_alert_mute_zombie_job_restart"; pub const DEV_INSTANCE_SETTING: &str = "dev_instance"; pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index e7cb368d25..89db0ee769 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -354,6 +354,7 @@ lazy_static::lazy_static! { pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false); pub static ref CRITICAL_ALERTS_ON_TOKEN_EXPIRY: AtomicBool = AtomicBool::new(false); + pub static ref CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART: AtomicBool = AtomicBool::new(false); pub static ref BASE_URL: arc_swap::ArcSwap = arc_swap::ArcSwap::from_pointee("".to_string()); pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 70121a1181..6c66af7d84 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -914,6 +914,15 @@ export const settings: Record = { storage: 'setting', ee_only: '' }, + { + label: 'Mute zombie job restart alerts', + description: + 'Stop sending critical alerts when a zombie job or flow is detected and automatically restarted. Jobs that exhaust all their restart attempts, and flows cancelled after hanging between steps, keep alerting.', + key: 'critical_alert_mute_zombie_job_restart', + fieldType: 'boolean', + storage: 'setting', + ee_only: '' + }, { label: 'Slack', key: 'slack',