feat(frontend): critical alerts UI (#4653)

backend/.sqlx/query-00ce4ed3ca0eac7cb6283b047353a64b9e78c4beb423f04baef9a53fbf87e9f9.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO alerts (alert_type, message) VALUES ('recovered_critical_error', $1)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "00ce4ed3ca0eac7cb6283b047353a64b9e78c4beb423f04baef9a53fbf87e9f9"
-}

backend/.sqlx/query-0b955f2cff82a2d4ba3840588143e08952f029480d4a42503ecc3c5e70437995.json

@@ -0,0 +1,48 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id, alert_type, message, created_at, acknowledged \n             FROM alerts \n             WHERE acknowledged = $1\n             ORDER BY created_at DESC \n             LIMIT $2 OFFSET $3",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 1,
+        "name": "alert_type",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "message",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "created_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 4,
+        "name": "acknowledged",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Bool",
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      true
+    ]
+  },
+  "hash": "0b955f2cff82a2d4ba3840588143e08952f029480d4a42503ecc3c5e70437995"
+}

backend/.sqlx/query-33cfb00f61dbae09467bd3732ce8e83c9835ccd30c90cb24788cdb54d1ceb575.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value::BOOLEAN as value FROM global_settings WHERE name = 'critical_alert_mute_ui'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "33cfb00f61dbae09467bd3732ce8e83c9835ccd30c90cb24788cdb54d1ceb575"
+}

backend/.sqlx/query-3a94ad52c6b7cde844fa868167248cd9ff63e5fdfa1d93d8fbec32a257b6b05e.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('recovered_critical_error', $1, $2)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "3a94ad52c6b7cde844fa868167248cd9ff63e5fdfa1d93d8fbec32a257b6b05e"
+}

backend/.sqlx/query-4c0067c2135a259aea5cc2db60f7375a9a33671be8ef406427d90f67a98c9c9f.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO alerts (alert_type, message) VALUES ('critical_error', $1)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "4c0067c2135a259aea5cc2db60f7375a9a33671be8ef406427d90f67a98c9c9f"
-}

backend/.sqlx/query-4d22084a5d9860832f30e8f08cbfa1848ed3c1336fa4790f45b8189c4ac97d91.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('critical_error', $1, $2)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "4d22084a5d9860832f30e8f08cbfa1848ed3c1336fa4790f45b8189c4ac97d91"
+}

backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-a7c5008aa7ea43d0afac7d9f19846976ed7af2e90270902f001115e023cb947d.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE alerts SET acknowledged = true WHERE acknowledged = false",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "a7c5008aa7ea43d0afac7d9f19846976ed7af2e90270902f001115e023cb947d"
+}

backend/.sqlx/query-be3ae231557e794172336bc27d725f862dcf039bbb3d75ced9d54c86f53d2580.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE alerts SET acknowledged = true WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "be3ae231557e794172336bc27d725f862dcf039bbb3d75ced9d54c86f53d2580"
+}

backend/.sqlx/query-cc5ab80241b88c5befea279f16c4ec68cec17b31dcd277b321f652917346496b.json

@@ -0,0 +1,47 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id, alert_type, message, created_at, acknowledged \n             FROM alerts \n             ORDER BY created_at DESC \n             LIMIT $1 OFFSET $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 1,
+        "name": "alert_type",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "message",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "created_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 4,
+        "name": "acknowledged",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      true
+    ]
+  },
+  "hash": "cc5ab80241b88c5befea279f16c4ec68cec17b31dcd277b321f652917346496b"
+}

backend/migrations/20241105225151_alert_add_acknowledged.down.sql

@@ -0,0 +1 @@
+ALTER TABLE alerts DROP COLUMN acknowledged;

backend/migrations/20241105225151_alert_add_acknowledged.up.sql

@@ -0,0 +1,9 @@
+-- Step 1: Add the new column 'acknowledged' to the 'alerts' table
+ALTER TABLE alerts
+ADD COLUMN acknowledged BOOLEAN DEFAULT NULL;
+
+-- Step 2: Update all existing rows to set 'acknowledged' to true
+-- we don't want to pop up notifications to all users after migrations
+-- but only show new alerts from the point of the upgrade
+UPDATE alerts
+SET acknowledged = TRUE;

backend/src/main.rs

@@ -36,7 +36,7 @@ use windmill_common::{
         ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
         EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
         JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING,
-        OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
@@ -78,7 +78,7 @@ use crate::monitor::{
     reload_hub_base_url_setting, reload_job_default_timeout_setting, reload_jwt_secret_setting,
     reload_license_key, reload_npm_config_registry_setting, reload_pip_index_url_setting,
     reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config,
-    reload_worker_config,
+    reload_worker_config, reload_critical_alert_mute_ui_setting,
 };
 
 #[cfg(feature = "parquet")]
@@ -817,6 +817,12 @@ Windmill Community Edition {GIT_VERSION}
                                                         tracing::error!(error = %e, "Could not reload jwt secret setting");
                                                     }
                                                 },
+                                                CRITICAL_ALERT_MUTE_UI_SETTING => {
+                                                    tracing::info!("Critical alert UI setting changed");
+                                                    if let Err(e) = reload_critical_alert_mute_ui_setting(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload critical alert UI setting");
+                                                    }
+                                                },
                                                 a @_ => {
                                                     tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a);
                                                 }

backend/src/monitor.rs

@@ -39,7 +39,7 @@ use windmill_common::{
         EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
         HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING,
         KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
-        PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
@@ -55,7 +55,7 @@ use windmill_common::{
         WORKER_GROUP,
     },
     BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS,
-    METRICS_DEBUG_ENABLED, METRICS_ENABLED,
+    METRICS_DEBUG_ENABLED, METRICS_ENABLED, CRITICAL_ALERT_MUTE_UI_ENABLED
 };
 use windmill_queue::cancel_job;
 use windmill_worker::{
@@ -131,6 +131,10 @@ pub async fn initial_load(
         tracing::error!("Error loading expose debug metrics: {e:#}");
     }
 
+    if let Err(e) = reload_critical_alert_mute_ui_setting(db).await {
+        tracing::error!("Error loading critical alert mute ui setting: {e:#}");
+    }
+
     if let Err(e) = load_tag_per_workspace_enabled(db).await {
         tracing::error!("Error loading default tag per workpsace: {e:#}");
     }
@@ -226,6 +230,17 @@ pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> {
     Ok(())
 }
 
+pub async fn reload_critical_alert_mute_ui_setting(db: &DB) -> error::Result<()> {
+    let mute = load_value_from_global_settings(db, CRITICAL_ALERT_MUTE_UI_SETTING).await;
+    match mute {
+        Ok(Some(serde_json::Value::Bool(t))) => {
+            CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed);
+        }
+        _ => (),
+    };
+    Ok(())
+}
+
 pub async fn load_metrics_debug_enabled(db: &DB) -> error::Result<()> {
     let metrics_enabled = load_value_from_global_settings(db, EXPOSE_DEBUG_METRICS_SETTING).await;
     match metrics_enabled {

backend/windmill-api/openapi.yaml

@@ -773,6 +773,78 @@ paths:
               schema:
                 type: string
 
+  /settings/critical_alerts:
+    get:
+      summary: Get all critical alerts
+      operationId: getCriticalAlerts
+      tags:
+        - setting
+      parameters:
+        - in: query
+          name: page
+          schema:
+            type: integer
+            default: 1
+            description: The page number to retrieve (minimum value is 1)
+        - in: query
+          name: page_size
+          schema:
+            type: integer
+            default: 10
+            maximum: 100
+            description: Number of alerts per page (maximum is 100)
+        - in: query
+          name: acknowledged
+          schema:
+            type: boolean
+            nullable: true
+            description: Filter by acknowledgment status; true for acknowledged, false for unacknowledged, and omit for all alerts
+      responses:
+        "200":
+          description: Successfully retrieved all critical alerts
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/CriticalAlert'
+
+  /settings/critical_alerts/{id}/acknowledge:
+    post:
+      summary: Acknowledge a critical alert
+      operationId: acknowledgeCriticalAlert
+      tags:
+        - setting
+      parameters:
+        - in: path
+          name: id
+          required: true
+          schema:
+            type: integer
+          description: The ID of the critical alert to acknowledge
+      responses:
+        "200":
+          description: Successfully acknowledged the critical alert
+          content:
+            application/json:
+              schema:
+                type: string
+                example: "Critical alert acknowledged"
+
+  /settings/critical_alerts/acknowledge_all:
+    post:
+      summary: Acknowledge all unacknowledged critical alerts
+      operationId: acknowledgeAllCriticalAlerts
+      tags:
+        - setting
+      responses:
+        "200":
+          description: Successfully acknowledged all unacknowledged critical alerts.
+          content:
+            application/json:
+              schema:
+                type: string
+                example: "All unacknowledged critical alerts acknowledged"
 
   /settings/test_license_key:
     post:
@@ -12721,3 +12793,23 @@ components:
           type: string
           format: date-time
 
+    CriticalAlert:
+      type: object
+      properties:
+        id:
+          type: integer
+          description: Unique identifier for the alert
+        alert_type:
+          type: string
+          description: Type of alert (e.g., critical_error)
+        message:
+          type: string
+          description: The message content of the alert
+        created_at:
+          type: string
+          format: date-time
+          description: Time when the alert was created
+        acknowledged:
+          type: boolean
+          nullable: true
+          description: Acknowledgment status of the alert, can be true, false, or null if not set

backend/windmill-api/src/settings.rs

@@ -58,7 +58,10 @@ pub fn global_service() -> Router {
         )
         .route("/renew_license_key", post(renew_license_key))
         .route("/customer_portal", post(create_customer_portal_session))
-        .route("/test_critical_channels", post(test_critical_channels));
+        .route("/test_critical_channels", post(test_critical_channels))
+        .route("/critical_alerts", get(get_critical_alerts))
+        .route("/critical_alerts/:id/acknowledge", post(acknowledge_critical_alert))
+        .route("/critical_alerts/acknowledge_all", post(acknowledge_all_critical_alerts));
 
     #[cfg(feature = "parquet")]
     {
@@ -430,3 +433,117 @@ pub async fn test_critical_channels(
 pub async fn test_critical_channels() -> Result<String> {
     Ok("Critical channels require EE".to_string())
 }
+
+use serde::Serialize;
+
+#[derive(Serialize)]
+pub struct CriticalAlert {
+    id: i32,
+    alert_type: String,
+    message: String,
+    created_at: chrono::DateTime<chrono::Utc>,
+    acknowledged: Option<bool>,
+}
+
+#[cfg(feature = "enterprise")]
+#[derive(Deserialize)]
+pub struct AlertQueryParams {
+    pub page: Option<i32>,
+    pub page_size: Option<i32>,
+    pub acknowledged: Option<bool>,
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn get_critical_alerts(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Query(params): Query<AlertQueryParams>,
+) -> JsonResult<Vec<CriticalAlert>> {
+    require_super_admin(&db, &authed.email).await?;
+
+    // Default pagination values if not provided
+    let page = params.page.unwrap_or(1).max(1);
+    let page_size = params.page_size.unwrap_or(10).min(100) as i64;
+    let offset = ((page - 1) * page_size as i32) as i64;
+
+    let alerts = if let Some(acknowledged) = params.acknowledged {
+        sqlx::query_as!(
+            CriticalAlert,
+            "SELECT id, alert_type, message, created_at, acknowledged 
+             FROM alerts 
+             WHERE acknowledged = $1
+             ORDER BY created_at DESC 
+             LIMIT $2 OFFSET $3",
+            acknowledged,
+            page_size,
+            offset
+        )
+        .fetch_all(&db)
+        .await?
+    } else {
+        sqlx::query_as!(
+            CriticalAlert,
+            "SELECT id, alert_type, message, created_at, acknowledged 
+             FROM alerts 
+             ORDER BY created_at DESC 
+             LIMIT $1 OFFSET $2",
+            page_size,
+            offset
+        )
+        .fetch_all(&db)
+        .await?
+    };
+
+    Ok(Json(alerts))
+}
+
+#[cfg(not(feature = "enterprise"))]
+pub async fn get_critical_alerts() -> error::Error {
+    error::Error::NotFound("Critical Alerts require EE".to_string())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn acknowledge_critical_alert(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Path(id): Path<i32>,
+) -> error::Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+    
+    sqlx::query!(
+        "UPDATE alerts SET acknowledged = true WHERE id = $1",
+        id
+    )
+    .execute(&db)
+    .await?;
+    
+    tracing::info!("Acknowledged critical alert with id: {}", id);
+    Ok("Critical alert acknowledged".to_string())
+}
+
+#[cfg(not(feature = "enterprise"))]
+pub async fn acknowledge_critical_alert() -> error::Error {
+    error::Error::NotFound("Critical Alerts require EE".to_string())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn acknowledge_all_critical_alerts(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+) -> error::Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+
+    sqlx::query!(
+        "UPDATE alerts SET acknowledged = true WHERE acknowledged = false"
+    )
+    .execute(&db)
+    .await?;
+
+    tracing::info!("Acknowledged all unacknowledged critical alerts");
+    Ok("All unacknowledged critical alerts acknowledged".to_string())
+}
+
+#[cfg(not(feature = "enterprise"))]
+pub async fn acknowledge_all_critical_alerts() -> error::Error {
+    error::Error::NotFound("Critical Alerts require EE".to_string())
+}

backend/windmill-common/src/global_settings.rs

@@ -29,6 +29,7 @@ pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation
 pub const HUB_BASE_URL_SETTING: &str = "hub_base_url";
 pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url";
 pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels";
+pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui";
 pub const DEV_INSTANCE_SETTING: &str = "dev_instance";
 pub const JWT_SECRET_SETTING: &str = "jwt_secret";
 pub const EMAIL_DOMAIN_SETTING: &str = "email_domain";

backend/windmill-common/src/lib.rs

@@ -86,6 +86,8 @@ lazy_static::lazy_static! {
     pub static ref METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("METRICS_PORT").is_ok() || std::env::var("METRICS_ADDR").is_ok());
     pub static ref METRICS_DEBUG_ENABLED: AtomicBool = AtomicBool::new(false);
 
+    pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false);
+
     pub static ref BASE_URL: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
     pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
 

backend/windmill-common/src/utils.rs

@@ -28,6 +28,9 @@ pub const DEFAULT_PER_PAGE: usize = 1000;
 pub const GIT_VERSION: &str =
     git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 
+use std::sync::atomic::Ordering;
+use crate::CRITICAL_ALERT_MUTE_UI_ENABLED;
+
 lazy_static::lazy_static! {
     pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
         .user_agent("windmill/beta")
@@ -244,9 +247,12 @@ pub fn generate_lock_id(database_name: &str) -> i64 {
 pub async fn report_critical_error(error_message: String, _db: DB) -> () {
     tracing::error!("CRITICAL ERROR: {error_message}");
 
+    let mute = CRITICAL_ALERT_MUTE_UI_ENABLED.load(Ordering::Relaxed);
+
     if let Err(err) = sqlx::query!(
-        "INSERT INTO alerts (alert_type, message) VALUES ('critical_error', $1)",
-        error_message
+        "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('critical_error', $1, $2)",
+        error_message,
+        mute
     )
     .execute(&_db)
     .await
@@ -261,9 +267,12 @@ pub async fn report_critical_error(error_message: String, _db: DB) -> () {
 pub async fn report_recovered_critical_error(message: String, _db: DB) -> () {
     tracing::info!("RECOVERED CRITICAL ERROR: {message}");
 
+    let mute = CRITICAL_ALERT_MUTE_UI_ENABLED.load(Ordering::Relaxed);
+
     if let Err(err) = sqlx::query!(
-        "INSERT INTO alerts (alert_type, message) VALUES ('recovered_critical_error', $1)",
-        message
+        "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('recovered_critical_error', $1, $2)",
+        message,
+        mute
     )
     .execute(&_db)
     .await

frontend/src/lib/components/SuperadminSettings.svelte

@@ -23,6 +23,7 @@
 	import { isCloudHosted } from '$lib/cloud'
 	import InstanceNameEditor from './InstanceNameEditor.svelte'
 	import Toggle from './Toggle.svelte'
+	import { instanceSettingsSelectedTab } from '$lib/stores';
 	let drawer: Drawer
 	let filter = ''
 
@@ -56,6 +57,9 @@
 
 	let tab: 'users' | string = 'users'
 
+	$: $instanceSettingsSelectedTab, tab = $instanceSettingsSelectedTab
+	$: tab, instanceSettingsSelectedTab.set(tab)
+
 	let nbDisplayed = 50
 
 	let instanceSettings

frontend/src/lib/components/instanceSettings.ts

@@ -146,6 +146,15 @@ export const settings: Record<string, Setting[]> = {
 			storage: 'setting',
 			ee_only: 'Channels other than tracing are only available in the EE version'
 		},
+		{
+			label: 'Mute critical alerts in UI',
+			description: 'Enable to mute critical alerts in the UI',
+			key: 'critical_alert_mute_ui',
+			fieldType: 'boolean',
+			storage: 'setting',
+			requiresReloadOnChange: true,
+			ee_only: 'Critical alerts in UI are only available in the EE version'
+		},
 		{
 			label: 'Azure OpenAI base path',
 			description:

frontend/src/lib/components/sidebar/CriticalAlertButton.svelte

@@ -0,0 +1,64 @@
+<script lang="ts">
+	import { twMerge } from 'tailwind-merge'
+	import Popover from '../Popover.svelte'
+	import { createEventDispatcher } from 'svelte'
+
+	export let label: string | undefined = undefined
+	export let numUnacknowledgedCriticalAlerts: number
+	export let isCollapsed: boolean
+	export let disabled: boolean = false
+	export let lightMode: boolean = false
+	export let stopPropagationOnClick: boolean = false
+	export let shortcut: string = ''
+
+	let dispatch = createEventDispatcher()
+</script>
+
+{#if !disabled}
+	<Popover appearTimeout={0} disappearTimeout={0} class="w-full" disablePopup={!isCollapsed}>
+		<div class="py-1.5 px-1 border-t border-gray-700">
+		<button
+			on:click={(e) => {
+				if (stopPropagationOnClick) e.preventDefault()
+				dispatch('click')
+			}}
+			class={twMerge(
+				'group flex items-center px-2 py-2 font-light rounded-md h-8 gap-3 w-full',
+				lightMode
+					? 'text-primary hover:bg-surface-hover '
+					: '  hover:bg-[#2A3648] text-primary-inverse dark:text-primary',
+				'transition-all',
+				$$props.class
+			)}
+			title={label}
+		>
+			<span
+				class="flex items-center justify-center h-6 w-6 rounded-lg bg-red-600 text-white font-bold text-xs"
+			>
+				{numUnacknowledgedCriticalAlerts > 9 ? '9+' : numUnacknowledgedCriticalAlerts}
+			</span>
+
+			{#if !isCollapsed && label}
+				<span
+					class={twMerge(
+						'whitespace-pre truncate',
+						lightMode ? 'text-primary' : 'text-primary-inverse  dark:text-primary',
+						'transition-all',
+						$$props.class
+					)}
+				>
+					{label}
+					<span class="pl-2 text-xs dark:text-secondary light:text-secondary-inverse font-semibold">
+						{shortcut}
+					</span>
+				</span>
+			{/if}
+		</button>
+	</div>
+		<svelte:fragment slot="text">
+			{#if label}
+				{label}
+			{/if}
+		</svelte:fragment>
+	</Popover>
+{/if}

frontend/src/lib/components/sidebar/CriticalAlertModal.svelte

@@ -0,0 +1,69 @@
+<script lang="ts">
+	import { onMount, onDestroy } from 'svelte'
+	import CriticalAlertModalInner from './CriticalAlertModalInner.svelte'
+	import { SettingService } from '$lib/gen'
+	import { sendUserToast } from '$lib/toast'
+
+	export let open: boolean = false
+	export let numUnacknowledgedCriticalAlerts: number = 0
+
+	let checkForNewAlertsInterval: ReturnType<typeof setInterval>
+	let checkingForNewAlerts = false
+
+	onMount(() => {
+		updateHasUnacknowledgedCriticalAlerts(true)
+		checkForNewAlertsInterval = setInterval(() => {
+			updateHasUnacknowledgedCriticalAlerts(true)
+		}, 15000)
+	})
+
+	onDestroy(() => {
+		clearInterval(checkForNewAlertsInterval)
+	})
+
+	async function updateHasUnacknowledgedCriticalAlerts(sendToast: boolean = false) {
+		if (checkingForNewAlerts) return
+		checkingForNewAlerts = true
+
+		try {
+			const unacknowledged = await SettingService.getCriticalAlerts({
+				page: 1,
+				pageSize: 10,
+				acknowledged: false
+			})
+
+			if (numUnacknowledgedCriticalAlerts === 0 && unacknowledged.length > 0 && sendToast) {
+				sendUserToast(
+					'Critical Alert:',
+					true,
+					[
+						{
+							label: 'View',
+							callback: () => {
+								open = true
+							}
+						},
+						{
+							label: 'Acknowledge',
+							callback: () => {
+								if (unacknowledged[0].id) acknowledgeAlert(unacknowledged[0].id)
+							}
+						}
+					],
+					unacknowledged[0].message,
+					10000
+				)
+			}
+			numUnacknowledgedCriticalAlerts = unacknowledged.length
+		} finally {
+			checkingForNewAlerts = false
+		}
+	}
+
+	async function acknowledgeAlert(id: number) {
+		await SettingService.acknowledgeCriticalAlert({ id })
+		updateHasUnacknowledgedCriticalAlerts()
+	}
+</script>
+
+<CriticalAlertModalInner bind:open {updateHasUnacknowledgedCriticalAlerts} />

frontend/src/lib/components/sidebar/CriticalAlertModalInner.svelte

@@ -0,0 +1,213 @@
+<script lang="ts">
+	import Modal from '../common/modal/Modal.svelte'
+	import Button from '../common/button/Button.svelte'
+	import Toggle from '$lib/components/Toggle.svelte'
+	import { SettingService } from '$lib/gen'
+	import { CheckCircle2, AlertCircle, RefreshCw, CheckSquare2, AlertTriangle } from 'lucide-svelte'
+	import type { CriticalAlert } from '$lib/gen'
+	import { onMount, beforeUpdate } from 'svelte'
+	import { instanceSettingsSelectedTab } from '$lib/stores'
+	import { goto } from '$app/navigation'
+
+	export let open: boolean = false
+	export let updateHasUnacknowledgedCriticalAlerts: () => void = () => {}
+
+	let alerts: CriticalAlert[] = []
+
+	let isRefreshing = false
+	let previousOpen = open
+	let hasCriticalAlertChannels = false
+
+	$: loading = isRefreshing
+
+	onMount(() => {
+		if (open) refreshAlerts()
+	})
+
+	beforeUpdate(() => {
+		if (open && !previousOpen) {
+			refreshAlerts()
+		}
+		previousOpen = open
+	})
+
+	// Pagination
+	let page = 1
+	let pageSize = 10
+	let hasMore = true
+
+	let hideAcknowledged = false
+
+	async function acknowledgeAll() {
+		await SettingService.acknowledgeAllCriticalAlerts()
+		getAlerts(false)
+	}
+
+	async function fetchAlerts(pageNumber: number) {
+		isRefreshing = true
+		try {
+			const newAlerts = await SettingService.getCriticalAlerts({
+				page: pageNumber,
+				pageSize: pageSize,
+				acknowledged: hideAcknowledged ? false : undefined
+			})
+
+			alerts = newAlerts
+			hasMore = newAlerts.length === pageSize
+			page = pageNumber
+			updateHasUnacknowledgedCriticalAlerts()
+		} finally {
+			setTimeout(() => {
+				isRefreshing = false
+			}, 500)
+		}
+	}
+
+	async function getAlerts(reset?: boolean) {
+		if (reset) page = 1
+		await fetchAlerts(page)
+	}
+
+	async function checkCriticalAlertChannels() {
+		const channels = (await SettingService.getGlobal({ key: 'critical_error_channels' })) as any[]
+		hasCriticalAlertChannels = channels && channels.length > 0
+	}
+
+	async function acknowledgeAlert(id: number) {
+		await SettingService.acknowledgeCriticalAlert({ id })
+		getAlerts(false)
+	}
+
+	function formatDate(dateString: string | undefined): string {
+		if (!dateString) return ''
+		const date = new Date(dateString)
+		return new Intl.DateTimeFormat('en-US', {
+			year: 'numeric',
+			month: 'short',
+			day: 'numeric',
+			hour: '2-digit',
+			minute: '2-digit',
+			second: '2-digit'
+		}).format(date)
+	}
+
+	async function refreshAlerts() {
+		checkCriticalAlertChannels()
+		await getAlerts(true)
+	}
+
+	function goToPreviousPage() {
+		if (page > 1) {
+			fetchAlerts(page - 1)
+		}
+	}
+
+	function goToNextPage() {
+		if (hasMore) {
+			fetchAlerts(page + 1)
+		}
+	}
+
+	function goToCoreTab() {
+		goto('/#superadmin-settings')
+		instanceSettingsSelectedTab.set('Core')
+	}
+</script>
+
+<Modal bind:open title="Critical Alerts" cancelText="Close" style="max-width: 66%;">
+	{#if !hasCriticalAlertChannels}
+		<div class="flex flex-row pb-4">
+			<AlertTriangle color="orange" class="w-6 h-6 mr-2" />
+			<p>
+				No critical alert channels are set up. Go to the
+				<a href="/#superadmin-settings" on:click|preventDefault={goToCoreTab}>Instance Settings</a>
+				page to configure critical alert channels.
+			</p>
+		</div>
+	{/if}
+	<!-- Row of action buttons above the table -->
+	<div class="flex justify-between items-center mb-4">
+		<div class="flex space-x-2">
+			<Button color="green" startIcon={{ icon: CheckSquare2 }} size="sm" on:click={acknowledgeAll}
+				>Acknowledge All</Button
+			>
+		</div>
+
+		<button class="p-2 rounded-full hover:bg-gray-200" on:click={refreshAlerts} disabled={loading}>
+			<RefreshCw class={loading ? 'animate-spin ' : ''} size="20" />
+		</button>
+	</div>
+
+	<!-- Pagination controls above the table -->
+	<div class="flex justify-between items-center mb-2">
+		<div class="flex items-center space-x-4">
+			<Button size="xs2" on:click={goToPreviousPage} disabled={page <= 1}>Previous</Button>
+			<span>Page {page}</span>
+			<Button size="xs2" on:click={goToNextPage} disabled={!hasMore}>Next</Button>
+		</div>
+		<div class="pr-3">
+			<Toggle
+				bind:checked={hideAcknowledged}
+				on:change={refreshAlerts}
+				options={{ right: 'Hide Acknowledged' }}
+				size="xs"
+			/>
+		</div>
+	</div>
+
+	<!-- Table of alerts with scrollable body -->
+	<div class="overflow-y-auto max-h-1/2">
+		<table class="min-w-full w-full">
+			<thead class="bg-gray-600 text-white sticky top-0 z-10">
+				<tr>
+					<th class="w-[60px] px-4 py-2 text-center">Type</th>
+					<th class="px-4 py-2 text-center">Message</th>
+					<th class="w-[150px] px-4 py-2 text-center">Created At</th>
+					<th class="w-[180px] px-4 py-2 text-center">Acknowledge</th>
+				</tr>
+			</thead>
+			<tbody>
+				{#each alerts as { id, alert_type, message, created_at, acknowledged }}
+					{#if !hideAcknowledged || !acknowledged}
+						<tr class="bg-gray-100 dark:bg-gray-700 dark:text-white text-center">
+							<td class="border px-4 py-2 w-[100px]">
+								{#if alert_type === 'recovered_critical_error'}
+									<span title="Recovered Critical Alert">
+										<CheckCircle2 size="20" color="green" />
+									</span>
+								{:else}
+									<span title="Critical Alert">
+										<AlertCircle size="20" color="red" />
+									</span>
+								{/if}
+							</td>
+							<td class="border px-4 py-2">{message}</td>
+							<!-- Flexible width -->
+							<td class="border px-4 py-2 w-[150px]">{formatDate(created_at)}</td>
+							<td class="border px-4 py-2 w-[180px]">
+								<div class="flex justify-center items-center">
+									{#if !acknowledged}
+										<Button
+											color="green"
+											startIcon={{ icon: CheckSquare2 }}
+											size="xs2"
+											on:click={() => {
+												if (id) acknowledgeAlert(id)
+											}}>Acknowledge</Button
+										>
+									{:else}
+										<CheckCircle2 size="20" color="green" />
+									{/if}
+								</div>
+							</td>
+						</tr>
+					{/if}
+				{/each}
+			</tbody>
+		</table>
+	</div>
+
+	{#if alerts.length === 0}
+		<p class="text-center text-gray-500 mt-4">No critical alerts available.</p>
+	{/if}
+</Modal>

frontend/src/lib/stores.ts

@@ -121,3 +121,5 @@ export type DBSchema = SQLSchema | GraphqlSchema
 export type DBSchemas = Partial<Record<string, DBSchema>>
 
 export const dbSchemas = writable<DBSchemas>({})
+
+export const instanceSettingsSelectedTab = writable('Core')

frontend/src/routes/(root)/(logged)/+layout.svelte

@@ -14,7 +14,9 @@
 	import { classNames, getModifierKey } from '$lib/utils'
 	import WorkspaceMenu from '$lib/components/sidebar/WorkspaceMenu.svelte'
 	import SidebarContent from '$lib/components/sidebar/SidebarContent.svelte'
+	import CriticalAlertModal from '$lib/components/sidebar/CriticalAlertModal.svelte'
 	import {
+		enterpriseLicense,
 		copilotInfo,
 		isPremiumStore,
 		starStore,
@@ -48,6 +50,7 @@
 	import MenuButton from '$lib/components/sidebar/MenuButton.svelte'
 	import { setContext } from 'svelte'
 	import { base } from '$app/paths'
+	import CriticalAlertButton from '$lib/components/sidebar/CriticalAlertButton.svelte'
 
 	OpenAPI.WITH_CREDENTIALS = true
 	let menuOpen = false
@@ -275,6 +278,26 @@
 	}
 
 	setContext('openSearchWithPrefilledText', openSearchModal)
+
+	$: {
+		if ($enterpriseLicense && $superadmin) {
+			loadCriticalAlertsMuted()
+		}
+	}
+
+	let numUnacknowledgedCriticalAlerts = 0
+	let isCriticalAlertsModalOpen = false
+	let isCriticalAlertsUiMuted = false
+
+	async function loadCriticalAlertsMuted() {
+		isCriticalAlertsUiMuted = (await SettingService.getGlobal({
+			key: 'critical_alert_mute_ui'
+		})) as boolean
+	}
+
+	function openCriticalAlertsModal(text?: string): void {
+		isCriticalAlertsModalOpen = true
+	}
 </script>
 
 <svelte:window bind:innerWidth />
@@ -291,6 +314,12 @@
 {:else if $userStore}
 	<GlobalSearchModal bind:this={globalSearchModal} />
 	{#if $superadmin}
+		{#if !isCriticalAlertsUiMuted && $enterpriseLicense}
+			<CriticalAlertModal
+				bind:open={isCriticalAlertsModalOpen}
+				bind:numUnacknowledgedCriticalAlerts
+			/>
+		{/if}
 		<SuperadminSettings bind:this={superadminSettings} />
 	{/if}
 	<div>
@@ -356,7 +385,6 @@
 										<WindmillIcon white={true} height="20px" width="20px" />
 										Windmill
 									</div>
-
 									<div class="px-2 py-4 space-y-2 border-y border-gray-500">
 										<WorkspaceMenu />
 										<FavoriteMenu {favoriteLinks} />
@@ -405,6 +433,17 @@
 									{/if}
 								</div>
 							</button>
+							{#if $superadmin && $enterpriseLicense}
+								<CriticalAlertButton
+									stopPropagationOnClick={true}
+									on:click={() => openCriticalAlertsModal()}
+									{numUnacknowledgedCriticalAlerts}
+									{isCollapsed}
+									label="Critical Alerts"
+									class="!text-xs"
+									disabled={numUnacknowledgedCriticalAlerts === 0}
+								/>
+							{/if}
 							<div class="px-2 py-4 space-y-2 border-y border-gray-700">
 								<WorkspaceMenu {isCollapsed} />
 								<FavoriteMenu {favoriteLinks} {isCollapsed} />