fix(monitor): cleanup stale server_heartbeat background_task_state rows (#9338)

`announce_server_started` writes a `server_heartbeat:{INSTANCE_NAME}` row
on each startup. INSTANCE_NAME is a fresh random string per process, so
the row is never updated again and a new row is inserted on every
restart, growing background_task_state unboundedly.

Add an hourly monitor task that deletes server_heartbeat:* rows older
than 7 days. Older rows cannot influence check_any_server_started (which
only considers heartbeats refreshed after the restart was initiated), so
they are safe to prune.

Fixes WIN-1990.

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/.sqlx/query-8597cd40f80e69edbf1bc7d7402baca32e33e871be454acb5175c11361fe1b0a.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM background_task_state\n         WHERE name LIKE $1\n           AND updated_at < NOW() - INTERVAL '7 days'",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8597cd40f80e69edbf1bc7d7402baca32e33e871be454acb5175c11361fe1b0a"
+}

backend/src/monitor.rs

@@ -2705,6 +2705,26 @@ pub async fn monitor_db(
         }
     };
 
+    // run every hour (120 iterations * 30s = 3600s)
+    let cleanup_stale_server_heartbeats_f = async {
+        if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(120) {
+            if let Some(db) = conn.as_sql() {
+                match windmill_api::cleanup_stale_server_heartbeats(db).await {
+                    Ok(count) if count > 0 => {
+                        tracing::info!(
+                            "Deleted {} stale server_heartbeat background_task_state rows",
+                            count
+                        );
+                    }
+                    Err(e) => {
+                        tracing::error!("Error cleaning up stale server_heartbeat rows: {:?}", e);
+                    }
+                    _ => {}
+                }
+            }
+        }
+    };
+
     // run every hour (120 iterations * 30s = 3600s)
     let manage_audit_partitions_f = async {
         if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(120) {
@@ -2763,6 +2783,7 @@ pub async fn monitor_db(
         native_triggers_sync_f,
         cleanup_notify_events_f,
         check_expiring_tokens_f,
+        cleanup_stale_server_heartbeats_f,
         manage_audit_partitions_f,
         export_audit_logs_to_object_store_f,
         cleanup_scheduled_job_deletions_f,

backend/windmill-api/src/lib.rs

@@ -1261,3 +1261,26 @@ pub async fn check_any_server_started(db: &DB, not_before: chrono::DateTime<chro
     .await
     .unwrap_or(false)
 }
+
+/// Delete `server_heartbeat:*` rows that have not been refreshed in a long
+/// time. Each server startup generates a fresh random `INSTANCE_NAME` and
+/// inserts a new row keyed by `server_heartbeat:{instance}`; because that
+/// row is only written once (on startup) and never updated thereafter, the
+/// table grows by one row per server restart and is otherwise never pruned.
+///
+/// The row is only consulted by `check_any_server_started`, which itself
+/// filters on `updated_at > not_before` (the moment a restart was initiated),
+/// so rows older than the cutoff cannot influence any restart decision and
+/// are safe to delete.
+pub async fn cleanup_stale_server_heartbeats(db: &DB) -> anyhow::Result<u64> {
+    let prefix = format!("{SERVER_HEARTBEAT_TASK}:");
+    let res = sqlx::query!(
+        "DELETE FROM background_task_state
+         WHERE name LIKE $1
+           AND updated_at < NOW() - INTERVAL '7 days'",
+        format!("{prefix}%"),
+    )
+    .execute(db)
+    .await?;
+    Ok(res.rows_affected())
+}