fix(health): detect read-only replica via pg_is_in_recovery() (#9722)

The /api/health/status database check used `SELECT 1`, which succeeds
even on a read-only standby. After a PostgreSQL failover where the
primary becomes a secondary, the health check kept reporting healthy
while all writes failed with "cannot execute INSERT in a read-only
transaction", so Kubernetes liveness probes never restarted the pod.

Use `SELECT NOT pg_is_in_recovery()` instead: it returns true on a
primary and false on a standby, so a read-only replica is now reported
unhealthy. Result handling checks the returned bool (Ok(Some(true)))
rather than just query success.

Fixes WIN-2085

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/.sqlx/query-282b56cfb8504312ac586cd4c1f3f914cf1651c08b8edd1b06d9a6454ed779bf.json

@@ -1,12 +1,12 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT 1",
+  "query": "SELECT NOT pg_is_in_recovery()",
   "describe": {
     "columns": [
       {
         "ordinal": 0,
         "name": "?column?",
-        "type_info": "Int4"
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -16,5 +16,5 @@
       null
     ]
   },
-  "hash": "e004ebd5b5532a4b85984a62f8ad48a81aa3460c1ca07701f386135d72cdecf5"
+  "hash": "282b56cfb8504312ac586cd4c1f3f914cf1651c08b8edd1b06d9a6454ed779bf"
 }

backend/windmill-api/src/health.rs

@@ -219,12 +219,16 @@ struct DatabaseCheckResult {
 
 async fn check_database_with_latency(db: &DB) -> DatabaseCheckResult {
     let start = std::time::Instant::now();
+    // `pg_is_in_recovery()` is true on standbys/read-only replicas, so a primary
+    // returns true here. A read-only replica (e.g. after a failover where the
+    // primary became a secondary) reports unhealthy, letting liveness probes
+    // restart the pod instead of silently failing all writes.
     let healthy = tokio::time::timeout(
         HEALTH_CHECK_TIMEOUT,
-        sqlx::query_scalar!("SELECT 1").fetch_one(db),
+        sqlx::query_scalar!("SELECT NOT pg_is_in_recovery()").fetch_one(db),
     )
     .await
-    .map(|r| r.is_ok())
+    .map(|r| matches!(r, Ok(Some(true))))
     .unwrap_or(false);
     let latency_ms = start.elapsed().as_millis() as i64;