From e16061df06babeae935a9396de5bdcd46e8119a9 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@windmill.dev>
Date: Tue, 23 Jun 2026 09:37:29 +0200
Subject: [PATCH] fix(health): detect read-only replica via pg_is_in_recovery()
 (#9722)

The /api/health/status database check used `SELECT 1`, which succeeds
even on a read-only standby. After a PostgreSQL failover where the
primary becomes a secondary, the health check kept reporting healthy
while all writes failed with "cannot execute INSERT in a read-only
transaction", so Kubernetes liveness probes never restarted the pod.

Use `SELECT NOT pg_is_in_recovery()` instead: it returns true on a
primary and false on a standby, so a read-only replica is now reported
unhealthy. Result handling checks the returned bool (Ok(Some(true)))
rather than just query success.

Fixes WIN-2085

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 ...ac586cd4c1f3f914cf1651c08b8edd1b06d9a6454ed779bf.json} | 6 +++---
 backend/windmill-api/src/health.rs                        | 8 ++++++--
 2 files changed, 9 insertions(+), 5 deletions(-)
 rename backend/.sqlx/{query-e004ebd5b5532a4b85984a62f8ad48a81aa3460c1ca07701f386135d72cdecf5.json => query-282b56cfb8504312ac586cd4c1f3f914cf1651c08b8edd1b06d9a6454ed779bf.json} (59%)

diff --git a/backend/.sqlx/query-e004ebd5b5532a4b85984a62f8ad48a81aa3460c1ca07701f386135d72cdecf5.json b/backend/.sqlx/query-282b56cfb8504312ac586cd4c1f3f914cf1651c08b8edd1b06d9a6454ed779bf.json
similarity index 59%
rename from backend/.sqlx/query-e004ebd5b5532a4b85984a62f8ad48a81aa3460c1ca07701f386135d72cdecf5.json
rename to backend/.sqlx/query-282b56cfb8504312ac586cd4c1f3f914cf1651c08b8edd1b06d9a6454ed779bf.json
index 0769d083d6..4e5f9f6ed7 100644
--- a/backend/.sqlx/query-e004ebd5b5532a4b85984a62f8ad48a81aa3460c1ca07701f386135d72cdecf5.json
+++ b/backend/.sqlx/query-282b56cfb8504312ac586cd4c1f3f914cf1651c08b8edd1b06d9a6454ed779bf.json
@@ -1,12 +1,12 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT 1",
+  "query": "SELECT NOT pg_is_in_recovery()",
   "describe": {
     "columns": [
       {
         "ordinal": 0,
         "name": "?column?",
-        "type_info": "Int4"
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -16,5 +16,5 @@
       null
     ]
   },
-  "hash": "e004ebd5b5532a4b85984a62f8ad48a81aa3460c1ca07701f386135d72cdecf5"
+  "hash": "282b56cfb8504312ac586cd4c1f3f914cf1651c08b8edd1b06d9a6454ed779bf"
 }
diff --git a/backend/windmill-api/src/health.rs b/backend/windmill-api/src/health.rs
index fab9979c12..4dc8947de6 100644
--- a/backend/windmill-api/src/health.rs
+++ b/backend/windmill-api/src/health.rs
@@ -219,12 +219,16 @@ struct DatabaseCheckResult {
 
 async fn check_database_with_latency(db: &DB) -> DatabaseCheckResult {
     let start = std::time::Instant::now();
+    // `pg_is_in_recovery()` is true on standbys/read-only replicas, so a primary
+    // returns true here. A read-only replica (e.g. after a failover where the
+    // primary became a secondary) reports unhealthy, letting liveness probes
+    // restart the pod instead of silently failing all writes.
     let healthy = tokio::time::timeout(
         HEALTH_CHECK_TIMEOUT,
-        sqlx::query_scalar!("SELECT 1").fetch_one(db),
+        sqlx::query_scalar!("SELECT NOT pg_is_in_recovery()").fetch_one(db),
     )
     .await
-    .map(|r| r.is_ok())
+    .map(|r| matches!(r, Ok(Some(true))))
     .unwrap_or(false);
     let latency_ms = start.elapsed().as_millis() as i64;