From cf81330fbc8ecbfc720f0cafaf6b1ec57793928f Mon Sep 17 00:00:00 2001
From: Alexey Kondratov <kondratov.aleksey@gmail.com>
Date: Thu, 22 May 2025 21:15:05 +0200
Subject: [PATCH] fix(compute_ctl): Wait for rsyslog longer and with backoff
 (#12002)

## Problem

https://github.com/neondatabase/neon/pull/11988 waits only for max
~200ms, so we still see failures, which self-resolve after several
operation retries.

## Summary of changes

Change it to waiting for at least 5 seconds, starting with 2 ms sleep
between iterations and x2 sleep on each next iteration. It could be that
it's not a problem with a slow `rsyslog` start, but a longer wait won't
hurt. If it won't start, we should debug why `inittab` doesn't start it,
or maybe there is another problem.
---
 compute_tools/src/rsyslog.rs | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/compute_tools/src/rsyslog.rs b/compute_tools/src/rsyslog.rs
index 23036e9234..c873697623 100644
--- a/compute_tools/src/rsyslog.rs
+++ b/compute_tools/src/rsyslog.rs
@@ -28,20 +28,37 @@ fn get_rsyslog_pid() -> Option<String> {
 }
 
 fn wait_for_rsyslog_pid() -> Result<String, anyhow::Error> {
-    for attempt in 1..=50 {
+    const MAX_WAIT: Duration = Duration::from_secs(5);
+    const INITIAL_SLEEP: Duration = Duration::from_millis(2);
+
+    let mut sleep_duration = INITIAL_SLEEP;
+    let start = std::time::Instant::now();
+    let mut attempts = 1;
+
+    for attempt in 1.. {
+        attempts = attempt;
         match get_rsyslog_pid() {
             Some(pid) => return Ok(pid),
             None => {
+                if start.elapsed() >= MAX_WAIT {
+                    break;
+                }
                 info!(
-                    "rsyslogd is not running, attempt {}/50. Waiting...",
-                    attempt
+                    "rsyslogd is not running, attempt {}. Sleeping for {} ms",
+                    attempt,
+                    sleep_duration.as_millis()
                 );
-                std::thread::sleep(std::time::Duration::from_millis(2));
+                std::thread::sleep(sleep_duration);
+                sleep_duration *= 2;
             }
         }
     }
 
-    Err(anyhow::anyhow!("rsyslogd did not start after 50 attempts"))
+    Err(anyhow::anyhow!(
+        "rsyslogd is not running after waiting for {} seconds and {} attempts",
+        attempts,
+        start.elapsed().as_secs()
+    ))
 }
 
 // Restart rsyslogd to apply the new configuration.