mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-17 08:20:12 +00:00
fix(worker): wait up to 3 min for migrations before exiting (#12316)
* fix(worker): wait up to 15 min for the server to migrate instead of 30s During a rolling upgrade new workers start alongside the new server, which needs longer than 10 x 3s to come up and run migrations. Workers exited, restarted and tripped crash loop alerts on every Cloud EE upgrade. Wait up to 15 minutes, and log progress every 30s instead of every 3s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PjsYvvEoe9TQtBwGumWBKo * fix(worker): cap the migration wait at 3 min, from observed rollout data Over the last 12 Cloud EE rollouts (3,430 worker starts) the median wait was 0s, p99 47s and the longest 94s, so 15 minutes was far more than needed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PjsYvvEoe9TQtBwGumWBKo --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
466885fd26
commit
a2685ef718
@@ -484,7 +484,16 @@ pub async fn wait_for_migrations(
|
||||
"This worker is on Windmill version {GIT_VERSION} (migration version {latest_version}). Only servers run migrations. Waiting for a server with version >= {GIT_VERSION} to apply the migration..."
|
||||
);
|
||||
|
||||
let mut attempts = 0;
|
||||
// During a rolling upgrade, new workers start at the same time as the new
|
||||
// server, which needs a while to come up and migrate. Exiting early only
|
||||
// makes the pod restart and look like a crash loop, so wait long enough to
|
||||
// cover a normal rollout and only give up if the server is truly not coming.
|
||||
const RECHECK_SECS: u64 = 3;
|
||||
// Over 12 rollouts (3,430 worker starts) the longest wait was 94s and
|
||||
// p99 47s; 3 minutes covers that with margin.
|
||||
const MAX_WAIT_SECS: u64 = 3 * 60;
|
||||
const LOG_EVERY: u64 = 10;
|
||||
let mut attempts: u64 = 0;
|
||||
|
||||
loop {
|
||||
let is_applied: Result<Option<bool>, sqlx::Error> =
|
||||
@@ -500,16 +509,19 @@ pub async fn wait_for_migrations(
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Ok(_) => {
|
||||
Ok(_) if attempts % LOG_EVERY == 0 => {
|
||||
tracing::info!(
|
||||
"Database not up to date yet. This worker (Windmill {GIT_VERSION}, migration version {latest_version}) is waiting for a server with version >= {GIT_VERSION} to run the migration. Rechecking in 3s..."
|
||||
"Database not up to date yet. This worker (Windmill {GIT_VERSION}, migration version {latest_version}) is waiting for a server with version >= {GIT_VERSION} to run the migration. Waited {}s so far, giving up after {MAX_WAIT_SECS}s...",
|
||||
attempts * RECHECK_SECS
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
Ok(_) => {}
|
||||
Err(e) if attempts % LOG_EVERY == 0 => {
|
||||
tracing::info!(
|
||||
"Could not check migration status (migrations table may not exist yet): {e:#}. Rechecking in 3s..."
|
||||
"Could not check migration status (migrations table may not exist yet): {e:#}. Rechecking every {RECHECK_SECS}s..."
|
||||
);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
|
||||
tokio::select! {
|
||||
@@ -517,13 +529,13 @@ pub async fn wait_for_migrations(
|
||||
tracing::info!("Killpill received, stopping migration wait");
|
||||
return Ok(());
|
||||
}
|
||||
_ = tokio::time::sleep(std::time::Duration::from_secs(3)) => {}
|
||||
_ = tokio::time::sleep(std::time::Duration::from_secs(RECHECK_SECS)) => {}
|
||||
}
|
||||
|
||||
attempts += 1;
|
||||
if attempts >= 10 {
|
||||
if attempts * RECHECK_SECS >= MAX_WAIT_SECS {
|
||||
tracing::error!(
|
||||
"Timed out after 10 attempts waiting for migration version {latest_version}. Exiting."
|
||||
"Timed out after {MAX_WAIT_SECS}s waiting for migration version {latest_version}. Exiting."
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user