From b5bd70b2a54e8459052239d59be99c1905ff80e1 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 2 Oct 2026 20:46:37 +0200 Subject: [PATCH] fix: leave a zombie for the next sweep when its scopes cannot be read Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014FHGE3ynoAu6yrgeg4kLwL --- backend/src/monitor.rs | 15 ++++++++++----- backend/windmill-queue/src/jobs.rs | 2 ++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 357d22202c..4de51ca00a 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -6199,16 +6199,21 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n let memory_peak = job.memory_peak.unwrap_or(0); let (_, killpill_rx_never_used) = KillpillSender::new(1); // Read while the job is still queued: a re-run (perpetual, retry) takes its cap from - // here once the `job_perms` row is swept after completion. - let job_token_scopes = sqlx::query_scalar!( + // here once the `job_perms` row is swept after completion. A failed read leaves the + // job for the next sweep rather than completing it with no cap. + let job_token_scopes = match sqlx::query_scalar!( "SELECT job_token_scopes FROM job_perms WHERE job_id = $1", job.id ) .fetch_optional(db) .await - .ok() - .flatten() - .flatten(); + { + Ok(scopes) => scopes.flatten(), + Err(e) => { + tracing::error!("Could not read the token scopes of zombie job {}: {e:#}", job.id); + continue; + } + }; let mut completed = windmill_queue::MiniCompletedJob::from(job); completed.job_token_scopes = job_token_scopes; let _ = handle_job_error( diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index c2792d4921..787abaca40 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -3744,6 +3744,8 @@ impl From for MiniCompletedJob { // `QueuedJobV2` carries no args, and nothing reaches the restart gate // through this conversion — the worker completes jobs from the pulled job. build_binary_only: false, + // Nor scopes: a caller whose completion can re-run the job fills them in from + // `job_perms` while the job is still queued. job_token_scopes: None, } }