From ab38e1418e67be8bcc37391bb21d2f86d1ca3fc6 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 10 Jul 2026 23:42:30 +0200 Subject: [PATCH] fix: keep agent-worker server job-completed processor alive & self-healing (#10033) * fix: keep agent-worker server job-completed processor alive on init-script failure The agent-worker API server's background job-completed processors relay completions on behalf of many remote agent workers. The processor loop exited (dropping its receiver) on an init-script failure, but on the server that failed init script belongs to a remote worker, not the server. Once enough processors exited, the shared completion channel disconnected and every /send_result POST returned 500, stranding completions and creating zombie-job restart loops. Add an is_agent_server flag so server relay processors don't self-terminate on init-script failure. Pins the EE companion change. Co-Authored-By: Claude Opus 4.8 (1M context) * chore: bump EE ref for send_result wait-for-processor change Co-Authored-By: Claude Opus 4.8 (1M context) * test: agent-worker server survives a failed init script End-to-end regression for the agent-worker-server processor bug: an agent worker runs a failing init script, POSTs the failed init-script completion to /send_result, and the test asserts the server's background job-completed processor stays alive (a subsequent job completes and no bg-processor critical alert is raised). Fails if the is_agent_server guard is removed (the processor breaks, the supervisor raises a critical alert). Requires --features enterprise,license,private,agent_worker_server. Co-Authored-By: Claude Opus 4.8 (1M context) * test: replace heavyweight init-script e2e with focused unit tests The panic/respawn/alert and 503 timeout paths are now covered by fast, deterministic unit tests in windmill-api-agent-workers (supervise_processor, classify_send). Drop the enterprise-only, global-config-mutating e2e in favor of those. Bump EE ref. Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to bc45d9275d4307132927dc8ad3e82049b1aed463 This commit updates the EE repository reference after PR #653 was merged in windmill-ee-private. Previous ee-repo-ref: 2aca03f28bb37e938ae548b81f1620b2e00dc0f7 New ee-repo-ref: bc45d9275d4307132927dc8ad3e82049b1aed463 Automated by sync-ee-ref workflow. * chore: bump EE ref for bg-processor alert rate-limiting Picks up windmill-ee-private#654: exponential backoff + rate-limited critical alerts in supervise_processor, addressing the code-review nit. Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to d48c0e01e8601a372353c032dd237ddb6fa3bbad This commit updates the EE repository reference after PR #654 was merged in windmill-ee-private. Previous ee-repo-ref: f89eeb6e333614850ef650e7df78e3c2335f107c New ee-repo-ref: d48c0e01e8601a372353c032dd237ddb6fa3bbad Automated by sync-ee-ref workflow. * chore: bump EE ref for graceful-shutdown-during-backoff fix Picks up windmill-ee-private#655: supervise_processor re-checks shutdown before respawn and selects on the shutdown broadcast during backoff, so a crash-loop backoff can't hang graceful shutdown. Addresses the Codex P1. Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to 9bc5dfb9ce73a2d9b981a1de86eea6aa26688b79 This commit updates the EE repository reference after PR #655 was merged in windmill-ee-private. Previous ee-repo-ref: 8dc3b3d9ec8f9c28b227d36c2a1327b4b2017665 New ee-repo-ref: 9bc5dfb9ce73a2d9b981a1de86eea6aa26688b79 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/tests/agent_workers.rs | 2 +- .../windmill-worker/src/result_processor.rs | 24 ++++++++++++++++--- backend/windmill-worker/src/worker.rs | 1 + 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index fdd2ea689e..cc29239893 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -e2df172596e00877068d4b0a98afaef62fe429d1 +9bc5dfb9ce73a2d9b981a1de86eea6aa26688b79 diff --git a/backend/tests/agent_workers.rs b/backend/tests/agent_workers.rs index 6ec3f9e410..791556b035 100644 --- a/backend/tests/agent_workers.rs +++ b/backend/tests/agent_workers.rs @@ -22,7 +22,7 @@ fn bun_code(code: &str) -> RawCode { .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), modules: None, - tag: None, + tag: None, } } diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 26fbdb21af..f2f36621a5 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -300,6 +300,12 @@ pub fn start_background_processor( worker_name: String, killpill_tx: KillpillSender, is_dedicated_worker: bool, + // True when this processor runs inside the agent-worker API server, relaying + // completions on behalf of many remote agent workers. Such a processor must + // never kill itself: dropping its receiver would disconnect the shared + // job-completed channel and make every future /send_result fail until the + // whole server is restarted. + is_agent_server: bool, stats_map: JobStatsMap, ) -> JoinHandle<()> { tokio::spawn(async move { @@ -376,6 +382,7 @@ pub fn start_background_processor( jc.job.kind, JobKind::Dependencies | JobKind::FlowDependencies ); + let jc_id = jc.job.id; #[cfg(feature = "benchmark")] let bench_job_id = jc.job.id; #[cfg(feature = "benchmark")] @@ -403,9 +410,20 @@ pub fn start_background_processor( .await; if is_init_script && !final_success { - tracing::error!("init script errored, exiting"); - killpill_tx.send(); - break; + if is_agent_server { + // The failed init script belongs to a remote agent + // worker, not to this server. That worker handles its + // own restart; killing the server relay here would + // strand every other agent worker's completions. + tracing::error!( + job_id = %jc_id, + "agent worker init script errored; failure recorded, keeping server bg processor alive" + ); + } else { + tracing::error!("init script errored, exiting"); + killpill_tx.send(); + break; + } } if is_dependency_job && is_dedicated_worker { tracing::error!("Dedicated worker executed a dependency job, a new script has been deployed. Exiting expecting to be restarted."); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 2335a78453..a4672145bb 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -2269,6 +2269,7 @@ pub async fn run_worker( worker_name.clone(), killpill_tx.clone(), is_dedicated_worker, + false, stats_map, )), _ => None,