From 801f96fc8ef697c6e5b7cb6c7f969d76a64e14af Mon Sep 17 00:00:00 2001 From: Shrinath Rao Date: Tue, 11 Aug 2026 12:18:19 +0530 Subject: [PATCH] ci(reaper): label zombie runs distinctly from runner-wait (#13745) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The Stuck-Run-Reaper conflated **two different situations** and described both as "waiting for an EKS runner / freed a runner": 1. **runner-wait** โ€” a job queued with no EKS runner assigned (the real target). 2. **zombie run** โ€” a run left in a non-completed state with **0 jobs ever dispatched**. It never held a runner. A real example: a **2026-05-26** run stuck ~**1842h** with 0 jobs was cancelled and announced as *"freed a runner stuck for 1842 hours"* โ€” which is nonsense; nothing was ever running. ## Fix โ€” label the two distinctly | Case | Threshold | Wording | |---|---|---| | runner-wait (job queued, no runner) | **60 min** (unchanged) | "waiting for an EKS runner that was never assigned" | | zombie run (0 jobs dispatched) | **24h** (new `DEFAULT_ZOMBIE_RUN_THRESHOLD_HOURS` / `zombie_run_threshold_hours` input) | "๐ŸงŸ abandoned in a non-completed state with no jobs ever dispatched โ€” **held no runner**, stale-run cleanup" | - **Cancellation behaviour is unchanged** โ€” zombies are still swept; they're just described accurately. - Separate longer threshold so a brief GitHub scheduling blip isn't labelled a zombie. - Wait time for zombies uses `run.created_at`; `humanize()` now renders days (`76d 18h`, not `1842h 0m`). - Job-summary table gains a **Reason** column; PR comment wording branches per kind. ## Validation - YAML parses; embedded `github-script` body passes `node --check` (async-wrapped as github-script runs it). - OSS/ENT files differ only in the `WORKFLOWS` list + label comment (verified by diff). - Companion PR on the same branch `test/reaper-zombie-run-labeling` in the sister repo. --- .github/workflows/stuck-run-reaper.yml | 149 +++++++++++++++++-------- 1 file changed, 102 insertions(+), 47 deletions(-) diff --git a/.github/workflows/stuck-run-reaper.yml b/.github/workflows/stuck-run-reaper.yml index 4274166c51..b848df9aab 100644 --- a/.github/workflows/stuck-run-reaper.yml +++ b/.github/workflows/stuck-run-reaper.yml @@ -1,10 +1,13 @@ name: Playwright e2e Stuck-Run-Reaper -# Cancels CI runs whose jobs are stuck waiting for a self-hosted EKS runner -# that is never assigned. GitHub's `timeout-minutes` only counts EXECUTION time -# (it starts once a job runs), so it cannot catch a job that never gets a runner. -# This reaper enforces a wait-for-runner cap and explains every cancellation on -# the associated pull request. +# Cancels CI runs stuck against the self-hosted EKS runners, in two distinct cases: +# 1. runner-wait โ€” a job is queued but no EKS runner is ever assigned. GitHub's +# `timeout-minutes` only counts EXECUTION time, so it cannot catch a job that +# never gets a runner. Reaped after the (short) runner-wait threshold. +# 2. zombie run โ€” a run left in a non-completed state with NO jobs ever dispatched. +# It never held a runner; swept after a separate, much longer threshold. +# Every cancellation is explained (on the PR when there is one) with wording +# specific to the case, so a cancellation is never misread as a blocked runner. on: schedule: @@ -20,6 +23,10 @@ on: description: "Minutes a job may wait for a runner before being reaped" type: string default: "60" + zombie_run_threshold_hours: + description: "Hours a run may sit with no jobs dispatched before being swept as a zombie" + type: string + default: "24" concurrency: group: stuck-run-reaper @@ -45,6 +52,10 @@ jobs: # Release workflows are intentionally excluded โ€” a stuck release build is left for a human. WORKFLOWS: "playwright.yml,playwright_regression.yml" DEFAULT_THRESHOLD_MINUTES: "60" + # Separate, much longer cap for "zombie" runs stuck with no jobs ever dispatched + # (these never held a runner, so 60m would be too aggressive โ€” a brief GitHub + # scheduling blip could look like one). Only genuinely abandoned runs are swept. + DEFAULT_ZOMBIE_RUN_THRESHOLD_HOURS: "24" # Only reap jobs whose requested runner label starts with this prefix. RUNNER_LABEL_PREFIX: "eks-" with: @@ -58,14 +69,22 @@ jobs: return; } const thresholdMs = threshold * 60 * 1000; + const zombieHours = parseInt(inputs.zombie_run_threshold_hours || process.env.DEFAULT_ZOMBIE_RUN_THRESHOLD_HOURS, 10); + if (isNaN(zombieHours) || zombieHours <= 0) { + core.setFailed(`Invalid zombie_run_threshold_hours: "${inputs.zombie_run_threshold_hours}". Provide a positive integer number of hours.`); + return; + } + const zombieMs = zombieHours * 60 * 60 * 1000; const workflows = process.env.WORKFLOWS.split(",").map(s => s.trim()).filter(Boolean); const labelPrefix = process.env.RUNNER_LABEL_PREFIX; const now = Date.now(); const humanize = (ms) => { const totalMin = Math.max(0, Math.floor(ms / 60000)); - const h = Math.floor(totalMin / 60); + const d = Math.floor(totalMin / 1440); + const h = Math.floor((totalMin % 1440) / 60); const m = totalMin % 60; + if (d > 0) return `${d}d ${h}h`; return h > 0 ? `${h}h ${m}m` : `${m}m`; }; const iso = (s) => s @@ -139,30 +158,43 @@ jobs: }); } - // --- Run-level fallback: run never created any jobs, past threshold --- + // --- Run-level (zombie) fallback: a run stuck in a non-completed state with + // NO jobs ever dispatched. It never held a runner, so it is judged against the + // separate, much longer zombie threshold and labelled as stale-run cleanup โ€” + // NOT runner-wait โ€” so the cancellation is not misread as "a runner was blocked + // for N hours" (which would be nonsense โ€” nothing was ever running). --- if (jobs.length === 0 && ["queued", "pending", "waiting", "requested"].includes(run.status)) { - const sinceStr = run.run_started_at || run.created_at; + const sinceStr = run.created_at || run.run_started_at; const waited = now - Date.parse(sinceStr); - if (waited > thresholdMs) { + if (waited > zombieMs) { eligible.push({ name: "(run never dispatched any jobs)", - runner: "n/a โ€” no jobs were scheduled", + runner: "n/a โ€” no jobs scheduled, no runner held", sinceStr, waited, - kind: "run", + kind: "zombie", }); } } if (eligible.length === 0) continue; + // Classify the reap so wording matches reality: a genuine runner-wait + // (jobs queued, no runner assigned) vs a zombie run (no jobs ever dispatched, + // never held a runner). Per run the eligible set is one kind or the other. + const isZombie = eligible.every(e => e.kind === "zombie"); + const reason = isZombie + ? "๐ŸงŸ stale run โ€” 0 jobs dispatched, held no runner" + : "โณ job waited for an EKS runner"; + const maxWait = humanize(Math.max(...eligible.map(e => e.waited))); + const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${run.id}`; - core.info(`[REAP${dryRun ? " (dry-run)" : ""}] ${wf} #${run.id} (${run.head_branch}) โ€” ${eligible.length} stuck job(s)`); + core.info(`[REAP${dryRun ? " (dry-run)" : ""}] ${isZombie ? "zombie" : "runner-wait"} ยท ${wf} #${run.id} (${run.head_branch}) ยท ${maxWait}`); summaryRows.push([ `${wf} #${run.id}`, run.head_branch || "(none)", - String(eligible.length), - humanize(Math.max(...eligible.map(e => e.waited))), + reason, + maxWait, ]); // Resolve the PR to comment on. @@ -199,36 +231,59 @@ jobs: } } - // Build and post the explanatory comment. - const jobRows = eligible.map(e => - `| \`${e.name}\` | \`${e.runner}\` | ${iso(e.sinceStr)} | ${humanize(e.waited)} | โŒ never assigned |` - ).join("\n"); - - const body = [ - `## ๐Ÿ›‘ Playwright run auto-cancelled by the runner-reaper`, - ``, - `A job in this run was **waiting for a self-hosted EKS runner that was never assigned**, past the ${threshold}-minute threshold. The reaper force-cancelled the run to free the queue.`, - ``, - `| | |`, - `|---|---|`, - `| **Run** | [\`${wf}\` #${run.id}](${runUrl}) (attempt ${run.run_attempt}) |`, - `| **Branch** | \`${run.head_branch}\` |`, - `| **Identified as stuck at** | ${nowIso} |`, - `| **Wait threshold** | ${threshold} minutes |`, - `| **Action taken** | \`POST /actions/runs/${run.id}/force-cancel\`${dryRun ? " _(dry-run โ€” not executed)_" : ""} |`, - ``, - `### Stuck job(s)`, - ``, - `| Job | Requested runner | Waiting since | Waited | Runner status |`, - `|---|---|---|---|---|`, - jobRows, - ``, - `**Why this happened:** the job(s) sat in \`queued\` because no matching EKS runner picked them up. GitHub's \`timeout-minutes\` does not cover runner-wait time โ€” it only starts once a job begins executing โ€” so the reaper enforces the ${threshold}-minute wait cap instead.`, - ``, - `**What to do:** re-run once EKS runner capacity is available (\`gh run rerun ${run.id}\`, or push a new commit). If this keeps happening, check the scale/health of the requested EKS runner pool(s) above.`, - ``, - `๐Ÿค– Automated by \`.github/workflows/stuck-run-reaper.yml\` ยท reaper run [#${context.runId}](${reaperUrl})`, - ].join("\n"); + // Build the explanatory comment โ€” wording is chosen per kind so a + // cancellation is never misread (a zombie run never held a runner). + let body; + if (isZombie) { + body = [ + `## ๐ŸงŸ Stale run auto-cancelled by the Stuck-Run-Reaper`, + ``, + `This run was **abandoned in a non-completed state with no jobs ever dispatched** for **${maxWait}** (since ${iso(eligible[0].sinceStr)}). It **never held a runner** โ€” this is **stale-run cleanup**, not a freed runner.`, + ``, + `| | |`, + `|---|---|`, + `| **Run** | [\`${wf}\` #${run.id}](${runUrl}) (attempt ${run.run_attempt}) |`, + `| **Branch** | \`${run.head_branch}\` |`, + `| **Stuck in state** | \`${run.status}\` with 0 jobs dispatched |`, + `| **Abandoned for** | ${maxWait} |`, + `| **Identified at** | ${nowIso} |`, + `| **Cleanup threshold** | ${zombieHours} hours |`, + `| **Action taken** | \`POST /actions/runs/${run.id}/force-cancel\`${dryRun ? " _(dry-run โ€” not executed)_" : ""} |`, + ``, + `**Why:** GitHub left this run in a non-completed state without ever creating its jobs, so it lingered in the "active" list indefinitely. **No EKS runner was ever occupied.** The reaper sweeps such zombie runs once they pass the ${zombieHours}-hour cleanup threshold. Nothing to re-run unless you still need this build.`, + ``, + `๐Ÿค– Automated by \`.github/workflows/stuck-run-reaper.yml\` ยท reaper run [#${context.runId}](${reaperUrl})`, + ].join("\n"); + } else { + const jobRows = eligible.map(e => + `| \`${e.name}\` | \`${e.runner}\` | ${iso(e.sinceStr)} | ${humanize(e.waited)} | โŒ never assigned |` + ).join("\n"); + body = [ + `## ๐Ÿ›‘ Run auto-cancelled by the Stuck-Run-Reaper`, + ``, + `A job in this run was **waiting for a self-hosted EKS runner that was never assigned**, past the ${threshold}-minute threshold. The reaper force-cancelled the run to free the queue.`, + ``, + `| | |`, + `|---|---|`, + `| **Run** | [\`${wf}\` #${run.id}](${runUrl}) (attempt ${run.run_attempt}) |`, + `| **Branch** | \`${run.head_branch}\` |`, + `| **Identified as stuck at** | ${nowIso} |`, + `| **Wait threshold** | ${threshold} minutes |`, + `| **Action taken** | \`POST /actions/runs/${run.id}/force-cancel\`${dryRun ? " _(dry-run โ€” not executed)_" : ""} |`, + ``, + `### Stuck job(s)`, + ``, + `| Job | Requested runner | Waiting since | Waited | Runner status |`, + `|---|---|---|---|---|`, + jobRows, + ``, + `**Why this happened:** the job(s) sat in \`queued\` because no matching EKS runner picked them up. GitHub's \`timeout-minutes\` does not cover runner-wait time โ€” it only starts once a job begins executing โ€” so the reaper enforces the ${threshold}-minute wait cap instead.`, + ``, + `**What to do:** re-run once EKS runner capacity is available (\`gh run rerun ${run.id}\`, or push a new commit). If this keeps happening, check the scale/health of the requested EKS runner pool(s) above.`, + ``, + `๐Ÿค– Automated by \`.github/workflows/stuck-run-reaper.yml\` ยท reaper run [#${context.runId}](${reaperUrl})`, + ].join("\n"); + } if (dryRun) { core.info(`[dry-run] Would comment on ${prNumber ? `PR #${prNumber}` : "(no PR found)"}:\n${body}`); @@ -251,10 +306,10 @@ jobs: // Job summary. await core.summary - .addHeading(`๐Ÿ›‘ Runner Reaper โ€” ${dryRun ? "dry-run" : `${reapedCount} run(s) reaped`}`) - .addRaw(`Threshold: **${threshold} min** ยท Workflows: \`${workflows.join(", ")}\``) + .addHeading(`๐Ÿ›‘ Stuck-Run-Reaper โ€” ${dryRun ? "dry-run" : `${reapedCount} run(s) reaped`}`) + .addRaw(`Runner-wait threshold: **${threshold} min** ยท Zombie-run cleanup: **${zombieHours}h** ยท Workflows: \`${workflows.join(", ")}\``) .addTable([ - [{ data: "Run", header: true }, { data: "Branch", header: true }, { data: "Stuck jobs", header: true }, { data: "Max wait", header: true }], + [{ data: "Run", header: true }, { data: "Branch", header: true }, { data: "Reason", header: true }, { data: "Waited / age", header: true }], ...summaryRows, ]) .write();