From d9a7aa89c8a6250583ca8e7110f4e2e0cd8ac03e Mon Sep 17 00:00:00 2001
From: Neil <4138956+nwparker@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:06:20 -0700
Subject: [PATCH] ci(wsl): rerun a WSL2 lane job once when its hosted runner
 freezes (#33532)

* ci(wsl): rerun a WSL2 lane job once when its hosted runner freezes

* ci(wsl): pin the retry to the lane workflow and keep a 30 min job cap
---
 .../wsl-orcad-e2e-frozen-runner-retry.yml     | 59 +++++++++++++++++++
 .github/workflows/wsl-orcad-e2e.yml           |  7 ++-
 2 files changed, 63 insertions(+), 3 deletions(-)
 create mode 100644 .github/workflows/wsl-orcad-e2e-frozen-runner-retry.yml

diff --git a/.github/workflows/wsl-orcad-e2e-frozen-runner-retry.yml b/.github/workflows/wsl-orcad-e2e-frozen-runner-retry.yml
new file mode 100644
index 00000000000..ebc97fc8330
--- /dev/null
+++ b/.github/workflows/wsl-orcad-e2e-frozen-runner-retry.yml
@@ -0,0 +1,59 @@
+name: WSL orcad E2E frozen-runner retry
+
+# ~3% of hosted Windows runners freeze during WSL2 setup (no step timeout, no logs); rerun such a job once.
+on:
+  workflow_run:
+    workflows: [WSL orcad E2E]
+    types: [completed]
+
+permissions: {}
+
+jobs:
+  retry:
+    # The name match also admits same-named fork workflows; the path pins ours.
+    if: >-
+      github.event.workflow_run.path == '.github/workflows/wsl-orcad-e2e.yml' &&
+      github.event.workflow_run.run_attempt == 1 && github.event.workflow_run.conclusion != 'success'
+    runs-on: blacksmith-2vcpu-ubuntu-2204
+    timeout-minutes: 5
+    permissions:
+      actions: write
+      checks: read
+    steps:
+      - name: Rerun a WSL2 job reaped at its time cap
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const run = context.payload.workflow_run
+            const { owner, repo } = context.repo
+            // A newer run for the same branch and repository supersedes this one; fork branch names collide.
+            const recent = await github.rest.actions.listWorkflowRuns({
+              owner, repo, workflow_id: run.workflow_id, branch: run.head_branch, event: run.event, per_page: 20
+            })
+            const latest = recent.data.workflow_runs.find((r) => r.head_repository?.id === run.head_repository?.id)
+            if (latest?.id !== run.id) {
+              core.info('A newer run exists; not retrying')
+              return
+            }
+            const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, {
+              owner, repo, run_id: run.id, attempt_number: 1, per_page: 100
+            })
+            const frozen = []
+            for (const job of jobs.filter((job) => job.name.startsWith('WSL2') && job.conclusion === 'cancelled')) {
+              const annotations = await github.rest.checks.listAnnotations({ owner, repo, check_run_id: job.id })
+              // The frozen worker's only signature: a lost lease reaped at timeout-minutes, not a cancel.
+              if (annotations.data.some((a) => a.message.includes('exceeded the maximum execution time'))) {
+                frozen.push(job)
+              }
+            }
+            if (frozen.length === 0) {
+              core.info('No job was reaped at its time cap; not retrying')
+              return
+            }
+            core.notice(`Rerunning ${run.html_url} after a frozen runner: ${frozen.map((job) => job.name).join(', ')}`)
+            // A single-job rerun starts the run, so several frozen jobs go through the failed-jobs rerun.
+            if (frozen.length === 1) {
+              await github.rest.actions.reRunJobForWorkflowRun({ owner, repo, job_id: frozen[0].id })
+            } else {
+              await github.rest.actions.reRunWorkflowFailedJobs({ owner, repo, run_id: run.id })
+            }
diff --git a/.github/workflows/wsl-orcad-e2e.yml b/.github/workflows/wsl-orcad-e2e.yml
index 4c11aa9fbde..b7785dc9d1e 100644
--- a/.github/workflows/wsl-orcad-e2e.yml
+++ b/.github/workflows/wsl-orcad-e2e.yml
@@ -149,7 +149,8 @@ jobs:
     name: WSL2 distro (${{ matrix.family }})
     needs: [template, container-rootfs]
     runs-on: windows-2025
-    timeout-minutes: 45
+    # Healthy runs take 4-11 min; a runner frozen during WSL2 setup is only reaped here (retried by wsl-orcad-e2e-frozen-runner-retry.yml).
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
@@ -320,7 +321,7 @@ jobs:
             "$name=$path" >> $env:GITHUB_ENV
           }
       - name: 'Phase 1: deploy, RPC, kill orcad, terminate distro'
-        timeout-minutes: 20
+        timeout-minutes: 10
         env:
           ORCA_RUN_WSL_ORCAD_LANE: '1'
           ORCA_WSL_ORCAD_LANE_PHASE: launch
@@ -331,7 +332,7 @@ jobs:
           if ($global:LASTEXITCODE -ne 0) { throw "launch phase failed: $global:LASTEXITCODE" }
       # A new process: the holders must have kept both distros Running since phase 1 exited.
       - name: 'Phase 2: holders survive, chat, reconnect, teardown'
-        timeout-minutes: 15
+        timeout-minutes: 8
         env:
           ORCA_RUN_WSL_ORCAD_LANE: '1'
           ORCA_WSL_ORCAD_LANE_PHASE: survive