From 5527dff1243105f368d69a7cff1bbd04b2be7d24 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Tue, 18 Aug 2026 02:02:22 -0400 Subject: [PATCH] fix(orchestration): say what died in the worker-exit escalation The escalation carried a subject line and a raw payload and nothing else, so the banner a coordinator actually reads was a code and three ids. Learning which task had died, or whether it was still retryable, meant looking the ids up by hand. Add a body naming the task and stating the resulting state, taken from what failDispatch settled to: a plain failure says the task is ready to dispatch again, a tripped circuit breaker says it will not be retried automatically. Task naming reuses buildOrchestrationTaskDisplayMetadata, so it matches the title shown everywhere else rather than inventing a second derivation. Before: Subject: Agent exited unexpectedly (code 137) [Payload: {"taskId":"task_c917b402bd49",...}] After: Subject: Agent exited unexpectedly (code 137) Worker term_62dc5050 exited with code 137 while running task "Migrate the billing schema to v2" (task_c917b402bd49). The task is ready to be dispatched again. [Payload: {"taskId":"task_c917b402bd49",...}] Both bodies are pinned by tests; the grading oracle now compares the body so the wording cannot rot silently. --- src/main/runtime/orca-runtime.ts | 33 ++++++++++++++++++- ...tweight-run-worker-exit-escalation.test.ts | 18 ++++++++-- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index b66921ef49f..e046e4acde4 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -128,6 +128,7 @@ import { mkdir, readFile, readdir, rm, stat } from 'node:fs/promises' import { resolveWorktreeCreateBase } from '../worktree-create-base' import { resolveWorktreeAddBaseRef } from '../../shared/worktree/base-ref' import { OrchestrationDb } from './orchestration/db' +import type { DispatchStatus } from './orchestration/types' import { reconcileRequestedWorkerTerminalReleases } from './orchestration/worker-terminal-release-reconciliation' import { classifyWorkerTerminalProcessIncarnation, @@ -16830,7 +16831,9 @@ export class OrcaRuntimeService { } const errorContext = `Agent exited with code ${exitCode}` - this._orchestrationDb.failDispatch(dispatch.id, errorContext, { workerProcessExited: true }) + const settled = this._orchestrationDb.failDispatch(dispatch.id, errorContext, { + workerProcessExited: true + }) // Why: create an escalation message so the coordinator is notified about // the unexpected exit, even if the circuit breaker hasn't tripped yet. @@ -16847,6 +16850,7 @@ export class OrcaRuntimeService { from: handle, to: recipient.to, subject: `Agent exited unexpectedly (code ${exitCode})`, + body: this.describeWorkerExit(dispatch, exitCode, handle, settled?.status), type: 'escalation', priority: 'high', // Why: applyEscalationToDispatch rejects an escalation without an exact Dispatch @@ -16872,6 +16876,33 @@ export class OrcaRuntimeService { } } + // Why: the banner shows the subject and a raw payload, so without prose the coordinator + // has to resolve ids by hand to learn what died and whether the task is still retryable. + private describeWorkerExit( + dispatch: { id: string; task_id: string; run_id: string }, + exitCode: number, + handle: string, + settledStatus: DispatchStatus | undefined + ): string { + const task = this._orchestrationDb?.getTask?.(dispatch.task_id, dispatch.run_id) + const title = + typeof task?.spec === 'string' + ? buildOrchestrationTaskDisplayMetadata({ + spec: task.spec, + taskTitle: task.task_title, + displayName: task.display_name + }).taskTitle + : '' + const named = title ? `"${title}" (${dispatch.task_id})` : dispatch.task_id + const outcome = + settledStatus === 'circuit_broken' + ? ' This task has now failed too many times, so it will not be retried automatically.' + : settledStatus === 'failed' + ? ' The task is ready to be dispatched again.' + : '' + return `Worker ${handle} exited with code ${exitCode} while running task ${named}.${outcome}` + } + // Why: a lightweight Run keeps its coordinator in runs/run_coordinator_handles and // never writes the legacy coordinator_runs table, so gating solely on that table // dropped every worker-death escalation (STA-4604). Address the Run mailbox the diff --git a/src/main/runtime/orchestration/lightweight-run-worker-exit-escalation.test.ts b/src/main/runtime/orchestration/lightweight-run-worker-exit-escalation.test.ts index d0c17afc502..3813b9a0ad9 100644 --- a/src/main/runtime/orchestration/lightweight-run-worker-exit-escalation.test.ts +++ b/src/main/runtime/orchestration/lightweight-run-worker-exit-escalation.test.ts @@ -113,6 +113,7 @@ async function gradeWorkerExit( escalation: Record | null workerHandle: string runId: string + taskId: string }> { const { runtime, workerHandle, coordinatorHandle } = makeRuntimeWithTwoPanes() const db = new OrchestrationDb(':memory:') @@ -154,12 +155,14 @@ async function gradeWorkerExit( type: escalation.type, priority: escalation.priority, subject: escalation.subject, + body: escalation.body, deliveryContract: escalation.delivery_contract, payload: JSON.parse(escalation.payload ?? 'null') } : null, workerHandle, - runId + runId, + taskId: task.id } } finally { db.close() @@ -183,6 +186,9 @@ describe('STA-4604 worker PTY exit escalation reaches the coordinator', () => { type: 'escalation', priority: 'high', subject: 'Agent exited unexpectedly (code 137)', + body: + `Worker ${graded.workerHandle} exited with code 137 while running task ` + + `"do the work" (${graded.taskId}). The task is ready to be dispatched again.`, deliveryContract: 'current_delivery', payload: { taskId: expect.any(String), @@ -349,10 +355,18 @@ describe('STA-4604 worker PTY exit escalation reaches the coordinator', () => { runtime.onPtyExit(WORKER_PTY_ID, 137) await settle() + const escalations = db.getUnreadRunMailbox(run.id, 100, ['escalation']) expect({ dispatch: db.getDispatchContextById(dispatch.id)?.status, - escalations: db.getUnreadRunMailbox(run.id, 100, ['escalation']).length + escalations: escalations.length }).toEqual({ dispatch: 'circuit_broken', escalations: 1 }) + // Why: a tripped breaker means nobody will retry it, so the body must not + // tell the coordinator the task is ready to go again. + expect(escalations.at(-1)?.body).toBe( + `Worker ${workerHandle} exited with code 137 while running task ` + + `"repeatedly failing work" (${task.id}). This task has now failed too many ` + + `times, so it will not be retried automatically.` + ) } finally { db.close() }