diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index b66921ef49f..e046e4acde4 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -128,6 +128,7 @@ import { mkdir, readFile, readdir, rm, stat } from 'node:fs/promises' import { resolveWorktreeCreateBase } from '../worktree-create-base' import { resolveWorktreeAddBaseRef } from '../../shared/worktree/base-ref' import { OrchestrationDb } from './orchestration/db' +import type { DispatchStatus } from './orchestration/types' import { reconcileRequestedWorkerTerminalReleases } from './orchestration/worker-terminal-release-reconciliation' import { classifyWorkerTerminalProcessIncarnation, @@ -16830,7 +16831,9 @@ export class OrcaRuntimeService { } const errorContext = `Agent exited with code ${exitCode}` - this._orchestrationDb.failDispatch(dispatch.id, errorContext, { workerProcessExited: true }) + const settled = this._orchestrationDb.failDispatch(dispatch.id, errorContext, { + workerProcessExited: true + }) // Why: create an escalation message so the coordinator is notified about // the unexpected exit, even if the circuit breaker hasn't tripped yet. @@ -16847,6 +16850,7 @@ export class OrcaRuntimeService { from: handle, to: recipient.to, subject: `Agent exited unexpectedly (code ${exitCode})`, + body: this.describeWorkerExit(dispatch, exitCode, handle, settled?.status), type: 'escalation', priority: 'high', // Why: applyEscalationToDispatch rejects an escalation without an exact Dispatch @@ -16872,6 +16876,33 @@ export class OrcaRuntimeService { } } + // Why: the banner shows the subject and a raw payload, so without prose the coordinator + // has to resolve ids by hand to learn what died and whether the task is still retryable. + private describeWorkerExit( + dispatch: { id: string; task_id: string; run_id: string }, + exitCode: number, + handle: string, + settledStatus: DispatchStatus | undefined + ): string { + const task = this._orchestrationDb?.getTask?.(dispatch.task_id, dispatch.run_id) + const title = + typeof task?.spec === 'string' + ? buildOrchestrationTaskDisplayMetadata({ + spec: task.spec, + taskTitle: task.task_title, + displayName: task.display_name + }).taskTitle + : '' + const named = title ? `"${title}" (${dispatch.task_id})` : dispatch.task_id + const outcome = + settledStatus === 'circuit_broken' + ? ' This task has now failed too many times, so it will not be retried automatically.' + : settledStatus === 'failed' + ? ' The task is ready to be dispatched again.' + : '' + return `Worker ${handle} exited with code ${exitCode} while running task ${named}.${outcome}` + } + // Why: a lightweight Run keeps its coordinator in runs/run_coordinator_handles and // never writes the legacy coordinator_runs table, so gating solely on that table // dropped every worker-death escalation (STA-4604). Address the Run mailbox the diff --git a/src/main/runtime/orchestration/lightweight-run-worker-exit-escalation.test.ts b/src/main/runtime/orchestration/lightweight-run-worker-exit-escalation.test.ts index d0c17afc502..3813b9a0ad9 100644 --- a/src/main/runtime/orchestration/lightweight-run-worker-exit-escalation.test.ts +++ b/src/main/runtime/orchestration/lightweight-run-worker-exit-escalation.test.ts @@ -113,6 +113,7 @@ async function gradeWorkerExit( escalation: Record | null workerHandle: string runId: string + taskId: string }> { const { runtime, workerHandle, coordinatorHandle } = makeRuntimeWithTwoPanes() const db = new OrchestrationDb(':memory:') @@ -154,12 +155,14 @@ async function gradeWorkerExit( type: escalation.type, priority: escalation.priority, subject: escalation.subject, + body: escalation.body, deliveryContract: escalation.delivery_contract, payload: JSON.parse(escalation.payload ?? 'null') } : null, workerHandle, - runId + runId, + taskId: task.id } } finally { db.close() @@ -183,6 +186,9 @@ describe('STA-4604 worker PTY exit escalation reaches the coordinator', () => { type: 'escalation', priority: 'high', subject: 'Agent exited unexpectedly (code 137)', + body: + `Worker ${graded.workerHandle} exited with code 137 while running task ` + + `"do the work" (${graded.taskId}). The task is ready to be dispatched again.`, deliveryContract: 'current_delivery', payload: { taskId: expect.any(String), @@ -349,10 +355,18 @@ describe('STA-4604 worker PTY exit escalation reaches the coordinator', () => { runtime.onPtyExit(WORKER_PTY_ID, 137) await settle() + const escalations = db.getUnreadRunMailbox(run.id, 100, ['escalation']) expect({ dispatch: db.getDispatchContextById(dispatch.id)?.status, - escalations: db.getUnreadRunMailbox(run.id, 100, ['escalation']).length + escalations: escalations.length }).toEqual({ dispatch: 'circuit_broken', escalations: 1 }) + // Why: a tripped breaker means nobody will retry it, so the body must not + // tell the coordinator the task is ready to go again. + expect(escalations.at(-1)?.body).toBe( + `Worker ${workerHandle} exited with code 137 while running task ` + + `"repeatedly failing work" (${task.id}). This task has now failed too many ` + + `times, so it will not be retried automatically.` + ) } finally { db.close() }