fix(orchestration): say what died in the worker-exit escalation

The escalation carried a subject line and a raw payload and nothing else, so the
banner a coordinator actually reads was a code and three ids. Learning which task
had died, or whether it was still retryable, meant looking the ids up by hand.

Add a body naming the task and stating the resulting state, taken from what
failDispatch settled to: a plain failure says the task is ready to dispatch again,
a tripped circuit breaker says it will not be retried automatically. Task naming
reuses buildOrchestrationTaskDisplayMetadata, so it matches the title shown
everywhere else rather than inventing a second derivation.

Before:
  Subject: Agent exited unexpectedly (code 137)
  [Payload: {"taskId":"task_c917b402bd49",...}]

After:
  Subject: Agent exited unexpectedly (code 137)
  Worker term_62dc5050 exited with code 137 while running task
  "Migrate the billing schema to v2" (task_c917b402bd49). The task is ready to be
  dispatched again.
  [Payload: {"taskId":"task_c917b402bd49",...}]

Both bodies are pinned by tests; the grading oracle now compares the body so the
wording cannot rot silently.
This commit is contained in:
Jinwoo-H
2026-08-18 02:02:22 -04:00
parent da22935550
commit 5527dff124
2 changed files with 48 additions and 3 deletions
+32 -1
View File
@@ -128,6 +128,7 @@ import { mkdir, readFile, readdir, rm, stat } from 'node:fs/promises'
import { resolveWorktreeCreateBase } from '../worktree-create-base'
import { resolveWorktreeAddBaseRef } from '../../shared/worktree/base-ref'
import { OrchestrationDb } from './orchestration/db'
import type { DispatchStatus } from './orchestration/types'
import { reconcileRequestedWorkerTerminalReleases } from './orchestration/worker-terminal-release-reconciliation'
import {
classifyWorkerTerminalProcessIncarnation,
@@ -16830,7 +16831,9 @@ export class OrcaRuntimeService {
}
const errorContext = `Agent exited with code ${exitCode}`
this._orchestrationDb.failDispatch(dispatch.id, errorContext, { workerProcessExited: true })
const settled = this._orchestrationDb.failDispatch(dispatch.id, errorContext, {
workerProcessExited: true
})
// Why: create an escalation message so the coordinator is notified about
// the unexpected exit, even if the circuit breaker hasn't tripped yet.
@@ -16847,6 +16850,7 @@ export class OrcaRuntimeService {
from: handle,
to: recipient.to,
subject: `Agent exited unexpectedly (code ${exitCode})`,
body: this.describeWorkerExit(dispatch, exitCode, handle, settled?.status),
type: 'escalation',
priority: 'high',
// Why: applyEscalationToDispatch rejects an escalation without an exact Dispatch
@@ -16872,6 +16876,33 @@ export class OrcaRuntimeService {
}
}
// Why: the banner shows the subject and a raw payload, so without prose the coordinator
// has to resolve ids by hand to learn what died and whether the task is still retryable.
private describeWorkerExit(
dispatch: { id: string; task_id: string; run_id: string },
exitCode: number,
handle: string,
settledStatus: DispatchStatus | undefined
): string {
const task = this._orchestrationDb?.getTask?.(dispatch.task_id, dispatch.run_id)
const title =
typeof task?.spec === 'string'
? buildOrchestrationTaskDisplayMetadata({
spec: task.spec,
taskTitle: task.task_title,
displayName: task.display_name
}).taskTitle
: ''
const named = title ? `"${title}" (${dispatch.task_id})` : dispatch.task_id
const outcome =
settledStatus === 'circuit_broken'
? ' This task has now failed too many times, so it will not be retried automatically.'
: settledStatus === 'failed'
? ' The task is ready to be dispatched again.'
: ''
return `Worker ${handle} exited with code ${exitCode} while running task ${named}.${outcome}`
}
// Why: a lightweight Run keeps its coordinator in runs/run_coordinator_handles and
// never writes the legacy coordinator_runs table, so gating solely on that table
// dropped every worker-death escalation (STA-4604). Address the Run mailbox the
@@ -113,6 +113,7 @@ async function gradeWorkerExit(
escalation: Record<string, unknown> | null
workerHandle: string
runId: string
taskId: string
}> {
const { runtime, workerHandle, coordinatorHandle } = makeRuntimeWithTwoPanes()
const db = new OrchestrationDb(':memory:')
@@ -154,12 +155,14 @@ async function gradeWorkerExit(
type: escalation.type,
priority: escalation.priority,
subject: escalation.subject,
body: escalation.body,
deliveryContract: escalation.delivery_contract,
payload: JSON.parse(escalation.payload ?? 'null')
}
: null,
workerHandle,
runId
runId,
taskId: task.id
}
} finally {
db.close()
@@ -183,6 +186,9 @@ describe('STA-4604 worker PTY exit escalation reaches the coordinator', () => {
type: 'escalation',
priority: 'high',
subject: 'Agent exited unexpectedly (code 137)',
body:
`Worker ${graded.workerHandle} exited with code 137 while running task ` +
`"do the work" (${graded.taskId}). The task is ready to be dispatched again.`,
deliveryContract: 'current_delivery',
payload: {
taskId: expect.any(String),
@@ -349,10 +355,18 @@ describe('STA-4604 worker PTY exit escalation reaches the coordinator', () => {
runtime.onPtyExit(WORKER_PTY_ID, 137)
await settle()
const escalations = db.getUnreadRunMailbox(run.id, 100, ['escalation'])
expect({
dispatch: db.getDispatchContextById(dispatch.id)?.status,
escalations: db.getUnreadRunMailbox(run.id, 100, ['escalation']).length
escalations: escalations.length
}).toEqual({ dispatch: 'circuit_broken', escalations: 1 })
// Why: a tripped breaker means nobody will retry it, so the body must not
// tell the coordinator the task is ready to go again.
expect(escalations.at(-1)?.body).toBe(
`Worker ${workerHandle} exited with code 137 while running task ` +
`"repeatedly failing work" (${task.id}). This task has now failed too many ` +
`times, so it will not be retried automatically.`
)
} finally {
db.close()
}