mirror of
https://github.com/stablyai/orca.git
synced 2026-10-22 16:03:47 +00:00
fix(orchestration): say what died in the worker-exit escalation
The escalation carried a subject line and a raw payload and nothing else, so the
banner a coordinator actually reads was a code and three ids. Learning which task
had died, or whether it was still retryable, meant looking the ids up by hand.
Add a body naming the task and stating the resulting state, taken from what
failDispatch settled to: a plain failure says the task is ready to dispatch again,
a tripped circuit breaker says it will not be retried automatically. Task naming
reuses buildOrchestrationTaskDisplayMetadata, so it matches the title shown
everywhere else rather than inventing a second derivation.
Before:
Subject: Agent exited unexpectedly (code 137)
[Payload: {"taskId":"task_c917b402bd49",...}]
After:
Subject: Agent exited unexpectedly (code 137)
Worker term_62dc5050 exited with code 137 while running task
"Migrate the billing schema to v2" (task_c917b402bd49). The task is ready to be
dispatched again.
[Payload: {"taskId":"task_c917b402bd49",...}]
Both bodies are pinned by tests; the grading oracle now compares the body so the
wording cannot rot silently.
This commit is contained in:
@@ -128,6 +128,7 @@ import { mkdir, readFile, readdir, rm, stat } from 'node:fs/promises'
|
||||
import { resolveWorktreeCreateBase } from '../worktree-create-base'
|
||||
import { resolveWorktreeAddBaseRef } from '../../shared/worktree/base-ref'
|
||||
import { OrchestrationDb } from './orchestration/db'
|
||||
import type { DispatchStatus } from './orchestration/types'
|
||||
import { reconcileRequestedWorkerTerminalReleases } from './orchestration/worker-terminal-release-reconciliation'
|
||||
import {
|
||||
classifyWorkerTerminalProcessIncarnation,
|
||||
@@ -16830,7 +16831,9 @@ export class OrcaRuntimeService {
|
||||
}
|
||||
|
||||
const errorContext = `Agent exited with code ${exitCode}`
|
||||
this._orchestrationDb.failDispatch(dispatch.id, errorContext, { workerProcessExited: true })
|
||||
const settled = this._orchestrationDb.failDispatch(dispatch.id, errorContext, {
|
||||
workerProcessExited: true
|
||||
})
|
||||
|
||||
// Why: create an escalation message so the coordinator is notified about
|
||||
// the unexpected exit, even if the circuit breaker hasn't tripped yet.
|
||||
@@ -16847,6 +16850,7 @@ export class OrcaRuntimeService {
|
||||
from: handle,
|
||||
to: recipient.to,
|
||||
subject: `Agent exited unexpectedly (code ${exitCode})`,
|
||||
body: this.describeWorkerExit(dispatch, exitCode, handle, settled?.status),
|
||||
type: 'escalation',
|
||||
priority: 'high',
|
||||
// Why: applyEscalationToDispatch rejects an escalation without an exact Dispatch
|
||||
@@ -16872,6 +16876,33 @@ export class OrcaRuntimeService {
|
||||
}
|
||||
}
|
||||
|
||||
// Why: the banner shows the subject and a raw payload, so without prose the coordinator
|
||||
// has to resolve ids by hand to learn what died and whether the task is still retryable.
|
||||
private describeWorkerExit(
|
||||
dispatch: { id: string; task_id: string; run_id: string },
|
||||
exitCode: number,
|
||||
handle: string,
|
||||
settledStatus: DispatchStatus | undefined
|
||||
): string {
|
||||
const task = this._orchestrationDb?.getTask?.(dispatch.task_id, dispatch.run_id)
|
||||
const title =
|
||||
typeof task?.spec === 'string'
|
||||
? buildOrchestrationTaskDisplayMetadata({
|
||||
spec: task.spec,
|
||||
taskTitle: task.task_title,
|
||||
displayName: task.display_name
|
||||
}).taskTitle
|
||||
: ''
|
||||
const named = title ? `"${title}" (${dispatch.task_id})` : dispatch.task_id
|
||||
const outcome =
|
||||
settledStatus === 'circuit_broken'
|
||||
? ' This task has now failed too many times, so it will not be retried automatically.'
|
||||
: settledStatus === 'failed'
|
||||
? ' The task is ready to be dispatched again.'
|
||||
: ''
|
||||
return `Worker ${handle} exited with code ${exitCode} while running task ${named}.${outcome}`
|
||||
}
|
||||
|
||||
// Why: a lightweight Run keeps its coordinator in runs/run_coordinator_handles and
|
||||
// never writes the legacy coordinator_runs table, so gating solely on that table
|
||||
// dropped every worker-death escalation (STA-4604). Address the Run mailbox the
|
||||
|
||||
@@ -113,6 +113,7 @@ async function gradeWorkerExit(
|
||||
escalation: Record<string, unknown> | null
|
||||
workerHandle: string
|
||||
runId: string
|
||||
taskId: string
|
||||
}> {
|
||||
const { runtime, workerHandle, coordinatorHandle } = makeRuntimeWithTwoPanes()
|
||||
const db = new OrchestrationDb(':memory:')
|
||||
@@ -154,12 +155,14 @@ async function gradeWorkerExit(
|
||||
type: escalation.type,
|
||||
priority: escalation.priority,
|
||||
subject: escalation.subject,
|
||||
body: escalation.body,
|
||||
deliveryContract: escalation.delivery_contract,
|
||||
payload: JSON.parse(escalation.payload ?? 'null')
|
||||
}
|
||||
: null,
|
||||
workerHandle,
|
||||
runId
|
||||
runId,
|
||||
taskId: task.id
|
||||
}
|
||||
} finally {
|
||||
db.close()
|
||||
@@ -183,6 +186,9 @@ describe('STA-4604 worker PTY exit escalation reaches the coordinator', () => {
|
||||
type: 'escalation',
|
||||
priority: 'high',
|
||||
subject: 'Agent exited unexpectedly (code 137)',
|
||||
body:
|
||||
`Worker ${graded.workerHandle} exited with code 137 while running task ` +
|
||||
`"do the work" (${graded.taskId}). The task is ready to be dispatched again.`,
|
||||
deliveryContract: 'current_delivery',
|
||||
payload: {
|
||||
taskId: expect.any(String),
|
||||
@@ -349,10 +355,18 @@ describe('STA-4604 worker PTY exit escalation reaches the coordinator', () => {
|
||||
runtime.onPtyExit(WORKER_PTY_ID, 137)
|
||||
await settle()
|
||||
|
||||
const escalations = db.getUnreadRunMailbox(run.id, 100, ['escalation'])
|
||||
expect({
|
||||
dispatch: db.getDispatchContextById(dispatch.id)?.status,
|
||||
escalations: db.getUnreadRunMailbox(run.id, 100, ['escalation']).length
|
||||
escalations: escalations.length
|
||||
}).toEqual({ dispatch: 'circuit_broken', escalations: 1 })
|
||||
// Why: a tripped breaker means nobody will retry it, so the body must not
|
||||
// tell the coordinator the task is ready to go again.
|
||||
expect(escalations.at(-1)?.body).toBe(
|
||||
`Worker ${workerHandle} exited with code 137 while running task ` +
|
||||
`"repeatedly failing work" (${task.id}). This task has now failed too many ` +
|
||||
`times, so it will not be retried automatically.`
|
||||
)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user