fix(orchestration): a worker that replied to its coordinator is not a silent stop

This commit is contained in:
Jinwoo-H
2026-11-10 03:02:05 -05:00
parent 643411cfa3
commit c871bdafa5
3 changed files with 149 additions and 7 deletions
@@ -3,6 +3,7 @@ import { OrchestrationError } from '../../orchestration-error'
import { exposeQuestionTimestamps } from '../utc-timestamp'
import type { OrchestrationDb } from '../orchestration-db'
import { handHelperMailToRun } from '../messages/helper-mailbox-hand-back'
import { isDispatchAssignee } from '../dispatch-context/dispatch-lookup'
export function createQuestion(
this: OrchestrationDb,
@@ -168,9 +169,14 @@ export function answerQuestion(
}
}
/** Whether the Dispatch's last turn ended on its own pending question: no mail has reached it since it asked. */
export function awaitsAnswerToLatestQuestion(this: OrchestrationDb, dispatchId: string): boolean {
const row = this.db
/** Whether the worker's last turn ended waiting on its coordinator: no mail has reached it since it asked,
* or since it replied to the coordinator's mail (a status like "paused, waiting for your call"). */
export function awaitsCoordinator(this: OrchestrationDb, dispatchId: string): boolean {
return awaitsAnswerToLatestQuestion(this, dispatchId) || repliedSinceLatestMail(this, dispatchId)
}
function awaitsAnswerToLatestQuestion(db: OrchestrationDb, dispatchId: string): boolean {
const row = db.db
.prepare(
`SELECT EXISTS (
SELECT 1 FROM question_threads q JOIN messages asked ON asked.id = q.message_id
@@ -193,6 +199,38 @@ export function awaitsAnswerToLatestQuestion(this: OrchestrationDb, dispatchId:
return row?.pending === 1
}
function repliedSinceLatestMail(db: OrchestrationDb, dispatchId: string): boolean {
const inbox = `dispatch:${dispatchId}`
const dispatch = db.getDispatchContextById(dispatchId)
if (!dispatch) {
// A paired worker's outgoing mail is in its relay to the Run home, so only timestamps order it here.
const last = db.db
.prepare(
`SELECT json_extract(r.payload, '$.threadId') AS thread_id FROM federation_relay_items r
WHERE r.dispatch_id = ? AND r.direction = 'to_home' AND r.kind <> 'heartbeat'
AND r.created_at > (SELECT MAX(m.created_at) FROM messages m WHERE m.to_handle = ?)
ORDER BY r.sequence DESC LIMIT 1`
)
.get(dispatchId, inbox)
return typeof last?.thread_id === 'string'
}
const sent = db.db
.prepare(
`SELECT from_handle, sender_pane_key, thread_id FROM messages
WHERE run_id = ? AND to_handle <> ? AND type <> 'heartbeat'
AND sequence > (SELECT MAX(sequence) FROM messages WHERE to_handle = ?)
ORDER BY sequence DESC`
)
.all(dispatch.run_id, inbox, inbox)
// Only a threaded reply answers the coordinator; an unprompted progress status does not excuse a stop.
const last = sent.find((message) => {
const from = typeof message.from_handle === 'string' ? message.from_handle : ''
const paneKey = typeof message.sender_pane_key === 'string' ? message.sender_pane_key : null
return from === inbox || isDispatchAssignee(dispatch, from, paneKey)
})
return typeof last?.thread_id === 'string'
}
export function closeQuestionsForDispatch(this: OrchestrationDb, dispatchId: string): string[] {
// Every settlement passes here, so transfer helper mail to the Run here too.
handHelperMailToRun(this, dispatchId)
@@ -215,7 +253,7 @@ export type QuestionThreadsMethods = {
getQuestion: typeof getQuestion
getQuestionRaw: typeof getQuestionRaw
answerQuestion: typeof answerQuestion
awaitsAnswerToLatestQuestion: typeof awaitsAnswerToLatestQuestion
awaitsCoordinator: typeof awaitsCoordinator
closeQuestionsForDispatch: typeof closeQuestionsForDispatch
}
@@ -225,7 +263,7 @@ export function attachQuestionThreads(ctor: { prototype: object }): void {
getQuestion,
getQuestionRaw,
answerQuestion,
awaitsAnswerToLatestQuestion,
awaitsCoordinator,
closeQuestionsForDispatch
})
}
@@ -87,12 +87,12 @@ export function reportWorkerTurnStop(
deps: WorkerTurnStopReportDeps
): void {
const dispatchId = workerTurnStopDispatchId(target)
// Waiting on mail, or a clean finish resting on its own question or helpers, is deliberate; an exit never is.
// Waiting on mail, or a clean finish resting on its own question, reply or helpers, is deliberate; an exit never is.
if (
stop.reason !== 'exited' &&
(deps.isWaitingForMail(`dispatch:${dispatchId}`) ||
(stop.reason === 'finished' &&
(db.awaitsAnswerToLatestQuestion(dispatchId) ||
(db.awaitsCoordinator(dispatchId) ||
(target.kind === 'local' && db.hasOpenHelperDispatch(dispatchId)))))
) {
return
@@ -329,6 +329,67 @@ describe('terminal worker turn stops', () => {
expect(ctx.mail().map((m) => m.subject)).toEqual(['Worker stopped without reporting'])
})
it('stays quiet after the worker replies to a hold and ends its turn, until the coordinator writes again', async () => {
const ctx = setup()
const hold = ctx.db.insertMessage({
from: 'term_coordinator',
to: `dispatch:${ctx.dispatch.id}`,
subject: 'HOLD: pause after your current step',
runId: ctx.run.id
})
ctx.db.markAsRead([hold.id])
ctx.db.insertMessage({
from: HANDLE,
senderPaneKey: PANE,
to: `run:${ctx.run.id}`,
subject: 'Re: HOLD',
body: 'Paused. Waiting for your call.',
threadId: hold.id,
runId: ctx.run.id
})
const current = { row: hookRow({}) }
const stops = terminal(ctx, current)
stops.observe(moved('working', 'done'))
await settle()
expect(ctx.mail()).toHaveLength(0)
const decision = ctx.db.insertMessage({
from: 'term_coordinator',
to: `dispatch:${ctx.dispatch.id}`,
subject: 'Go ahead',
runId: ctx.run.id
})
ctx.db.markAsRead([decision.id])
current.row = hookRow({ stateStartedAt: 300 })
stops.observe(moved('working', 'done'))
await settle()
expect(ctx.mail().map((m) => m.subject)).toEqual(['Worker stopped without reporting'])
})
it('reports a stop after an unprompted progress status', async () => {
const ctx = setup()
const start = ctx.db.insertMessage({
from: 'term_coordinator',
to: `dispatch:${ctx.dispatch.id}`,
subject: 'Start',
runId: ctx.run.id
})
ctx.db.markAsRead([start.id])
ctx.db.insertMessage({
from: HANDLE,
senderPaneKey: PANE,
to: `run:${ctx.run.id}`,
subject: 'Progress',
body: 'Halfway there.',
runId: ctx.run.id
})
const current = { row: hookRow({}) }
const stops = terminal(ctx, current)
stops.observe(moved('working', 'done'))
await settle()
expect(ctx.mail().map((m) => m.subject)).toEqual(['Worker stopped without reporting'])
})
it('reports a later stall once new mail reached the worker, though its question is pending', async () => {
const ctx = setup()
// Mail stored before the question, even in the same second, does not end the asking turn's mute.
@@ -507,4 +568,47 @@ describe('remote worker turn stops', () => {
reportWorkerTurnStop(db, target, { ...stop, stopKey: 'k2' }, deps)
expect(enqueueFederationRelay).not.toHaveBeenCalled()
})
it('stays quiet for a paired worker whose relayed reply answered its latest mail', () => {
const db = new OrchestrationDb(':memory:')
const run = db.createRun({
objective: 'paired',
coordinatorHandle: 'term_coordinator',
coordinatorPaneKey: makePaneKey('tab-coordinator', '22222222-2222-4222-8222-222222222222')
})
const hold = db.insertMessage({
from: 'term_home',
to: 'dispatch:dsp_remote',
subject: 'HOLD',
runId: run.id
})
db.db
.prepare("UPDATE messages SET created_at = datetime('now', '-1 minute') WHERE id = ?")
.run(hold.id)
const target = {
kind: 'remote' as const,
attachment: { dispatch_id: 'dsp_remote', task_id: 'task_remote' }
}
const deps = { getDb: () => db, isWaitingForMail: () => false, notify: vi.fn() }
const stop = { from: HANDLE, stopKey: 'k', reason: 'finished' as const, lastOutput: 'paused' }
db.enqueueFederationRelay({
dispatchId: 'dsp_remote',
direction: 'to_home',
kind: 'status',
payload: JSON.stringify({ from: HANDLE, subject: 'Re: HOLD', threadId: hold.id })
})
const enqueueFederationRelay = vi.spyOn(db, 'enqueueFederationRelay')
reportWorkerTurnStop(db, target, stop, deps)
expect(enqueueFederationRelay).not.toHaveBeenCalled()
db.enqueueFederationRelay({
dispatchId: 'dsp_remote',
direction: 'to_home',
kind: 'status',
payload: JSON.stringify({ from: HANDLE, subject: 'Progress', threadId: null })
})
enqueueFederationRelay.mockClear()
reportWorkerTurnStop(db, target, { ...stop, stopKey: 'k2' }, deps)
expect(enqueueFederationRelay.mock.calls[0]?.[0]).toMatchObject({ kind: 'escalation' })
})
})