diff --git a/src/main/runtime/orchestration/db/questions/question-threads.ts b/src/main/runtime/orchestration/db/questions/question-threads.ts index 4f0be287206..ed4890f6a76 100644 --- a/src/main/runtime/orchestration/db/questions/question-threads.ts +++ b/src/main/runtime/orchestration/db/questions/question-threads.ts @@ -3,6 +3,7 @@ import { OrchestrationError } from '../../orchestration-error' import { exposeQuestionTimestamps } from '../utc-timestamp' import type { OrchestrationDb } from '../orchestration-db' import { handHelperMailToRun } from '../messages/helper-mailbox-hand-back' +import { isDispatchAssignee } from '../dispatch-context/dispatch-lookup' export function createQuestion( this: OrchestrationDb, @@ -168,9 +169,14 @@ export function answerQuestion( } } -/** Whether the Dispatch's last turn ended on its own pending question: no mail has reached it since it asked. */ -export function awaitsAnswerToLatestQuestion(this: OrchestrationDb, dispatchId: string): boolean { - const row = this.db +/** Whether the worker's last turn ended waiting on its coordinator: no mail has reached it since it asked, + * or since it replied to the coordinator's mail (a status like "paused, waiting for your call"). */ +export function awaitsCoordinator(this: OrchestrationDb, dispatchId: string): boolean { + return awaitsAnswerToLatestQuestion(this, dispatchId) || repliedSinceLatestMail(this, dispatchId) +} + +function awaitsAnswerToLatestQuestion(db: OrchestrationDb, dispatchId: string): boolean { + const row = db.db .prepare( `SELECT EXISTS ( SELECT 1 FROM question_threads q JOIN messages asked ON asked.id = q.message_id @@ -193,6 +199,38 @@ export function awaitsAnswerToLatestQuestion(this: OrchestrationDb, dispatchId: return row?.pending === 1 } +function repliedSinceLatestMail(db: OrchestrationDb, dispatchId: string): boolean { + const inbox = `dispatch:${dispatchId}` + const dispatch = db.getDispatchContextById(dispatchId) + if (!dispatch) { + // A paired worker's outgoing mail is in its relay to the Run home, so only timestamps order it here. + const last = db.db + .prepare( + `SELECT json_extract(r.payload, '$.threadId') AS thread_id FROM federation_relay_items r + WHERE r.dispatch_id = ? AND r.direction = 'to_home' AND r.kind <> 'heartbeat' + AND r.created_at > (SELECT MAX(m.created_at) FROM messages m WHERE m.to_handle = ?) + ORDER BY r.sequence DESC LIMIT 1` + ) + .get(dispatchId, inbox) + return typeof last?.thread_id === 'string' + } + const sent = db.db + .prepare( + `SELECT from_handle, sender_pane_key, thread_id FROM messages + WHERE run_id = ? AND to_handle <> ? AND type <> 'heartbeat' + AND sequence > (SELECT MAX(sequence) FROM messages WHERE to_handle = ?) + ORDER BY sequence DESC` + ) + .all(dispatch.run_id, inbox, inbox) + // Only a threaded reply answers the coordinator; an unprompted progress status does not excuse a stop. + const last = sent.find((message) => { + const from = typeof message.from_handle === 'string' ? message.from_handle : '' + const paneKey = typeof message.sender_pane_key === 'string' ? message.sender_pane_key : null + return from === inbox || isDispatchAssignee(dispatch, from, paneKey) + }) + return typeof last?.thread_id === 'string' +} + export function closeQuestionsForDispatch(this: OrchestrationDb, dispatchId: string): string[] { // Every settlement passes here, so transfer helper mail to the Run here too. handHelperMailToRun(this, dispatchId) @@ -215,7 +253,7 @@ export type QuestionThreadsMethods = { getQuestion: typeof getQuestion getQuestionRaw: typeof getQuestionRaw answerQuestion: typeof answerQuestion - awaitsAnswerToLatestQuestion: typeof awaitsAnswerToLatestQuestion + awaitsCoordinator: typeof awaitsCoordinator closeQuestionsForDispatch: typeof closeQuestionsForDispatch } @@ -225,7 +263,7 @@ export function attachQuestionThreads(ctor: { prototype: object }): void { getQuestion, getQuestionRaw, answerQuestion, - awaitsAnswerToLatestQuestion, + awaitsCoordinator, closeQuestionsForDispatch }) } diff --git a/src/main/runtime/orchestration/workers/turn-stops/worker-turn-stop-report.ts b/src/main/runtime/orchestration/workers/turn-stops/worker-turn-stop-report.ts index 22e0a81df73..a52336f5ba7 100644 --- a/src/main/runtime/orchestration/workers/turn-stops/worker-turn-stop-report.ts +++ b/src/main/runtime/orchestration/workers/turn-stops/worker-turn-stop-report.ts @@ -87,12 +87,12 @@ export function reportWorkerTurnStop( deps: WorkerTurnStopReportDeps ): void { const dispatchId = workerTurnStopDispatchId(target) - // Waiting on mail, or a clean finish resting on its own question or helpers, is deliberate; an exit never is. + // Waiting on mail, or a clean finish resting on its own question, reply or helpers, is deliberate; an exit never is. if ( stop.reason !== 'exited' && (deps.isWaitingForMail(`dispatch:${dispatchId}`) || (stop.reason === 'finished' && - (db.awaitsAnswerToLatestQuestion(dispatchId) || + (db.awaitsCoordinator(dispatchId) || (target.kind === 'local' && db.hasOpenHelperDispatch(dispatchId))))) ) { return diff --git a/src/main/runtime/orchestration/workers/turn-stops/worker-turn-stops.test.ts b/src/main/runtime/orchestration/workers/turn-stops/worker-turn-stops.test.ts index eaf3299653e..1ef0dfdcfe2 100644 --- a/src/main/runtime/orchestration/workers/turn-stops/worker-turn-stops.test.ts +++ b/src/main/runtime/orchestration/workers/turn-stops/worker-turn-stops.test.ts @@ -329,6 +329,67 @@ describe('terminal worker turn stops', () => { expect(ctx.mail().map((m) => m.subject)).toEqual(['Worker stopped without reporting']) }) + it('stays quiet after the worker replies to a hold and ends its turn, until the coordinator writes again', async () => { + const ctx = setup() + const hold = ctx.db.insertMessage({ + from: 'term_coordinator', + to: `dispatch:${ctx.dispatch.id}`, + subject: 'HOLD: pause after your current step', + runId: ctx.run.id + }) + ctx.db.markAsRead([hold.id]) + ctx.db.insertMessage({ + from: HANDLE, + senderPaneKey: PANE, + to: `run:${ctx.run.id}`, + subject: 'Re: HOLD', + body: 'Paused. Waiting for your call.', + threadId: hold.id, + runId: ctx.run.id + }) + const current = { row: hookRow({}) } + const stops = terminal(ctx, current) + stops.observe(moved('working', 'done')) + await settle() + expect(ctx.mail()).toHaveLength(0) + + const decision = ctx.db.insertMessage({ + from: 'term_coordinator', + to: `dispatch:${ctx.dispatch.id}`, + subject: 'Go ahead', + runId: ctx.run.id + }) + ctx.db.markAsRead([decision.id]) + current.row = hookRow({ stateStartedAt: 300 }) + stops.observe(moved('working', 'done')) + await settle() + expect(ctx.mail().map((m) => m.subject)).toEqual(['Worker stopped without reporting']) + }) + + it('reports a stop after an unprompted progress status', async () => { + const ctx = setup() + const start = ctx.db.insertMessage({ + from: 'term_coordinator', + to: `dispatch:${ctx.dispatch.id}`, + subject: 'Start', + runId: ctx.run.id + }) + ctx.db.markAsRead([start.id]) + ctx.db.insertMessage({ + from: HANDLE, + senderPaneKey: PANE, + to: `run:${ctx.run.id}`, + subject: 'Progress', + body: 'Halfway there.', + runId: ctx.run.id + }) + const current = { row: hookRow({}) } + const stops = terminal(ctx, current) + stops.observe(moved('working', 'done')) + await settle() + expect(ctx.mail().map((m) => m.subject)).toEqual(['Worker stopped without reporting']) + }) + it('reports a later stall once new mail reached the worker, though its question is pending', async () => { const ctx = setup() // Mail stored before the question, even in the same second, does not end the asking turn's mute. @@ -507,4 +568,47 @@ describe('remote worker turn stops', () => { reportWorkerTurnStop(db, target, { ...stop, stopKey: 'k2' }, deps) expect(enqueueFederationRelay).not.toHaveBeenCalled() }) + + it('stays quiet for a paired worker whose relayed reply answered its latest mail', () => { + const db = new OrchestrationDb(':memory:') + const run = db.createRun({ + objective: 'paired', + coordinatorHandle: 'term_coordinator', + coordinatorPaneKey: makePaneKey('tab-coordinator', '22222222-2222-4222-8222-222222222222') + }) + const hold = db.insertMessage({ + from: 'term_home', + to: 'dispatch:dsp_remote', + subject: 'HOLD', + runId: run.id + }) + db.db + .prepare("UPDATE messages SET created_at = datetime('now', '-1 minute') WHERE id = ?") + .run(hold.id) + const target = { + kind: 'remote' as const, + attachment: { dispatch_id: 'dsp_remote', task_id: 'task_remote' } + } + const deps = { getDb: () => db, isWaitingForMail: () => false, notify: vi.fn() } + const stop = { from: HANDLE, stopKey: 'k', reason: 'finished' as const, lastOutput: 'paused' } + db.enqueueFederationRelay({ + dispatchId: 'dsp_remote', + direction: 'to_home', + kind: 'status', + payload: JSON.stringify({ from: HANDLE, subject: 'Re: HOLD', threadId: hold.id }) + }) + const enqueueFederationRelay = vi.spyOn(db, 'enqueueFederationRelay') + reportWorkerTurnStop(db, target, stop, deps) + expect(enqueueFederationRelay).not.toHaveBeenCalled() + + db.enqueueFederationRelay({ + dispatchId: 'dsp_remote', + direction: 'to_home', + kind: 'status', + payload: JSON.stringify({ from: HANDLE, subject: 'Progress', threadId: null }) + }) + enqueueFederationRelay.mockClear() + reportWorkerTurnStop(db, target, { ...stop, stopKey: 'k2' }, deps) + expect(enqueueFederationRelay.mock.calls[0]?.[0]).toMatchObject({ kind: 'escalation' }) + }) })