mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
fix(orchestration): a worker that replied to its coordinator is not a silent stop
This commit is contained in:
@@ -3,6 +3,7 @@ import { OrchestrationError } from '../../orchestration-error'
|
||||
import { exposeQuestionTimestamps } from '../utc-timestamp'
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
import { handHelperMailToRun } from '../messages/helper-mailbox-hand-back'
|
||||
import { isDispatchAssignee } from '../dispatch-context/dispatch-lookup'
|
||||
|
||||
export function createQuestion(
|
||||
this: OrchestrationDb,
|
||||
@@ -168,9 +169,14 @@ export function answerQuestion(
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether the Dispatch's last turn ended on its own pending question: no mail has reached it since it asked. */
|
||||
export function awaitsAnswerToLatestQuestion(this: OrchestrationDb, dispatchId: string): boolean {
|
||||
const row = this.db
|
||||
/** Whether the worker's last turn ended waiting on its coordinator: no mail has reached it since it asked,
|
||||
* or since it replied to the coordinator's mail (a status like "paused, waiting for your call"). */
|
||||
export function awaitsCoordinator(this: OrchestrationDb, dispatchId: string): boolean {
|
||||
return awaitsAnswerToLatestQuestion(this, dispatchId) || repliedSinceLatestMail(this, dispatchId)
|
||||
}
|
||||
|
||||
function awaitsAnswerToLatestQuestion(db: OrchestrationDb, dispatchId: string): boolean {
|
||||
const row = db.db
|
||||
.prepare(
|
||||
`SELECT EXISTS (
|
||||
SELECT 1 FROM question_threads q JOIN messages asked ON asked.id = q.message_id
|
||||
@@ -193,6 +199,38 @@ export function awaitsAnswerToLatestQuestion(this: OrchestrationDb, dispatchId:
|
||||
return row?.pending === 1
|
||||
}
|
||||
|
||||
function repliedSinceLatestMail(db: OrchestrationDb, dispatchId: string): boolean {
|
||||
const inbox = `dispatch:${dispatchId}`
|
||||
const dispatch = db.getDispatchContextById(dispatchId)
|
||||
if (!dispatch) {
|
||||
// A paired worker's outgoing mail is in its relay to the Run home, so only timestamps order it here.
|
||||
const last = db.db
|
||||
.prepare(
|
||||
`SELECT json_extract(r.payload, '$.threadId') AS thread_id FROM federation_relay_items r
|
||||
WHERE r.dispatch_id = ? AND r.direction = 'to_home' AND r.kind <> 'heartbeat'
|
||||
AND r.created_at > (SELECT MAX(m.created_at) FROM messages m WHERE m.to_handle = ?)
|
||||
ORDER BY r.sequence DESC LIMIT 1`
|
||||
)
|
||||
.get(dispatchId, inbox)
|
||||
return typeof last?.thread_id === 'string'
|
||||
}
|
||||
const sent = db.db
|
||||
.prepare(
|
||||
`SELECT from_handle, sender_pane_key, thread_id FROM messages
|
||||
WHERE run_id = ? AND to_handle <> ? AND type <> 'heartbeat'
|
||||
AND sequence > (SELECT MAX(sequence) FROM messages WHERE to_handle = ?)
|
||||
ORDER BY sequence DESC`
|
||||
)
|
||||
.all(dispatch.run_id, inbox, inbox)
|
||||
// Only a threaded reply answers the coordinator; an unprompted progress status does not excuse a stop.
|
||||
const last = sent.find((message) => {
|
||||
const from = typeof message.from_handle === 'string' ? message.from_handle : ''
|
||||
const paneKey = typeof message.sender_pane_key === 'string' ? message.sender_pane_key : null
|
||||
return from === inbox || isDispatchAssignee(dispatch, from, paneKey)
|
||||
})
|
||||
return typeof last?.thread_id === 'string'
|
||||
}
|
||||
|
||||
export function closeQuestionsForDispatch(this: OrchestrationDb, dispatchId: string): string[] {
|
||||
// Every settlement passes here, so transfer helper mail to the Run here too.
|
||||
handHelperMailToRun(this, dispatchId)
|
||||
@@ -215,7 +253,7 @@ export type QuestionThreadsMethods = {
|
||||
getQuestion: typeof getQuestion
|
||||
getQuestionRaw: typeof getQuestionRaw
|
||||
answerQuestion: typeof answerQuestion
|
||||
awaitsAnswerToLatestQuestion: typeof awaitsAnswerToLatestQuestion
|
||||
awaitsCoordinator: typeof awaitsCoordinator
|
||||
closeQuestionsForDispatch: typeof closeQuestionsForDispatch
|
||||
}
|
||||
|
||||
@@ -225,7 +263,7 @@ export function attachQuestionThreads(ctor: { prototype: object }): void {
|
||||
getQuestion,
|
||||
getQuestionRaw,
|
||||
answerQuestion,
|
||||
awaitsAnswerToLatestQuestion,
|
||||
awaitsCoordinator,
|
||||
closeQuestionsForDispatch
|
||||
})
|
||||
}
|
||||
|
||||
@@ -87,12 +87,12 @@ export function reportWorkerTurnStop(
|
||||
deps: WorkerTurnStopReportDeps
|
||||
): void {
|
||||
const dispatchId = workerTurnStopDispatchId(target)
|
||||
// Waiting on mail, or a clean finish resting on its own question or helpers, is deliberate; an exit never is.
|
||||
// Waiting on mail, or a clean finish resting on its own question, reply or helpers, is deliberate; an exit never is.
|
||||
if (
|
||||
stop.reason !== 'exited' &&
|
||||
(deps.isWaitingForMail(`dispatch:${dispatchId}`) ||
|
||||
(stop.reason === 'finished' &&
|
||||
(db.awaitsAnswerToLatestQuestion(dispatchId) ||
|
||||
(db.awaitsCoordinator(dispatchId) ||
|
||||
(target.kind === 'local' && db.hasOpenHelperDispatch(dispatchId)))))
|
||||
) {
|
||||
return
|
||||
|
||||
@@ -329,6 +329,67 @@ describe('terminal worker turn stops', () => {
|
||||
expect(ctx.mail().map((m) => m.subject)).toEqual(['Worker stopped without reporting'])
|
||||
})
|
||||
|
||||
it('stays quiet after the worker replies to a hold and ends its turn, until the coordinator writes again', async () => {
|
||||
const ctx = setup()
|
||||
const hold = ctx.db.insertMessage({
|
||||
from: 'term_coordinator',
|
||||
to: `dispatch:${ctx.dispatch.id}`,
|
||||
subject: 'HOLD: pause after your current step',
|
||||
runId: ctx.run.id
|
||||
})
|
||||
ctx.db.markAsRead([hold.id])
|
||||
ctx.db.insertMessage({
|
||||
from: HANDLE,
|
||||
senderPaneKey: PANE,
|
||||
to: `run:${ctx.run.id}`,
|
||||
subject: 'Re: HOLD',
|
||||
body: 'Paused. Waiting for your call.',
|
||||
threadId: hold.id,
|
||||
runId: ctx.run.id
|
||||
})
|
||||
const current = { row: hookRow({}) }
|
||||
const stops = terminal(ctx, current)
|
||||
stops.observe(moved('working', 'done'))
|
||||
await settle()
|
||||
expect(ctx.mail()).toHaveLength(0)
|
||||
|
||||
const decision = ctx.db.insertMessage({
|
||||
from: 'term_coordinator',
|
||||
to: `dispatch:${ctx.dispatch.id}`,
|
||||
subject: 'Go ahead',
|
||||
runId: ctx.run.id
|
||||
})
|
||||
ctx.db.markAsRead([decision.id])
|
||||
current.row = hookRow({ stateStartedAt: 300 })
|
||||
stops.observe(moved('working', 'done'))
|
||||
await settle()
|
||||
expect(ctx.mail().map((m) => m.subject)).toEqual(['Worker stopped without reporting'])
|
||||
})
|
||||
|
||||
it('reports a stop after an unprompted progress status', async () => {
|
||||
const ctx = setup()
|
||||
const start = ctx.db.insertMessage({
|
||||
from: 'term_coordinator',
|
||||
to: `dispatch:${ctx.dispatch.id}`,
|
||||
subject: 'Start',
|
||||
runId: ctx.run.id
|
||||
})
|
||||
ctx.db.markAsRead([start.id])
|
||||
ctx.db.insertMessage({
|
||||
from: HANDLE,
|
||||
senderPaneKey: PANE,
|
||||
to: `run:${ctx.run.id}`,
|
||||
subject: 'Progress',
|
||||
body: 'Halfway there.',
|
||||
runId: ctx.run.id
|
||||
})
|
||||
const current = { row: hookRow({}) }
|
||||
const stops = terminal(ctx, current)
|
||||
stops.observe(moved('working', 'done'))
|
||||
await settle()
|
||||
expect(ctx.mail().map((m) => m.subject)).toEqual(['Worker stopped without reporting'])
|
||||
})
|
||||
|
||||
it('reports a later stall once new mail reached the worker, though its question is pending', async () => {
|
||||
const ctx = setup()
|
||||
// Mail stored before the question, even in the same second, does not end the asking turn's mute.
|
||||
@@ -507,4 +568,47 @@ describe('remote worker turn stops', () => {
|
||||
reportWorkerTurnStop(db, target, { ...stop, stopKey: 'k2' }, deps)
|
||||
expect(enqueueFederationRelay).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('stays quiet for a paired worker whose relayed reply answered its latest mail', () => {
|
||||
const db = new OrchestrationDb(':memory:')
|
||||
const run = db.createRun({
|
||||
objective: 'paired',
|
||||
coordinatorHandle: 'term_coordinator',
|
||||
coordinatorPaneKey: makePaneKey('tab-coordinator', '22222222-2222-4222-8222-222222222222')
|
||||
})
|
||||
const hold = db.insertMessage({
|
||||
from: 'term_home',
|
||||
to: 'dispatch:dsp_remote',
|
||||
subject: 'HOLD',
|
||||
runId: run.id
|
||||
})
|
||||
db.db
|
||||
.prepare("UPDATE messages SET created_at = datetime('now', '-1 minute') WHERE id = ?")
|
||||
.run(hold.id)
|
||||
const target = {
|
||||
kind: 'remote' as const,
|
||||
attachment: { dispatch_id: 'dsp_remote', task_id: 'task_remote' }
|
||||
}
|
||||
const deps = { getDb: () => db, isWaitingForMail: () => false, notify: vi.fn() }
|
||||
const stop = { from: HANDLE, stopKey: 'k', reason: 'finished' as const, lastOutput: 'paused' }
|
||||
db.enqueueFederationRelay({
|
||||
dispatchId: 'dsp_remote',
|
||||
direction: 'to_home',
|
||||
kind: 'status',
|
||||
payload: JSON.stringify({ from: HANDLE, subject: 'Re: HOLD', threadId: hold.id })
|
||||
})
|
||||
const enqueueFederationRelay = vi.spyOn(db, 'enqueueFederationRelay')
|
||||
reportWorkerTurnStop(db, target, stop, deps)
|
||||
expect(enqueueFederationRelay).not.toHaveBeenCalled()
|
||||
|
||||
db.enqueueFederationRelay({
|
||||
dispatchId: 'dsp_remote',
|
||||
direction: 'to_home',
|
||||
kind: 'status',
|
||||
payload: JSON.stringify({ from: HANDLE, subject: 'Progress', threadId: null })
|
||||
})
|
||||
enqueueFederationRelay.mockClear()
|
||||
reportWorkerTurnStop(db, target, { ...stop, stopKey: 'k2' }, deps)
|
||||
expect(enqueueFederationRelay.mock.calls[0]?.[0]).toMatchObject({ kind: 'escalation' })
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user