fix(orchestration): refuse worker-start on a busy terminal before any wait

worker-start --terminal and the follow-up a send to a finished worker starts
now claim the terminal through one check, before a Run binds, a Task or
Dispatch is written, or readiness is awaited. The Dispatch insert repeats the
claim atomically, and the late authority check stays as a backstop.
This commit is contained in:
Jinwoo-H
2026-11-08 15:17:07 -05:00
parent 49d74f57b1
commit 3ee53c89d5
6 changed files with 181 additions and 15 deletions
@@ -12,6 +12,7 @@ import {
taskNotStartableError
} from '../../coordinator/dispatch/task-dispatch-refusal'
import { getFollowUpDispatch, takeFollowUpMail } from '../dispatch-context/worker-follow-up-lineage'
import { refuseClaimedWorkerTerminal, type WorkerTerminalClaim } from './worker-terminal-claim'
export function createStartingWorkerDispatch(
this: OrchestrationDb,
@@ -53,6 +54,8 @@ export function createStartingWorkerDispatch(
acceptedBy?: { callerFingerprint: string; requestId: string }
/** The Dispatch whose unread mail this follow-up's Task tells it to check, moved to it in the claim. */
takesMailOf?: string
/** The existing terminal this start takes; claimed atomically with the Dispatch row. */
claimsTerminal?: WorkerTerminalClaim
}
): { dispatch: DispatchContextRow; worker: WorkerDispatchRow; task: TaskRow } {
this.db.exec('BEGIN IMMEDIATE')
@@ -89,6 +92,9 @@ export function createStartingWorkerDispatch(
{ followUpDispatchId: claimedFollowUp.id }
)
}
if (params.claimsTerminal) {
refuseClaimedWorkerTerminal(this, params.claimsTerminal)
}
const task = params.taskId
? this.getTask(params.taskId)
: params.taskSpec
@@ -0,0 +1,46 @@
import { OrchestrationError } from '../../orchestration-error'
import type { OrchestrationDb } from '../orchestration-db'
import type { DispatchContextRow } from '../orchestration-records'
export type WorkerTerminalClaim = {
handle: string
paneKey?: string
/** A racing follow-up of the same settled Dispatch is left to `follow_up_claimed`, which steers the send into it. */
followUpOf?: string
}
// A starting Dispatch names its terminal only in its start options until its authority attaches.
const STARTING_ON_TERMINAL_SQL = `SELECT d.* FROM dispatch_contexts d
JOIN worker_dispatches w ON w.dispatch_id = d.id
WHERE d.status = 'pending' AND w.state = 'starting'
AND json_extract(w.start_options, '$.terminal') = ?
ORDER BY d.rowid LIMIT 1`
/** Refuse a worker-start or follow-up onto a terminal another open Dispatch holds or is starting on.
* worker-start and the follow-up a send starts both claim through here, so whichever comes second is refused before any wait. */
export function refuseClaimedWorkerTerminal(db: OrchestrationDb, claim: WorkerTerminalClaim): void {
const open =
db.findActiveDispatchForAssignee(claim.handle, claim.paneKey) ??
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: selects whole dispatch_contexts rows.
(db.db.prepare(STARTING_ON_TERMINAL_SQL).get(claim.handle) as DispatchContextRow | undefined)
if (open && !(claim.followUpOf && open.follow_up_of_dispatch_id === claim.followUpOf)) {
throw terminalBusyError(db, claim.handle, open)
}
}
function terminalBusyError(
db: OrchestrationDb,
handle: string,
open: DispatchContextRow
): OrchestrationError {
const task = db.getTask(open.task_id)
const title = task?.task_title ?? task?.spec.split('\n')[0]?.slice(0, 80)
const followUp = open.follow_up_of_dispatch_id
? `, the follow-up of Dispatch ${open.follow_up_of_dispatch_id}`
: ''
return new OrchestrationError(
'terminal_busy',
`Terminal ${handle} is busy: Dispatch ${open.id}${followUp} (task ${open.task_id}${title ? ` "${title}"` : ''}) is still open on it. To answer it or give it more work, use your Orca CLI's \`orchestration send --to dispatch:${open.id}\`; to replace it, run \`orchestration worker-stop --dispatch ${open.id}\` and then start again. No effects were applied.`,
{ effectsApplied: false, dispatchId: open.id, taskId: open.task_id }
)
}
@@ -6,6 +6,10 @@ import type {
OrchestrationPartyIdentity as OrchestrationParty
} from '../../../../../../shared/orchestration/mailbox/orchestration-party-identity'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import {
refuseClaimedWorkerTerminal,
type WorkerTerminalClaim
} from '../../../../orchestration/db/worker-dispatch/worker-terminal-claim'
import type { OrcaSessionId } from '../../../../../../shared/orca-session-address'
import { formatOrcaSessionAddress } from '../../../../../../shared/orca-session-address'
import {
@@ -56,7 +60,8 @@ export async function assertExplicitWorkerTerminalUsable(args: {
}
}
/** A `--terminal` party: a terminal pane, or a chat named by its Orca session ID. */
/** A `--terminal` party: a terminal pane, or a chat named by its Orca session ID.
* Returns the claim its Dispatch insert repeats atomically; refused here first, before a Run binds or anything waits. */
export async function assertExplicitWorkerUsable(args: {
runtime: OrcaRuntimeService
db: OrchestrationDb
@@ -64,11 +69,20 @@ export async function assertExplicitWorkerUsable(args: {
from: string
coordinator: OrchestrationCallerIdentity | null
resolvedWorktreeId: string | undefined
}): Promise<void> {
followUpOf: string | undefined
}): Promise<WorkerTerminalClaim> {
const chat = chatAssigneeOf(args.terminal)
await (chat
? assertExplicitWorkerChatUsable({ ...args, chat })
: assertExplicitWorkerTerminalUsable({ ...args, terminal: args.terminal.address }))
const handle = args.terminal.address
const claim = {
handle,
paneKey: args.runtime.getTerminalPaneKey(handle) ?? undefined,
followUpOf: args.followUpOf
}
refuseClaimedWorkerTerminal(args.db, claim)
return claim
}
/** The same refusals for a chat named by its Orca session ID: itself, unreachable, another worktree. */
@@ -5,6 +5,7 @@ import type { WorkerStartInput } from './worker-start-schema'
import type { RunRow } from '../../../../orchestration/db/orchestration-records'
import type { startLocalWorker } from './local-worker-start'
import { resolveDispatchCreator } from '../runs/dispatch-creator'
import type { WorkerTerminalClaim } from '../../../../orchestration/db/worker-dispatch/worker-terminal-claim'
export function createLocalWorkerStartDispatch(
{
@@ -27,7 +28,8 @@ export function createLocalWorkerStartDispatch(
resolvedWorktreeId,
repoSelector,
agent,
launchReceipt
launchReceipt,
claimsTerminal
}: {
params: WorkerStartInput
run: RunRow
@@ -38,6 +40,7 @@ export function createLocalWorkerStartDispatch(
repoSelector: string | undefined
agent: TuiAgent | undefined
launchReceipt: OrchestrationWorkerLaunchReceipt
claimsTerminal: WorkerTerminalClaim | undefined
}
) {
const startOptions = {
@@ -82,6 +85,7 @@ export function createLocalWorkerStartDispatch(
mutationReceipt: orchestrationMutation,
followUpOf: followUp?.of,
acceptedBy: followUp?.acceptedBy,
takesMailOf: followUp?.takesMailOf
takesMailOf: followUp?.takesMailOf,
claimsTerminal
})
}
@@ -125,16 +125,17 @@ export async function startLocalWorker(args: {
: requestedWorktree === 'current'
? await runtime.showManagedTerminalWorkspace(`id:${coordinatorWorktreeId}`)
: await runtime.showManagedTerminalWorkspace(requestedWorktree)
if (terminal) {
await assertExplicitWorkerUsable({
runtime,
db,
terminal,
from: params.from,
coordinator,
resolvedWorktreeId: resolvedWorktree?.id
})
}
const claimsTerminal = terminal
? await assertExplicitWorkerUsable({
runtime,
db,
terminal,
from: params.from,
coordinator,
resolvedWorktreeId: resolvedWorktree?.id,
followUpOf: args.followUp?.of
})
: undefined
const hostMode = resolveWorkerStartModeOnHost(runtime, args.mode, resolvedWorktree?.id, agent)
let mode = chatWorkerMode(await hostMode, chat)
@@ -148,7 +149,8 @@ export async function startLocalWorker(args: {
resolvedWorktreeId: resolvedWorktree?.id,
repoSelector: creationTarget?.repoSelector,
agent,
launchReceipt: launch.receipt
launchReceipt: launch.receipt,
claimsTerminal
})
releaseRetriedWorkerTerminal(runtime, db, params)
const effects: WorkerEffect[] = []
@@ -0,0 +1,94 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { getFollowUpDispatch } from '../../../../orchestration/db/dispatch-context/worker-follow-up-lineage'
import { getWorkerTerminalAutoReleaseScheduler } from '../../../../orchestration/terminal-authority/release/worker-terminal-auto-release-scheduler'
import { createOrchestrationWorkerReleaseHarness } from './worker-release.test-support'
describe('worker-start onto a terminal another Dispatch holds', () => {
const h = createOrchestrationWorkerReleaseHarness()
afterEach(() => {
getWorkerTerminalAutoReleaseScheduler(h.runtime).clear()
h.cleanup()
})
const rows = () =>
h.db.db
.prepare(
'SELECT (SELECT count(*) FROM tasks) AS tasks, (SELECT count(*) FROM dispatch_contexts) AS dispatches'
)
.get()
const startOnWorker = () =>
h.call('orchestration.workerStart', {
from: 'term_coord',
terminal: 'term_worker',
spec: 'next task'
})
const send = (dispatchId: string) =>
h.call('orchestration.send', {
from: 'term_coord',
to: `dispatch:${dispatchId}`,
subject: 'More',
body: 'and the docs'
})
/** Holds the next readiness wait on the reused terminal until released. */
const holdReadiness = () => {
const held = h.deferred<void>()
const wait = vi.mocked(h.runtime.waitForTerminal).getMockImplementation()
vi.mocked(h.runtime.waitForTerminal).mockImplementationOnce(async (...args) => {
await held.promise
return wait!(...args)
})
return held
}
it('is refused before any Task, Dispatch or readiness wait, naming the open Dispatch', async () => {
h.setup()
const { dispatchId } = await h.startWorker()
const before = rows()
const waits = vi.mocked(h.runtime.waitForTerminal).mock.calls.length
const refusal = await startOnWorker().catch((error: unknown) => error)
expect(refusal).toMatchObject({
code: 'terminal_busy',
data: { effectsApplied: false, dispatchId }
})
expect((refusal as Error).message).toContain(`send --to dispatch:${dispatchId}`)
expect((refusal as Error).message).toContain(`worker-stop --dispatch ${dispatchId}`)
expect(rows()).toEqual(before)
expect(vi.mocked(h.runtime.waitForTerminal).mock.calls).toHaveLength(waits)
})
it('refuses worker-start while a follow-up is starting on the finished worker', async () => {
h.setup()
const { dispatchId } = await h.startSettledWorker()
const held = holdReadiness()
const followUp = send(dispatchId)
await vi.waitFor(() => expect(getFollowUpDispatch(h.db, dispatchId)).toBeDefined())
const followUpId = getFollowUpDispatch(h.db, dispatchId)!.id
await expect(startOnWorker()).rejects.toMatchObject({
code: 'terminal_busy',
data: { effectsApplied: false, dispatchId: followUpId }
})
held.resolve()
await expect(followUp).resolves.toMatchObject({ state: 'ready', dispatchId: followUpId })
})
it('refuses the follow-up a send would start while worker-start is starting there', async () => {
h.setup()
const { dispatchId } = await h.startSettledWorker()
const { dispatches } = rows() as { dispatches: number }
const held = holdReadiness()
const start = startOnWorker()
await vi.waitFor(() => expect(rows()).toMatchObject({ dispatches: dispatches + 1 }))
await expect(send(dispatchId)).rejects.toMatchObject({
code: 'terminal_busy',
data: { effectsApplied: false }
})
expect(getFollowUpDispatch(h.db, dispatchId)).toBeUndefined()
held.resolve()
await expect(start).resolves.toMatchObject({ state: 'ready' })
})
})