From 3ee53c89d550fa3fe3bfbcf2b2012378b712e024 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Sun, 8 Nov 2026 03:34:28 -0500 Subject: [PATCH] fix(orchestration): refuse worker-start on a busy terminal before any wait worker-start --terminal and the follow-up a send to a finished worker starts now claim the terminal through one check, before a Run binds, a Task or Dispatch is written, or readiness is awaited. The Dispatch insert repeats the claim atomically, and the late authority check stays as a backstop. --- .../worker-dispatch/worker-dispatch-start.ts | 6 ++ .../worker-dispatch/worker-terminal-claim.ts | 46 +++++++++ .../explicit-worker-terminal-validation.ts | 18 +++- .../worker/local-worker-start-dispatch.ts | 8 +- .../worker/local-worker-start.ts | 24 ++--- .../worker/worker-start-busy-terminal.test.ts | 94 +++++++++++++++++++ 6 files changed, 181 insertions(+), 15 deletions(-) create mode 100644 src/main/runtime/orchestration/db/worker-dispatch/worker-terminal-claim.ts create mode 100644 src/main/runtime/rpc/methods/orchestration/worker/worker-start-busy-terminal.test.ts diff --git a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts index b65da6e2f61..7a01323841b 100644 --- a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts +++ b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts @@ -12,6 +12,7 @@ import { taskNotStartableError } from '../../coordinator/dispatch/task-dispatch-refusal' import { getFollowUpDispatch, takeFollowUpMail } from '../dispatch-context/worker-follow-up-lineage' +import { refuseClaimedWorkerTerminal, type WorkerTerminalClaim } from './worker-terminal-claim' export function createStartingWorkerDispatch( this: OrchestrationDb, @@ -53,6 +54,8 @@ export function createStartingWorkerDispatch( acceptedBy?: { callerFingerprint: string; requestId: string } /** The Dispatch whose unread mail this follow-up's Task tells it to check, moved to it in the claim. */ takesMailOf?: string + /** The existing terminal this start takes; claimed atomically with the Dispatch row. */ + claimsTerminal?: WorkerTerminalClaim } ): { dispatch: DispatchContextRow; worker: WorkerDispatchRow; task: TaskRow } { this.db.exec('BEGIN IMMEDIATE') @@ -89,6 +92,9 @@ export function createStartingWorkerDispatch( { followUpDispatchId: claimedFollowUp.id } ) } + if (params.claimsTerminal) { + refuseClaimedWorkerTerminal(this, params.claimsTerminal) + } const task = params.taskId ? this.getTask(params.taskId) : params.taskSpec diff --git a/src/main/runtime/orchestration/db/worker-dispatch/worker-terminal-claim.ts b/src/main/runtime/orchestration/db/worker-dispatch/worker-terminal-claim.ts new file mode 100644 index 00000000000..d6945836df5 --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-dispatch/worker-terminal-claim.ts @@ -0,0 +1,46 @@ +import { OrchestrationError } from '../../orchestration-error' +import type { OrchestrationDb } from '../orchestration-db' +import type { DispatchContextRow } from '../orchestration-records' + +export type WorkerTerminalClaim = { + handle: string + paneKey?: string + /** A racing follow-up of the same settled Dispatch is left to `follow_up_claimed`, which steers the send into it. */ + followUpOf?: string +} + +// A starting Dispatch names its terminal only in its start options until its authority attaches. +const STARTING_ON_TERMINAL_SQL = `SELECT d.* FROM dispatch_contexts d + JOIN worker_dispatches w ON w.dispatch_id = d.id + WHERE d.status = 'pending' AND w.state = 'starting' + AND json_extract(w.start_options, '$.terminal') = ? + ORDER BY d.rowid LIMIT 1` + +/** Refuse a worker-start or follow-up onto a terminal another open Dispatch holds or is starting on. + * worker-start and the follow-up a send starts both claim through here, so whichever comes second is refused before any wait. */ +export function refuseClaimedWorkerTerminal(db: OrchestrationDb, claim: WorkerTerminalClaim): void { + const open = + db.findActiveDispatchForAssignee(claim.handle, claim.paneKey) ?? + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: selects whole dispatch_contexts rows. + (db.db.prepare(STARTING_ON_TERMINAL_SQL).get(claim.handle) as DispatchContextRow | undefined) + if (open && !(claim.followUpOf && open.follow_up_of_dispatch_id === claim.followUpOf)) { + throw terminalBusyError(db, claim.handle, open) + } +} + +function terminalBusyError( + db: OrchestrationDb, + handle: string, + open: DispatchContextRow +): OrchestrationError { + const task = db.getTask(open.task_id) + const title = task?.task_title ?? task?.spec.split('\n')[0]?.slice(0, 80) + const followUp = open.follow_up_of_dispatch_id + ? `, the follow-up of Dispatch ${open.follow_up_of_dispatch_id}` + : '' + return new OrchestrationError( + 'terminal_busy', + `Terminal ${handle} is busy: Dispatch ${open.id}${followUp} (task ${open.task_id}${title ? ` "${title}"` : ''}) is still open on it. To answer it or give it more work, use your Orca CLI's \`orchestration send --to dispatch:${open.id}\`; to replace it, run \`orchestration worker-stop --dispatch ${open.id}\` and then start again. No effects were applied.`, + { effectsApplied: false, dispatchId: open.id, taskId: open.task_id } + ) +} diff --git a/src/main/runtime/rpc/methods/orchestration/worker/explicit-worker-terminal-validation.ts b/src/main/runtime/rpc/methods/orchestration/worker/explicit-worker-terminal-validation.ts index a2d53e01de2..d5918f77a60 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/explicit-worker-terminal-validation.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/explicit-worker-terminal-validation.ts @@ -6,6 +6,10 @@ import type { OrchestrationPartyIdentity as OrchestrationParty } from '../../../../../../shared/orchestration/mailbox/orchestration-party-identity' import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db' +import { + refuseClaimedWorkerTerminal, + type WorkerTerminalClaim +} from '../../../../orchestration/db/worker-dispatch/worker-terminal-claim' import type { OrcaSessionId } from '../../../../../../shared/orca-session-address' import { formatOrcaSessionAddress } from '../../../../../../shared/orca-session-address' import { @@ -56,7 +60,8 @@ export async function assertExplicitWorkerTerminalUsable(args: { } } -/** A `--terminal` party: a terminal pane, or a chat named by its Orca session ID. */ +/** A `--terminal` party: a terminal pane, or a chat named by its Orca session ID. + * Returns the claim its Dispatch insert repeats atomically; refused here first, before a Run binds or anything waits. */ export async function assertExplicitWorkerUsable(args: { runtime: OrcaRuntimeService db: OrchestrationDb @@ -64,11 +69,20 @@ export async function assertExplicitWorkerUsable(args: { from: string coordinator: OrchestrationCallerIdentity | null resolvedWorktreeId: string | undefined -}): Promise { + followUpOf: string | undefined +}): Promise { const chat = chatAssigneeOf(args.terminal) await (chat ? assertExplicitWorkerChatUsable({ ...args, chat }) : assertExplicitWorkerTerminalUsable({ ...args, terminal: args.terminal.address })) + const handle = args.terminal.address + const claim = { + handle, + paneKey: args.runtime.getTerminalPaneKey(handle) ?? undefined, + followUpOf: args.followUpOf + } + refuseClaimedWorkerTerminal(args.db, claim) + return claim } /** The same refusals for a chat named by its Orca session ID: itself, unreachable, another worktree. */ diff --git a/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start-dispatch.ts b/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start-dispatch.ts index 5d26db76cb2..e336eb99252 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start-dispatch.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start-dispatch.ts @@ -5,6 +5,7 @@ import type { WorkerStartInput } from './worker-start-schema' import type { RunRow } from '../../../../orchestration/db/orchestration-records' import type { startLocalWorker } from './local-worker-start' import { resolveDispatchCreator } from '../runs/dispatch-creator' +import type { WorkerTerminalClaim } from '../../../../orchestration/db/worker-dispatch/worker-terminal-claim' export function createLocalWorkerStartDispatch( { @@ -27,7 +28,8 @@ export function createLocalWorkerStartDispatch( resolvedWorktreeId, repoSelector, agent, - launchReceipt + launchReceipt, + claimsTerminal }: { params: WorkerStartInput run: RunRow @@ -38,6 +40,7 @@ export function createLocalWorkerStartDispatch( repoSelector: string | undefined agent: TuiAgent | undefined launchReceipt: OrchestrationWorkerLaunchReceipt + claimsTerminal: WorkerTerminalClaim | undefined } ) { const startOptions = { @@ -82,6 +85,7 @@ export function createLocalWorkerStartDispatch( mutationReceipt: orchestrationMutation, followUpOf: followUp?.of, acceptedBy: followUp?.acceptedBy, - takesMailOf: followUp?.takesMailOf + takesMailOf: followUp?.takesMailOf, + claimsTerminal }) } diff --git a/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts b/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts index 4e718de9a00..b70b8e072a9 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts @@ -125,16 +125,17 @@ export async function startLocalWorker(args: { : requestedWorktree === 'current' ? await runtime.showManagedTerminalWorkspace(`id:${coordinatorWorktreeId}`) : await runtime.showManagedTerminalWorkspace(requestedWorktree) - if (terminal) { - await assertExplicitWorkerUsable({ - runtime, - db, - terminal, - from: params.from, - coordinator, - resolvedWorktreeId: resolvedWorktree?.id - }) - } + const claimsTerminal = terminal + ? await assertExplicitWorkerUsable({ + runtime, + db, + terminal, + from: params.from, + coordinator, + resolvedWorktreeId: resolvedWorktree?.id, + followUpOf: args.followUp?.of + }) + : undefined const hostMode = resolveWorkerStartModeOnHost(runtime, args.mode, resolvedWorktree?.id, agent) let mode = chatWorkerMode(await hostMode, chat) @@ -148,7 +149,8 @@ export async function startLocalWorker(args: { resolvedWorktreeId: resolvedWorktree?.id, repoSelector: creationTarget?.repoSelector, agent, - launchReceipt: launch.receipt + launchReceipt: launch.receipt, + claimsTerminal }) releaseRetriedWorkerTerminal(runtime, db, params) const effects: WorkerEffect[] = [] diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-busy-terminal.test.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-busy-terminal.test.ts new file mode 100644 index 00000000000..a10cd6abab8 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-busy-terminal.test.ts @@ -0,0 +1,94 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { getFollowUpDispatch } from '../../../../orchestration/db/dispatch-context/worker-follow-up-lineage' +import { getWorkerTerminalAutoReleaseScheduler } from '../../../../orchestration/terminal-authority/release/worker-terminal-auto-release-scheduler' +import { createOrchestrationWorkerReleaseHarness } from './worker-release.test-support' + +describe('worker-start onto a terminal another Dispatch holds', () => { + const h = createOrchestrationWorkerReleaseHarness() + + afterEach(() => { + getWorkerTerminalAutoReleaseScheduler(h.runtime).clear() + h.cleanup() + }) + + const rows = () => + h.db.db + .prepare( + 'SELECT (SELECT count(*) FROM tasks) AS tasks, (SELECT count(*) FROM dispatch_contexts) AS dispatches' + ) + .get() + const startOnWorker = () => + h.call('orchestration.workerStart', { + from: 'term_coord', + terminal: 'term_worker', + spec: 'next task' + }) + const send = (dispatchId: string) => + h.call('orchestration.send', { + from: 'term_coord', + to: `dispatch:${dispatchId}`, + subject: 'More', + body: 'and the docs' + }) + /** Holds the next readiness wait on the reused terminal until released. */ + const holdReadiness = () => { + const held = h.deferred() + const wait = vi.mocked(h.runtime.waitForTerminal).getMockImplementation() + vi.mocked(h.runtime.waitForTerminal).mockImplementationOnce(async (...args) => { + await held.promise + return wait!(...args) + }) + return held + } + + it('is refused before any Task, Dispatch or readiness wait, naming the open Dispatch', async () => { + h.setup() + const { dispatchId } = await h.startWorker() + const before = rows() + const waits = vi.mocked(h.runtime.waitForTerminal).mock.calls.length + + const refusal = await startOnWorker().catch((error: unknown) => error) + + expect(refusal).toMatchObject({ + code: 'terminal_busy', + data: { effectsApplied: false, dispatchId } + }) + expect((refusal as Error).message).toContain(`send --to dispatch:${dispatchId}`) + expect((refusal as Error).message).toContain(`worker-stop --dispatch ${dispatchId}`) + expect(rows()).toEqual(before) + expect(vi.mocked(h.runtime.waitForTerminal).mock.calls).toHaveLength(waits) + }) + + it('refuses worker-start while a follow-up is starting on the finished worker', async () => { + h.setup() + const { dispatchId } = await h.startSettledWorker() + const held = holdReadiness() + const followUp = send(dispatchId) + await vi.waitFor(() => expect(getFollowUpDispatch(h.db, dispatchId)).toBeDefined()) + const followUpId = getFollowUpDispatch(h.db, dispatchId)!.id + + await expect(startOnWorker()).rejects.toMatchObject({ + code: 'terminal_busy', + data: { effectsApplied: false, dispatchId: followUpId } + }) + held.resolve() + await expect(followUp).resolves.toMatchObject({ state: 'ready', dispatchId: followUpId }) + }) + + it('refuses the follow-up a send would start while worker-start is starting there', async () => { + h.setup() + const { dispatchId } = await h.startSettledWorker() + const { dispatches } = rows() as { dispatches: number } + const held = holdReadiness() + const start = startOnWorker() + await vi.waitFor(() => expect(rows()).toMatchObject({ dispatches: dispatches + 1 })) + + await expect(send(dispatchId)).rejects.toMatchObject({ + code: 'terminal_busy', + data: { effectsApplied: false } + }) + expect(getFollowUpDispatch(h.db, dispatchId)).toBeUndefined() + held.resolve() + await expect(start).resolves.toMatchObject({ state: 'ready' }) + }) +})