mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 19:07:41 +00:00
fix(orchestration): refuse worker-start on a busy terminal before any wait
worker-start --terminal and the follow-up a send to a finished worker starts now claim the terminal through one check, before a Run binds, a Task or Dispatch is written, or readiness is awaited. The Dispatch insert repeats the claim atomically, and the late authority check stays as a backstop.
This commit is contained in:
@@ -12,6 +12,7 @@ import {
|
||||
taskNotStartableError
|
||||
} from '../../coordinator/dispatch/task-dispatch-refusal'
|
||||
import { getFollowUpDispatch, takeFollowUpMail } from '../dispatch-context/worker-follow-up-lineage'
|
||||
import { refuseClaimedWorkerTerminal, type WorkerTerminalClaim } from './worker-terminal-claim'
|
||||
|
||||
export function createStartingWorkerDispatch(
|
||||
this: OrchestrationDb,
|
||||
@@ -53,6 +54,8 @@ export function createStartingWorkerDispatch(
|
||||
acceptedBy?: { callerFingerprint: string; requestId: string }
|
||||
/** The Dispatch whose unread mail this follow-up's Task tells it to check, moved to it in the claim. */
|
||||
takesMailOf?: string
|
||||
/** The existing terminal this start takes; claimed atomically with the Dispatch row. */
|
||||
claimsTerminal?: WorkerTerminalClaim
|
||||
}
|
||||
): { dispatch: DispatchContextRow; worker: WorkerDispatchRow; task: TaskRow } {
|
||||
this.db.exec('BEGIN IMMEDIATE')
|
||||
@@ -89,6 +92,9 @@ export function createStartingWorkerDispatch(
|
||||
{ followUpDispatchId: claimedFollowUp.id }
|
||||
)
|
||||
}
|
||||
if (params.claimsTerminal) {
|
||||
refuseClaimedWorkerTerminal(this, params.claimsTerminal)
|
||||
}
|
||||
const task = params.taskId
|
||||
? this.getTask(params.taskId)
|
||||
: params.taskSpec
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { OrchestrationError } from '../../orchestration-error'
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
import type { DispatchContextRow } from '../orchestration-records'
|
||||
|
||||
export type WorkerTerminalClaim = {
|
||||
handle: string
|
||||
paneKey?: string
|
||||
/** A racing follow-up of the same settled Dispatch is left to `follow_up_claimed`, which steers the send into it. */
|
||||
followUpOf?: string
|
||||
}
|
||||
|
||||
// A starting Dispatch names its terminal only in its start options until its authority attaches.
|
||||
const STARTING_ON_TERMINAL_SQL = `SELECT d.* FROM dispatch_contexts d
|
||||
JOIN worker_dispatches w ON w.dispatch_id = d.id
|
||||
WHERE d.status = 'pending' AND w.state = 'starting'
|
||||
AND json_extract(w.start_options, '$.terminal') = ?
|
||||
ORDER BY d.rowid LIMIT 1`
|
||||
|
||||
/** Refuse a worker-start or follow-up onto a terminal another open Dispatch holds or is starting on.
|
||||
* worker-start and the follow-up a send starts both claim through here, so whichever comes second is refused before any wait. */
|
||||
export function refuseClaimedWorkerTerminal(db: OrchestrationDb, claim: WorkerTerminalClaim): void {
|
||||
const open =
|
||||
db.findActiveDispatchForAssignee(claim.handle, claim.paneKey) ??
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: selects whole dispatch_contexts rows.
|
||||
(db.db.prepare(STARTING_ON_TERMINAL_SQL).get(claim.handle) as DispatchContextRow | undefined)
|
||||
if (open && !(claim.followUpOf && open.follow_up_of_dispatch_id === claim.followUpOf)) {
|
||||
throw terminalBusyError(db, claim.handle, open)
|
||||
}
|
||||
}
|
||||
|
||||
function terminalBusyError(
|
||||
db: OrchestrationDb,
|
||||
handle: string,
|
||||
open: DispatchContextRow
|
||||
): OrchestrationError {
|
||||
const task = db.getTask(open.task_id)
|
||||
const title = task?.task_title ?? task?.spec.split('\n')[0]?.slice(0, 80)
|
||||
const followUp = open.follow_up_of_dispatch_id
|
||||
? `, the follow-up of Dispatch ${open.follow_up_of_dispatch_id}`
|
||||
: ''
|
||||
return new OrchestrationError(
|
||||
'terminal_busy',
|
||||
`Terminal ${handle} is busy: Dispatch ${open.id}${followUp} (task ${open.task_id}${title ? ` "${title}"` : ''}) is still open on it. To answer it or give it more work, use your Orca CLI's \`orchestration send --to dispatch:${open.id}\`; to replace it, run \`orchestration worker-stop --dispatch ${open.id}\` and then start again. No effects were applied.`,
|
||||
{ effectsApplied: false, dispatchId: open.id, taskId: open.task_id }
|
||||
)
|
||||
}
|
||||
+16
-2
@@ -6,6 +6,10 @@ import type {
|
||||
OrchestrationPartyIdentity as OrchestrationParty
|
||||
} from '../../../../../../shared/orchestration/mailbox/orchestration-party-identity'
|
||||
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
|
||||
import {
|
||||
refuseClaimedWorkerTerminal,
|
||||
type WorkerTerminalClaim
|
||||
} from '../../../../orchestration/db/worker-dispatch/worker-terminal-claim'
|
||||
import type { OrcaSessionId } from '../../../../../../shared/orca-session-address'
|
||||
import { formatOrcaSessionAddress } from '../../../../../../shared/orca-session-address'
|
||||
import {
|
||||
@@ -56,7 +60,8 @@ export async function assertExplicitWorkerTerminalUsable(args: {
|
||||
}
|
||||
}
|
||||
|
||||
/** A `--terminal` party: a terminal pane, or a chat named by its Orca session ID. */
|
||||
/** A `--terminal` party: a terminal pane, or a chat named by its Orca session ID.
|
||||
* Returns the claim its Dispatch insert repeats atomically; refused here first, before a Run binds or anything waits. */
|
||||
export async function assertExplicitWorkerUsable(args: {
|
||||
runtime: OrcaRuntimeService
|
||||
db: OrchestrationDb
|
||||
@@ -64,11 +69,20 @@ export async function assertExplicitWorkerUsable(args: {
|
||||
from: string
|
||||
coordinator: OrchestrationCallerIdentity | null
|
||||
resolvedWorktreeId: string | undefined
|
||||
}): Promise<void> {
|
||||
followUpOf: string | undefined
|
||||
}): Promise<WorkerTerminalClaim> {
|
||||
const chat = chatAssigneeOf(args.terminal)
|
||||
await (chat
|
||||
? assertExplicitWorkerChatUsable({ ...args, chat })
|
||||
: assertExplicitWorkerTerminalUsable({ ...args, terminal: args.terminal.address }))
|
||||
const handle = args.terminal.address
|
||||
const claim = {
|
||||
handle,
|
||||
paneKey: args.runtime.getTerminalPaneKey(handle) ?? undefined,
|
||||
followUpOf: args.followUpOf
|
||||
}
|
||||
refuseClaimedWorkerTerminal(args.db, claim)
|
||||
return claim
|
||||
}
|
||||
|
||||
/** The same refusals for a chat named by its Orca session ID: itself, unreachable, another worktree. */
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { WorkerStartInput } from './worker-start-schema'
|
||||
import type { RunRow } from '../../../../orchestration/db/orchestration-records'
|
||||
import type { startLocalWorker } from './local-worker-start'
|
||||
import { resolveDispatchCreator } from '../runs/dispatch-creator'
|
||||
import type { WorkerTerminalClaim } from '../../../../orchestration/db/worker-dispatch/worker-terminal-claim'
|
||||
|
||||
export function createLocalWorkerStartDispatch(
|
||||
{
|
||||
@@ -27,7 +28,8 @@ export function createLocalWorkerStartDispatch(
|
||||
resolvedWorktreeId,
|
||||
repoSelector,
|
||||
agent,
|
||||
launchReceipt
|
||||
launchReceipt,
|
||||
claimsTerminal
|
||||
}: {
|
||||
params: WorkerStartInput
|
||||
run: RunRow
|
||||
@@ -38,6 +40,7 @@ export function createLocalWorkerStartDispatch(
|
||||
repoSelector: string | undefined
|
||||
agent: TuiAgent | undefined
|
||||
launchReceipt: OrchestrationWorkerLaunchReceipt
|
||||
claimsTerminal: WorkerTerminalClaim | undefined
|
||||
}
|
||||
) {
|
||||
const startOptions = {
|
||||
@@ -82,6 +85,7 @@ export function createLocalWorkerStartDispatch(
|
||||
mutationReceipt: orchestrationMutation,
|
||||
followUpOf: followUp?.of,
|
||||
acceptedBy: followUp?.acceptedBy,
|
||||
takesMailOf: followUp?.takesMailOf
|
||||
takesMailOf: followUp?.takesMailOf,
|
||||
claimsTerminal
|
||||
})
|
||||
}
|
||||
|
||||
@@ -125,16 +125,17 @@ export async function startLocalWorker(args: {
|
||||
: requestedWorktree === 'current'
|
||||
? await runtime.showManagedTerminalWorkspace(`id:${coordinatorWorktreeId}`)
|
||||
: await runtime.showManagedTerminalWorkspace(requestedWorktree)
|
||||
if (terminal) {
|
||||
await assertExplicitWorkerUsable({
|
||||
runtime,
|
||||
db,
|
||||
terminal,
|
||||
from: params.from,
|
||||
coordinator,
|
||||
resolvedWorktreeId: resolvedWorktree?.id
|
||||
})
|
||||
}
|
||||
const claimsTerminal = terminal
|
||||
? await assertExplicitWorkerUsable({
|
||||
runtime,
|
||||
db,
|
||||
terminal,
|
||||
from: params.from,
|
||||
coordinator,
|
||||
resolvedWorktreeId: resolvedWorktree?.id,
|
||||
followUpOf: args.followUp?.of
|
||||
})
|
||||
: undefined
|
||||
const hostMode = resolveWorkerStartModeOnHost(runtime, args.mode, resolvedWorktree?.id, agent)
|
||||
let mode = chatWorkerMode(await hostMode, chat)
|
||||
|
||||
@@ -148,7 +149,8 @@ export async function startLocalWorker(args: {
|
||||
resolvedWorktreeId: resolvedWorktree?.id,
|
||||
repoSelector: creationTarget?.repoSelector,
|
||||
agent,
|
||||
launchReceipt: launch.receipt
|
||||
launchReceipt: launch.receipt,
|
||||
claimsTerminal
|
||||
})
|
||||
releaseRetriedWorkerTerminal(runtime, db, params)
|
||||
const effects: WorkerEffect[] = []
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { getFollowUpDispatch } from '../../../../orchestration/db/dispatch-context/worker-follow-up-lineage'
|
||||
import { getWorkerTerminalAutoReleaseScheduler } from '../../../../orchestration/terminal-authority/release/worker-terminal-auto-release-scheduler'
|
||||
import { createOrchestrationWorkerReleaseHarness } from './worker-release.test-support'
|
||||
|
||||
describe('worker-start onto a terminal another Dispatch holds', () => {
|
||||
const h = createOrchestrationWorkerReleaseHarness()
|
||||
|
||||
afterEach(() => {
|
||||
getWorkerTerminalAutoReleaseScheduler(h.runtime).clear()
|
||||
h.cleanup()
|
||||
})
|
||||
|
||||
const rows = () =>
|
||||
h.db.db
|
||||
.prepare(
|
||||
'SELECT (SELECT count(*) FROM tasks) AS tasks, (SELECT count(*) FROM dispatch_contexts) AS dispatches'
|
||||
)
|
||||
.get()
|
||||
const startOnWorker = () =>
|
||||
h.call('orchestration.workerStart', {
|
||||
from: 'term_coord',
|
||||
terminal: 'term_worker',
|
||||
spec: 'next task'
|
||||
})
|
||||
const send = (dispatchId: string) =>
|
||||
h.call('orchestration.send', {
|
||||
from: 'term_coord',
|
||||
to: `dispatch:${dispatchId}`,
|
||||
subject: 'More',
|
||||
body: 'and the docs'
|
||||
})
|
||||
/** Holds the next readiness wait on the reused terminal until released. */
|
||||
const holdReadiness = () => {
|
||||
const held = h.deferred<void>()
|
||||
const wait = vi.mocked(h.runtime.waitForTerminal).getMockImplementation()
|
||||
vi.mocked(h.runtime.waitForTerminal).mockImplementationOnce(async (...args) => {
|
||||
await held.promise
|
||||
return wait!(...args)
|
||||
})
|
||||
return held
|
||||
}
|
||||
|
||||
it('is refused before any Task, Dispatch or readiness wait, naming the open Dispatch', async () => {
|
||||
h.setup()
|
||||
const { dispatchId } = await h.startWorker()
|
||||
const before = rows()
|
||||
const waits = vi.mocked(h.runtime.waitForTerminal).mock.calls.length
|
||||
|
||||
const refusal = await startOnWorker().catch((error: unknown) => error)
|
||||
|
||||
expect(refusal).toMatchObject({
|
||||
code: 'terminal_busy',
|
||||
data: { effectsApplied: false, dispatchId }
|
||||
})
|
||||
expect((refusal as Error).message).toContain(`send --to dispatch:${dispatchId}`)
|
||||
expect((refusal as Error).message).toContain(`worker-stop --dispatch ${dispatchId}`)
|
||||
expect(rows()).toEqual(before)
|
||||
expect(vi.mocked(h.runtime.waitForTerminal).mock.calls).toHaveLength(waits)
|
||||
})
|
||||
|
||||
it('refuses worker-start while a follow-up is starting on the finished worker', async () => {
|
||||
h.setup()
|
||||
const { dispatchId } = await h.startSettledWorker()
|
||||
const held = holdReadiness()
|
||||
const followUp = send(dispatchId)
|
||||
await vi.waitFor(() => expect(getFollowUpDispatch(h.db, dispatchId)).toBeDefined())
|
||||
const followUpId = getFollowUpDispatch(h.db, dispatchId)!.id
|
||||
|
||||
await expect(startOnWorker()).rejects.toMatchObject({
|
||||
code: 'terminal_busy',
|
||||
data: { effectsApplied: false, dispatchId: followUpId }
|
||||
})
|
||||
held.resolve()
|
||||
await expect(followUp).resolves.toMatchObject({ state: 'ready', dispatchId: followUpId })
|
||||
})
|
||||
|
||||
it('refuses the follow-up a send would start while worker-start is starting there', async () => {
|
||||
h.setup()
|
||||
const { dispatchId } = await h.startSettledWorker()
|
||||
const { dispatches } = rows() as { dispatches: number }
|
||||
const held = holdReadiness()
|
||||
const start = startOnWorker()
|
||||
await vi.waitFor(() => expect(rows()).toMatchObject({ dispatches: dispatches + 1 }))
|
||||
|
||||
await expect(send(dispatchId)).rejects.toMatchObject({
|
||||
code: 'terminal_busy',
|
||||
data: { effectsApplied: false }
|
||||
})
|
||||
expect(getFollowUpDispatch(h.db, dispatchId)).toBeUndefined()
|
||||
held.resolve()
|
||||
await expect(start).resolves.toMatchObject({ state: 'ready' })
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user