fix(orchestration): expose unsupervised dispatch lanes (#15105)

This commit is contained in:
Jinwoo Hong
2026-08-17 13:53:26 -07:00
committed by GitHub
parent 32ee3b0536
commit 0bedeea642
22 changed files with 600 additions and 67 deletions
+2
View File
@@ -270,6 +270,8 @@ Recovery is conditional, never a fixed destructive sequence:
Low-level `worktree create`, `terminal create`, and `dispatch --inject` remain valid recipes for custom argv or topology that `worker-start` does not express.
`dispatch --inject` deliberately keeps an operator-started terminal unsupervised: it never creates a `worker_dispatches` row and `worker-stop`/`worker-abandon` never close that process. The dispatch context is still authoritative, so `worker-show`, `worker-read`, and `worker-list` report it as `unsupervised`; settled `worker-retain` and `worker-release` report `retained` with `no_owned_resource` and take no process action. Use `worker-start --terminal <handle>` when supervision and worker lifecycle state are required.
## Gates And Legacy Inspection
```bash
File diff suppressed because one or more lines are too long
+9 -2
View File
@@ -42,7 +42,10 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [
path: ['orchestration', 'worker-show'],
summary: 'Inspect one supervised worker Dispatch',
usage: 'orca orchestration worker-show --dispatch <dispatch_id> [--json]',
allowedFlags: [...GLOBAL_FLAGS, 'dispatch']
allowedFlags: [...GLOBAL_FLAGS, 'dispatch'],
notes: [
'A Dispatch created by orchestration dispatch is shown as unsupervised and reports the exact adopted terminal when its identity is still provable.'
]
},
{
path: ['orchestration', 'worker-read'],
@@ -52,6 +55,7 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [
allowedFlags: [...GLOBAL_FLAGS, 'dispatch', 'source', 'cursor', 'limit'],
notes: [
'The default auto source uses an exact hook-reported transcript when available and otherwise returns labeled terminal output.',
'A Dispatch created by orchestration dispatch reads from its adopted terminal with worker status unsupervised.',
'A returned cursor is pinned to the exact source; start a fresh read if Orca reports source_changed.'
]
},
@@ -82,6 +86,7 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [
allowedFlags: [...GLOBAL_FLAGS, 'dispatch', 'retry-request'],
notes: [
'Post-completion cleanup for a settled (succeeded or failed) worker; closes only the exact coordinator-owned agent terminal of that worker.',
'A settled Dispatch created by orchestration dispatch has no owned terminal resource and is reported retained without process action.',
'An inspectable output archive is preserved before the terminal closes, so worker-read still returns output afterwards.',
'Never closes setup terminals, configured tabs, reused or pre-existing terminals, user-taken-over terminals, or unproven identities.',
'Idempotent: repeating the call reports already_released. Only release_unknown exits 1; retained, release_pending, and already_released exit 0.'
@@ -95,6 +100,7 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [
allowedFlags: [...GLOBAL_FLAGS, 'dispatch', 'retry-request'],
notes: [
'Records a durable user-requested exception; a later explicit worker-release clears it and releases the terminal.',
'A settled Dispatch created by orchestration dispatch has no owned terminal resource and is reported retained without process action.',
'Performs no process or filesystem action.'
]
},
@@ -105,7 +111,8 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [
'orca orchestration worker-list [--run <run_id>] [--terminal-state <active|reclaimable|retained|release_pending|release_unknown|released>] [--json]',
allowedFlags: [...GLOBAL_FLAGS, 'run', 'terminal-state'],
notes: [
'Terminal state is process accounting and is reported separately from Task status; a completed Task can still own a live terminal.'
'Terminal state is process accounting and is reported separately from Task status; a completed Task can still own a live terminal.',
'Context-only Dispatches created by orchestration dispatch are included as unsupervised with terminal state retained.'
]
}
]
@@ -9,6 +9,19 @@ export type ContextOnlyDispatchReleaseResult = {
releasedCurrentTask: boolean
}
export function contextOnlyAbandonWarning(result: {
state: string
alreadySettled: boolean
releasedCurrentTask: boolean
}): string {
if (result.alreadySettled) {
return `Dispatch was already ${result.state}; no state or process changed.`
}
return result.releasedCurrentTask
? 'The assignment was abandoned; its unsupervised terminal process was retained.'
: 'The superseded assignment was abandoned without changing the current Task or terminal process.'
}
export function releaseContextOnlyDispatch(
db: Database.Database,
dispatch: DispatchContextRow,
@@ -160,7 +160,7 @@ describe('Task/Dispatch invariant transactions', () => {
sqliteFor(db).prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(task.id)
const second = db.createDispatchContext(task.id, 'term_second')
expect(db.beginWorkerStop(second.id)).toMatchObject({
expect(db.beginWorkerStop(second.id, 'runtime_test')).toMatchObject({
disposition: 'context_only',
releasedCurrentTask: false
})
@@ -228,7 +228,7 @@ describe('Task/Dispatch lifecycle guards', () => {
const released =
operation === 'stop'
? database.beginWorkerStop(contextOnly.id)
? database.beginWorkerStop(contextOnly.id, 'runtime_test')
: database.abandonWorkerDispatch(contextOnly.id)
expect(released).toMatchObject({
disposition: 'context_only',
@@ -257,7 +257,9 @@ describe('Task/Dispatch lifecycle guards', () => {
const released = startWorker(database, task.id, `${operation}_released`)
if (operation === 'stop') {
expect(database.beginWorkerStop(released.dispatchId).disposition).toBe('stopping')
expect(database.beginWorkerStop(released.dispatchId, 'runtime_test').disposition).toBe(
'stopping'
)
expect(database.settleWorkerStop(released.dispatchId).state).toBe('stopped')
} else {
expect(database.abandonWorkerDispatch(released.dispatchId).disposition).toBe('abandoned')
@@ -286,7 +288,9 @@ describe('Task/Dispatch lifecycle guards', () => {
sqliteFor(database).prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(task.id)
const abandoned = startWorker(database, task.id, 'interleaved_abandoned')
expect(database.beginWorkerStop(stopping.dispatchId).disposition).toBe('stopping')
expect(database.beginWorkerStop(stopping.dispatchId, 'runtime_test').disposition).toBe(
'stopping'
)
expect(database.abandonWorkerDispatch(abandoned.dispatchId).disposition).toBe('abandoned')
expect(database.getTask(task.id)?.status).toBe('dispatched')
@@ -305,7 +309,9 @@ describe('Task/Dispatch lifecycle guards', () => {
database.markWorkerStartUnknown(uncertain.dispatch.id, 'agent_readiness', 'outcome unknown')
expect(database.getTask(task.id)?.status).toBe('blocked')
expect(database.beginWorkerStop(uncertain.dispatch.id).disposition).toBe('stopping')
expect(database.beginWorkerStop(uncertain.dispatch.id, 'runtime_test').disposition).toBe(
'stopping'
)
expect(database.settleWorkerStop(uncertain.dispatch.id).state).toBe('stopped')
expect(database.getTask(task.id)?.status).toBe('dispatched')
expect(
@@ -39,6 +39,12 @@ export function abandonWorkerDispatch(
this.db.exec('COMMIT')
return { disposition: 'stale', worker }
}
if (worker.state === 'stopping') {
throw new OrchestrationError(
'dispatch_inactive',
`Dispatch ${dispatchId} is stopping; wait for worker-stop to settle before abandoning.`
)
}
if (worker.state === 'succeeded') {
throw new OrchestrationError(
'dispatch_inactive',
@@ -28,7 +28,8 @@ export function isDispatchProcessCurrent(
export function beginWorkerStop(
this: OrchestrationDb,
dispatchId: string
dispatchId: string,
runtimeEpoch: string
):
| { disposition: 'stopping'; worker: WorkerDispatchRow; dispatch: DispatchContextRow }
| { disposition: 'already_settled'; worker: WorkerDispatchRow; dispatch: DispatchContextRow }
@@ -61,10 +62,11 @@ export function beginWorkerStop(
this.db
.prepare(
`UPDATE worker_dispatches
SET state = 'stopping', stage = 'stop_requested', updated_at = datetime('now')
SET state = 'stopping', stage = 'stop_requested',
runtime_epoch = COALESCE(?, runtime_epoch), updated_at = datetime('now')
WHERE dispatch_id = ? AND state IN ('ready', 'start_unknown')`
)
.run(dispatchId)
.run(runtimeEpoch, dispatchId)
this.db
.prepare(
`UPDATE dispatch_contexts
@@ -150,6 +150,17 @@ export function retainWorkerTerminalResource(
| { disposition: 'no_owned_resource'; resource: null } {
this.db.exec('BEGIN IMMEDIATE')
try {
const dispatch = this.getDispatchContextById(dispatchId)
if (!dispatch) {
throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`)
}
const worker = this.getWorkerDispatch(dispatchId)
if (!worker && !['completed', 'failed', 'circuit_broken'].includes(dispatch.status)) {
throw new OrchestrationError(
'dispatch_inactive',
`Dispatch ${dispatchId} is ${dispatch.status}; only a settled dispatch can retain.`
)
}
const resource = this.getWorkerTerminalResourceByOwner(dispatchId)
if (!resource) {
this.db.exec('COMMIT')
@@ -1,6 +1,7 @@
import type { DispatchStatus, WorkerDispatchState } from '../../types'
import type { DispatchStatus } from '../../types'
import { deriveWorkerTerminalListState } from '../../worker-terminal-ownership'
import type {
WorkerDispatchListState,
WorkerTerminalResourceRow,
WorkerTerminalListState
} from '../../worker-terminal-ownership'
@@ -16,7 +17,11 @@ export function markWorkerTerminalUserOwned(this: OrchestrationDb, paneKey: stri
.prepare(
`SELECT id, owner_dispatch_id, pane_key FROM worker_terminal_resources
WHERE pane_key = ? AND ownership_state = 'owned'
AND release_state IN ('not_requested', 'retained', 'requested')`
AND release_state IN ('not_requested', 'retained', 'requested')
AND NOT EXISTS (
SELECT 1 FROM worker_dispatches w
WHERE w.dispatch_id = owner_dispatch_id AND w.state = 'stopping'
)`
)
.all(paneKey) as { id: string; owner_dispatch_id: string; pane_key: string }[]
const candidates =
@@ -28,6 +33,10 @@ export function markWorkerTerminalUserOwned(this: OrchestrationDb, paneKey: stri
`SELECT id, owner_dispatch_id, pane_key FROM worker_terminal_resources
WHERE ownership_state = 'owned'
AND release_state IN ('not_requested', 'retained', 'requested')
AND NOT EXISTS (
SELECT 1 FROM worker_dispatches w
WHERE w.dispatch_id = owner_dispatch_id AND w.state = 'stopping'
)
AND pane_key IS NOT NULL`
)
.all() as { id: string; owner_dispatch_id: string; pane_key: string }[]
@@ -37,7 +46,11 @@ export function markWorkerTerminalUserOwned(this: OrchestrationDb, paneKey: stri
SET ownership_state = 'user_owned', release_state = 'retained',
retained_reason = 'user_takeover', updated_at = datetime('now')
WHERE id = ? AND ownership_state = 'owned'
AND release_state IN ('not_requested', 'retained', 'requested')`
AND release_state IN ('not_requested', 'retained', 'requested')
AND NOT EXISTS (
SELECT 1 FROM worker_dispatches w
WHERE w.dispatch_id = owner_dispatch_id AND w.state = 'stopping'
)`
)
let changed = 0
for (const candidate of candidates) {
@@ -76,7 +89,7 @@ export function listWorkerTerminalResources(
dispatchId: string
taskId: string
runId: string
workerState: WorkerDispatchState
workerState: WorkerDispatchListState
dispatchStatus: DispatchStatus
agentTerminalHandle: string | null
terminalState: WorkerTerminalListState | null
@@ -84,16 +97,18 @@ export function listWorkerTerminalResources(
}[] {
const rows = this.db
.prepare(
`SELECT w.dispatch_id, w.state AS worker_state, w.agent_terminal_handle,
`SELECT d.id AS dispatch_id,
COALESCE(w.state, 'unsupervised') AS worker_state,
COALESCE(w.agent_terminal_handle, d.assignee_handle) AS agent_terminal_handle,
d.task_id, d.run_id, d.status AS dispatch_status
FROM worker_dispatches w
JOIN dispatch_contexts d ON d.id = w.dispatch_id
FROM dispatch_contexts d
LEFT JOIN worker_dispatches w ON w.dispatch_id = d.id
${params.runId ? 'WHERE d.run_id = ?' : ''}
ORDER BY w.created_at ASC`
ORDER BY COALESCE(w.created_at, d.created_at) ASC`
)
.all(...(params.runId ? [params.runId] : [])) as {
dispatch_id: string
worker_state: WorkerDispatchState
worker_state: WorkerDispatchListState
agent_terminal_handle: string | null
task_id: string
run_id: string
@@ -20,10 +20,21 @@ export function requestWorkerTerminalRelease(
} {
this.db.exec('BEGIN IMMEDIATE')
try {
const dispatch = this.getDispatchContextById(dispatchId)
const worker = this.getWorkerDispatch(dispatchId)
if (!worker) {
if (!dispatch) {
throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`)
}
if (!worker) {
if (!['completed', 'failed', 'circuit_broken'].includes(dispatch.status)) {
throw new OrchestrationError(
'dispatch_inactive',
`Dispatch ${dispatchId} is ${dispatch.status}; only a settled dispatch can release.`
)
}
this.db.exec('COMMIT')
return { disposition: 'retained', resource: null, reason: 'no_owned_resource' }
}
if (!WORKER_SETTLED_STATES.includes(worker.state)) {
// Why: release is post-completion cleanup only; recording intent for an unsettled or
// uncertain worker would let recovery close a terminal the coordinator never reviewed.
@@ -54,6 +54,29 @@ describe('OrchestrationDb worker Dispatch state', () => {
])
})
it('retains an active supervised worker terminal', () => {
const d = createDb()
const task = d.createTask({ spec: 'retain active worker' })
const started = d.createStartingWorkerDispatch({ taskId: task.id, startOptions: {} })
d.prepareStartingWorkerAuthority({
dispatchId: started.dispatch.id,
handle: 'term_worker',
paneKey: 'tab_worker:leaf_worker',
processIncarnation: 'runtime:pty:1',
worktreeId: 'repo::worktree',
setupState: 'not_applicable',
effects: [],
terminalOwnership: 'created'
})
d.markWorkerDispatchReady(started.dispatch.id)
expect(d.retainWorkerTerminalResource(started.dispatch.id)).toMatchObject({
disposition: 'retained',
resource: { release_state: 'retained', retained_reason: 'user_requested' }
})
expect(d.getWorkerDispatch(started.dispatch.id)?.state).toBe('ready')
})
it('requeues an active Task before settling a worker whose terminal is missing', () => {
const d = createDb()
const task = d.createTask({ spec: 'recover missing worker' })
@@ -237,7 +260,7 @@ describe('OrchestrationDb worker Dispatch state', () => {
})
d.markWorkerDispatchReady(started.dispatch.id)
expect(d.beginWorkerStop(started.dispatch.id).disposition).toBe('stopping')
expect(d.beginWorkerStop(started.dispatch.id, 'runtime_test').disposition).toBe('stopping')
expect(
d.settleWorkerReport({
taskId: task.id,
@@ -256,7 +279,7 @@ describe('OrchestrationDb worker Dispatch state', () => {
const started = d.createStartingWorkerDispatch({ taskId: task.id, startOptions: {} })
d.markWorkerStartUnknown(started.dispatch.id, 'agent_readiness', 'connection lost')
expect(d.beginWorkerStop(started.dispatch.id)).toMatchObject({
expect(d.beginWorkerStop(started.dispatch.id, 'runtime_test')).toMatchObject({
disposition: 'stopping',
worker: { state: 'stopping' }
})
@@ -356,7 +379,7 @@ describe('OrchestrationDb worker Dispatch state', () => {
})
).toMatchObject({ action: 'settled' })
expect(d.beginWorkerStop(started.dispatch.id)).toMatchObject({
expect(d.beginWorkerStop(started.dispatch.id, 'runtime_test')).toMatchObject({
disposition: 'already_settled',
worker: { state: 'succeeded' }
})
@@ -58,6 +58,8 @@ export type WorkerTerminalListState =
| 'release_unknown'
| 'released'
export type WorkerDispatchListState = WorkerDispatchState | 'unsupervised'
export type WorkerTerminalArchiveRow = {
dispatch_id: string
resource_id: string
@@ -77,7 +79,7 @@ export const WORKER_RELEASABLE_STATES: readonly WorkerDispatchState[] = ['succee
// Process accounting for worker-list; deliberately independent of Task/Dispatch outcome.
export function deriveWorkerTerminalListState(params: {
workerState: WorkerDispatchState
workerState: WorkerDispatchListState
agentTerminalHandle: string | null
resource: WorkerTerminalResourceRow | null
}): WorkerTerminalListState | null {
@@ -97,8 +99,13 @@ export function deriveWorkerTerminalListState(params: {
if (resource.ownership_state !== 'owned' || resource.release_state === 'retained') {
return 'retained'
}
if (WORKER_RELEASABLE_STATES.includes(params.workerState)) {
if (
params.workerState !== 'unsupervised' &&
WORKER_RELEASABLE_STATES.includes(params.workerState)
) {
return 'reclaimable'
}
return WORKER_SETTLED_STATES.includes(params.workerState) ? 'retained' : 'active'
return params.workerState !== 'unsupervised' && WORKER_SETTLED_STATES.includes(params.workerState)
? 'retained'
: 'active'
}
@@ -0,0 +1,243 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { OrcaRuntimeService } from '../../orca-runtime'
import { OrchestrationDb } from '../../orchestration/db'
import { ORCHESTRATION_METHODS } from './orchestration'
describe('manual Dispatch observation', () => {
let db: OrchestrationDb | undefined
afterEach(() => db?.close())
it('covers the real dispatch --inject entry path before observing the lane', async () => {
db = new OrchestrationDb(':memory:')
const runtime = new OrcaRuntimeService()
runtime.setOrchestrationDb(db)
const coordinatorPaneKey = 'tab_coord:leaf_coord'
const workerPaneKey = 'tab_worker:leaf_worker'
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) =>
handle === 'term_coord' ? coordinatorPaneKey : workerPaneKey
)
vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockReturnValue({
terminalHandle: 'term_worker',
paneKey: workerPaneKey,
processIncarnation: 'runtime_test:term_worker:1'
} as never)
vi.spyOn(runtime, 'isTerminalRunningAgent').mockResolvedValue(true)
vi.spyOn(runtime, 'sendTerminalAgentPrompt').mockResolvedValue({
handle: 'term_worker',
accepted: true,
bytesWritten: 1
})
vi.spyOn(runtime, 'showTerminal').mockResolvedValue({
handle: 'term_worker',
connected: true,
status: 'running'
} as never)
vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockReturnValue('runtime_test:term_worker:1')
vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue({
status: 'live',
ptyIds: ['runtime_test:term_worker:1']
})
vi.spyOn(runtime, 'getTerminalOrchestrationCliCommand').mockReturnValue('orca')
const run = db.createRun({
objective: 'STA-3848 repro',
coordinatorHandle: 'term_coord',
coordinatorPaneKey
})
const task = db.createTask({ spec: 'injected lane', runId: run.id })
const dispatchMethod = ORCHESTRATION_METHODS.find(
(candidate) => candidate.name === 'orchestration.dispatch'
)
if (!dispatchMethod) {
throw new Error('Missing method orchestration.dispatch')
}
const result = (await dispatchMethod.handler(
dispatchMethod.params?.parse({
task: task.id,
to: 'term_worker',
from: 'term_coord',
run: run.id,
inject: true
}),
{ runtime, legacyCoordinatorRunId: run.id }
)) as { dispatch: { id: string } }
expect(db.getWorkerDispatch(result.dispatch.id)).toBeUndefined()
expect(db.getDispatchContextById(result.dispatch.id)).toMatchObject({
assignee_handle: 'term_worker',
assignee_pane_key: workerPaneKey,
process_incarnation: 'runtime_test:term_worker:1',
capability_hash: expect.any(String)
})
const workerShowMethod = ORCHESTRATION_METHODS.find(
(candidate) => candidate.name === 'orchestration.workerShow'
)
if (!workerShowMethod) {
throw new Error('Missing method orchestration.workerShow')
}
await expect(
workerShowMethod.handler(workerShowMethod.params?.parse({ dispatch: result.dispatch.id }), {
runtime
})
).resolves.toMatchObject({
worker: { state: 'unsupervised', stage: 'injected' },
observation: { status: 'live', exactWorker: true }
})
})
it('keeps context-only reads truthful without supervising the operator pane', async () => {
db = new OrchestrationDb(':memory:')
const runtime = new OrcaRuntimeService()
runtime.setOrchestrationDb(db)
vi.spyOn(runtime, 'showTerminal').mockResolvedValue({
handle: 'term_worker',
connected: true,
status: 'running'
} as never)
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker')
vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockReturnValue('runtime_test:term_worker:1')
vi.spyOn(runtime, 'readTerminal').mockResolvedValue({
handle: 'term_worker',
status: 'running',
tail: ['injected output'],
truncated: false,
nextCursor: null
})
const run = db.createRun({
objective: 'STA-3848 repro',
coordinatorHandle: 'term_coord',
coordinatorPaneKey: 'tab_coord:leaf_coord'
})
const task = db.createTask({ spec: 'injected lane', runId: run.id })
const dispatch = db.createDispatchContext(
task.id,
'term_worker',
'tab_worker:leaf_worker',
'launch-hash',
'runtime_test:term_worker:1'
)
db.mintDispatchCapability({
dispatchId: dispatch.id,
paneKey: 'tab_worker:leaf_worker',
processIncarnation: 'runtime_test:term_worker:1'
})
const context = { runtime }
const call = async (name: string, params: Record<string, unknown>) => {
const method = ORCHESTRATION_METHODS.find((candidate) => candidate.name === name)
if (!method) {
throw new Error(`Missing method ${name}`)
}
return method.handler(method.params?.parse(params), context)
}
const dispatchShow = (await call('orchestration.dispatchShow', { task: task.id })) as {
dispatch: {
id: string
assignee_handle: string
assignee_pane_key: string
process_incarnation: string
}
}
expect(dispatchShow.dispatch).toMatchObject({
id: dispatch.id,
assignee_handle: 'term_worker',
assignee_pane_key: 'tab_worker:leaf_worker',
process_incarnation: 'runtime_test:term_worker:1'
})
expect(db.getWorkerDispatch(dispatch.id)).toBeUndefined()
const workerList = (await call('orchestration.workerList', { run: run.id })) as {
workers: {
dispatchId: string
workerState: string
terminalState: string | null
agentTerminalHandle: string | null
}[]
}
expect(workerList.workers).toEqual([
expect.objectContaining({
dispatchId: dispatch.id,
workerState: 'unsupervised',
terminalState: 'retained',
agentTerminalHandle: 'term_worker'
})
])
await expect(
call('orchestration.workerShow', { dispatch: dispatch.id })
).resolves.toMatchObject({
worker: { state: 'unsupervised', stage: 'injected', agent_terminal_handle: 'term_worker' },
observation: { status: 'live', exactWorker: true }
})
await expect(
call('orchestration.workerRead', { dispatch: dispatch.id, source: 'terminal' })
).resolves.toMatchObject({
dispatchId: dispatch.id,
status: { worker: 'unsupervised' },
terminal: { tail: ['injected output'] }
})
await expect(call('orchestration.workerRetain', { dispatch: dispatch.id })).rejects.toThrow(
/only a settled dispatch can retain/
)
expect(
db.settleWorkerReport({
taskId: task.id,
dispatchId: dispatch.id,
outcome: 'succeeded',
result: 'done'
})
).toEqual({ action: 'settled', outcome: 'succeeded', duplicate: false })
await expect(
call('orchestration.workerRetain', { dispatch: dispatch.id })
).resolves.toMatchObject({
state: 'retained',
reason: 'no_owned_resource',
processAction: 'none'
})
await expect(
call('orchestration.workerRelease', { dispatch: dispatch.id })
).resolves.toMatchObject({
state: 'retained',
reason: 'no_owned_resource',
processAction: 'none'
})
})
it.each([
['orchestration.workerStop', 'stopped'],
['orchestration.workerAbandon', 'abandoned']
] as const)('%s fences the assignment without closing the operator pane', async (name, state) => {
db = new OrchestrationDb(':memory:')
const runtime = new OrcaRuntimeService()
runtime.setOrchestrationDb(db)
const closeTerminal = vi.spyOn(runtime, 'closeTerminal')
const task = db.createTask({ spec: 'operator-owned lane' })
const dispatch = db.createDispatchContext(
task.id,
'term_worker',
'tab_worker:leaf_worker',
'launch-hash',
'runtime_test:term_worker:1'
)
const method = ORCHESTRATION_METHODS.find((candidate) => candidate.name === name)
if (!method) {
throw new Error(`Missing method ${name}`)
}
const result = await method.handler(method.params?.parse({ dispatch: dispatch.id }), {
runtime
})
expect(result).toMatchObject({ state, processAction: 'none' })
expect(closeTerminal).not.toHaveBeenCalled()
expect(db.getWorkerDispatch(dispatch.id)).toBeUndefined()
expect(db.getDispatchContextById(dispatch.id)).toMatchObject({
status: 'failed',
last_failure: state
})
})
})
@@ -3,6 +3,7 @@ import {
ORCHESTRATION_WORKER_READ_SOURCES,
type OrchestrationWorkerReadResult
} from '../../../../shared/orchestration-worker-output'
import { contextOnlyAbandonWarning } from '../../orchestration/context-only-dispatch-release'
import { OrchestrationError } from '../../orchestration/orchestration-error'
import { defineMethod, type RpcMethod } from '../core'
import { OptionalFiniteNumber, requiredString } from '../schemas'
@@ -10,7 +11,8 @@ import {
callFederatedWorkerShow,
exposeWorker,
inspectWorkerTerminal,
resolvePinnedFederatedServer
resolvePinnedFederatedServer,
showContextOnlyWorker
} from './orchestration-worker-observation'
import { readArchivedWorkerOutput } from './orchestration-worker-archive-read'
import { readLegacyFederatedTerminal } from './orchestration-worker-legacy-federated-read'
@@ -32,7 +34,7 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
const db = runtime.getOrchestrationDb()
const dispatch = db.getDispatchContextById(params.dispatch)
let worker = db.getWorkerDispatch(params.dispatch)
if (!dispatch || !worker) {
if (!dispatch) {
throw new OrchestrationError(
'dispatch_not_found',
`Worker Dispatch ${params.dispatch} was not found.`
@@ -40,6 +42,12 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
}
const federated = db.getFederatedDispatch(params.dispatch)
if (federated) {
if (!worker) {
throw new OrchestrationError(
'dispatch_not_found',
`Federated Worker Dispatch ${params.dispatch} has no worker record.`
)
}
const server = resolvePinnedFederatedServer(runtime, federated)
runtime.ensureOrchestrationFederationRelay(dispatch.run_id)
const remote = await callFederatedWorkerShow(runtime, federated)
@@ -106,6 +114,9 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
}
}
}
if (!worker) {
return showContextOnlyWorker(runtime, db, dispatch)
}
if (worker.runtime_epoch && worker.runtime_epoch !== runtime.getRuntimeId()) {
if (worker.state === 'starting') {
worker = db.markWorkerStartUnknown(
@@ -177,8 +188,16 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
})
}
}
const dispatch = db.getDispatchContextById(params.dispatch)
const worker = db.getWorkerDispatch(params.dispatch)
if (!worker?.agent_terminal_handle) {
const terminalHandle = worker?.agent_terminal_handle ?? dispatch?.assignee_handle
if (!dispatch) {
throw new OrchestrationError(
'dispatch_not_found',
`Dispatch ${params.dispatch} was not found.`
)
}
if (!terminalHandle) {
throw new OrchestrationError(
'dispatch_not_found',
`Worker Dispatch ${params.dispatch} has no agent terminal.`
@@ -189,7 +208,7 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
return readArchivedWorkerOutput({
db,
dispatchId: params.dispatch,
workerState: worker.state,
workerState: worker?.state ?? 'unsupervised',
resource,
source: params.source,
cursor: params.cursor,
@@ -206,8 +225,8 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
const output = await readExactWorkerOutput({
runtime,
dispatchId: params.dispatch,
terminalHandle: worker.agent_terminal_handle,
workerState: worker.state,
terminalHandle,
workerState: worker?.state ?? 'unsupervised',
terminalStatus:
observation.status === 'exited'
? 'exited'
@@ -220,7 +239,7 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
: observation.status === 'exited'
? 'exited'
: 'live',
attachedAt: worker.created_at,
attachedAt: worker?.created_at ?? dispatch.dispatched_at ?? dispatch.created_at,
source: params.source,
cursor: params.cursor,
limit: params.limit
@@ -273,16 +292,3 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
}
})
]
function contextOnlyAbandonWarning(result: {
state: string
alreadySettled: boolean
releasedCurrentTask: boolean
}): string {
if (result.alreadySettled) {
return `Dispatch was already ${result.state}; no state or process changed.`
}
return result.releasedCurrentTask
? 'The assignment was abandoned; its unsupervised terminal process was retained.'
: 'The superseded assignment was abandoned without changing the current Task or terminal process.'
}
@@ -1,7 +1,11 @@
import type { OrcaRuntimeService } from '../../orca-runtime'
import type { OrchestrationDb } from '../../orchestration/db'
import { OrchestrationError } from '../../orchestration/orchestration-error'
import type { FederatedDispatchRow, WorkerDispatchRow } from '../../orchestration/types'
import type {
DispatchContextRow,
FederatedDispatchRow,
WorkerDispatchRow
} from '../../orchestration/types'
export async function inspectWorkerTerminal(
runtime: OrcaRuntimeService,
@@ -15,17 +19,19 @@ export async function inspectWorkerTerminal(
reason?: string
}> {
const worker = db.getWorkerDispatch(dispatchId)
if (!worker?.agent_terminal_handle) {
const terminalHandle =
worker?.agent_terminal_handle ?? db.getDispatchContextById(dispatchId)?.assignee_handle
if (!terminalHandle) {
return { terminal: null, exact: false, status: 'unattached' }
}
const terminal = await runtime.showTerminal(worker.agent_terminal_handle).catch(() => null)
const terminal = await runtime.showTerminal(terminalHandle).catch(() => null)
if (!terminal) {
return { terminal: null, exact: false, status: 'missing' }
}
const exact = db.isDispatchProcessCurrent({
dispatchId,
paneKey: runtime.getTerminalPaneKey(worker.agent_terminal_handle),
processIncarnation: runtime.getTerminalProcessIncarnation(worker.agent_terminal_handle)
paneKey: runtime.getTerminalPaneKey(terminalHandle),
processIncarnation: runtime.getTerminalProcessIncarnation(terminalHandle)
})
if (!exact) {
return { terminal, exact, status: 'identity_changed' }
@@ -33,7 +39,7 @@ export async function inspectWorkerTerminal(
// Why: the aggregate inventory only iterates registered providers, so a dropped
// relay clears `connected` for every remote PTY at once. Lost contact is not a
// death certificate, and the verdict is the only field that can tell them apart.
const verdict = runtime.getTerminalLivenessVerdict?.(worker.agent_terminal_handle) ?? null
const verdict = runtime.getTerminalLivenessVerdict?.(terminalHandle) ?? null
if (verdict?.status === 'unverifiable') {
return { terminal, exact, status: 'unverifiable', reason: verdict.reason }
}
@@ -47,6 +53,43 @@ export async function inspectWorkerTerminal(
}
}
export function exposeContextOnlyWorker(dispatch: DispatchContextRow) {
return {
dispatch_id: dispatch.id,
runtime_epoch: null,
state: 'unsupervised' as const,
stage: dispatch.capability_hash ? 'injected' : 'context_only',
worktree_id: null,
agent_terminal_handle: dispatch.assignee_handle,
setup_state: 'not_applicable',
effects: [],
residualResources: [],
startOptions: {},
last_error: dispatch.last_failure,
created_at: dispatch.created_at,
updated_at: dispatch.completed_at ?? dispatch.created_at
}
}
export async function showContextOnlyWorker(
runtime: OrcaRuntimeService,
db: OrchestrationDb,
dispatch: DispatchContextRow
) {
const observation = await inspectWorkerTerminal(runtime, db, dispatch.id)
return {
dispatch,
worker: exposeContextOnlyWorker(dispatch),
terminal: observation.exact ? observation.terminal : null,
observation: {
status: observation.status,
exactWorker: observation.exact,
...(observation.reason ? { reason: observation.reason } : {})
},
terminalResource: null
}
}
export function exposeWorker(worker: WorkerDispatchRow) {
return {
...worker,
@@ -320,7 +320,7 @@ describe('orchestration worker release', () => {
setup()
const { dispatchId } = await startWorker()
if (state === 'stopped') {
db.beginWorkerStop(dispatchId)
db.beginWorkerStop(dispatchId, runtime.getRuntimeId())
db.settleWorkerStop(dispatchId)
} else {
db.abandonWorkerDispatch(dispatchId)
@@ -1,5 +1,4 @@
import { z } from 'zod'
import { OrchestrationError } from '../../orchestration/orchestration-error'
import type { WorkerTerminalListState } from '../../orchestration/worker-terminal-ownership'
import { defineMethod, type RpcMethod } from '../core'
import { requiredString } from '../schemas'
@@ -99,12 +98,6 @@ export const ORCHESTRATION_WORKER_RELEASE_METHODS: RpcMethod[] = [
params: WorkerDispatchParams,
handler: (params, { runtime }) => {
const db = runtime.getOrchestrationDb()
if (!db.getWorkerDispatch(params.dispatch)) {
throw new OrchestrationError(
'dispatch_not_found',
`Worker Dispatch ${params.dispatch} was not found.`
)
}
const retained = db.retainWorkerTerminalResource(params.dispatch)
if (retained.disposition === 'already_released') {
return {
@@ -28,6 +28,7 @@ describe('federated worker stop capability', () => {
})
const runtime = {
getOrchestrationDb: () => db,
getRuntimeId: () => 'runtime_current',
resolveOrchestrationWorkerServer: () => ({
environmentId: 'environment_linux',
name: 'linux',
@@ -48,6 +49,7 @@ describe('federated worker stop capability', () => {
}
})
).resolves.toMatchObject({ state: 'stop_unknown', processAction: 'none' })
expect(db.beginWorkerStop).toHaveBeenCalledWith('ctx_remote', 'runtime_current')
expect(markWorkerStopUnknown).toHaveBeenCalledWith(
'ctx_remote',
'Connected server linux cannot prove the worker stop outcome.'
@@ -55,7 +55,8 @@ describe('worker-stop against a terminal we lost contact with', () => {
processIncarnation: 'runtime:pty:1',
worktreeId: 'repo::worktree',
setupState: 'not_applicable',
effects: [{ kind: 'terminal', action: 'created', id: 'term_worker' }]
effects: [{ kind: 'terminal', action: 'created', id: 'term_worker' }],
terminalOwnership: 'created'
})
db.markWorkerDispatchReady(started.dispatch.id)
return started.dispatch
@@ -141,6 +142,94 @@ describe('worker-stop against a terminal we lost contact with', () => {
})
})
it('does not close a pane after user input transfers ownership', async () => {
vi.spyOn(runtime, 'showTerminal').mockResolvedValue({
handle: 'term_worker',
worktreeId: 'repo::worktree',
connected: true,
status: 'running'
} as never)
vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue({
status: 'live',
ptyIds: ['runtime:pty:1']
})
const closeTerminal = vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({
handle: 'term_worker',
tabId: 'tab_worker',
ptyKilled: true
})
const dispatch = createWorker()
await expect(
call('orchestration.workerTerminalUserInput', {
paneKey: 'tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'
})
).resolves.toMatchObject({ changed: 1 })
await expect(
call('orchestration.workerStop', { dispatch: dispatch.id })
).resolves.toMatchObject({
state: 'stop_unknown',
processAction: 'none',
lastError: 'The worker terminal is user_owned; no terminal was closed.'
})
expect(closeTerminal).not.toHaveBeenCalled()
})
it('does not close a legacy worker terminal without an ownership record', async () => {
vi.spyOn(runtime, 'showTerminal').mockResolvedValue({
handle: 'term_worker',
worktreeId: 'repo::worktree',
connected: true,
status: 'running'
} as never)
vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue({
status: 'live',
ptyIds: ['runtime:pty:1']
})
const closeTerminal = vi.spyOn(runtime, 'closeTerminal')
const dispatch = createWorker()
const resource = db.getWorkerTerminalResourceByOwner(dispatch.id)
if (!resource) {
throw new Error('Expected worker terminal resource')
}
;(db as unknown as { db: { prepare: (sql: string) => { run: (id: string) => void } } }).db
.prepare('DELETE FROM worker_terminal_resources WHERE id = ?')
.run(resource.id)
await expect(
call('orchestration.workerStop', { dispatch: dispatch.id })
).resolves.toMatchObject({
state: 'stop_unknown',
processAction: 'none',
lastError: 'The worker terminal is unproven; no terminal was closed.'
})
expect(closeTerminal).not.toHaveBeenCalled()
})
it('linearizes stop before a concurrent user takeover', () => {
const dispatch = createWorker()
expect(db.beginWorkerStop(dispatch.id, runtime.getRuntimeId()).disposition).toBe('stopping')
expect(db.markWorkerTerminalUserOwned('tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb')).toBe(
0
)
expect(db.getWorkerTerminalResourceByOwner(dispatch.id)).toMatchObject({
ownership_state: 'owned',
release_state: 'not_requested'
})
})
it('does not let abandon overwrite a stop in progress', () => {
const dispatch = createWorker()
expect(db.beginWorkerStop(dispatch.id, runtime.getRuntimeId()).disposition).toBe('stopping')
expect(() => db.abandonWorkerDispatch(dispatch.id)).toThrow(
'is stopping; wait for worker-stop to settle before abandoning'
)
expect(db.getWorkerDispatch(dispatch.id)?.state).toBe('stopping')
})
it('still reports a locally observed exit as exited', async () => {
const dispatch = createWorker()
@@ -27,7 +27,7 @@ export const ORCHESTRATION_WORKER_STOP_METHODS: RpcMethod[] = [
)
}
const server = resolvePinnedFederatedServer(runtime, federated)
const begun = db.beginWorkerStop(params.dispatch)
const begun = db.beginWorkerStop(params.dispatch, runtime.getRuntimeId())
if (begun.disposition === 'already_settled') {
return settledReceipt(params.dispatch, begun.worker.state)
}
@@ -97,7 +97,7 @@ export const ORCHESTRATION_WORKER_STOP_METHODS: RpcMethod[] = [
}
}
const begun = db.beginWorkerStop(params.dispatch)
const begun = db.beginWorkerStop(params.dispatch, runtime.getRuntimeId())
if (begun.disposition === 'already_settled') {
return settledReceipt(params.dispatch, begun.worker.state)
}
@@ -137,6 +137,18 @@ export const ORCHESTRATION_WORKER_STOP_METHODS: RpcMethod[] = [
'none'
)
}
const resource = db.getWorkerTerminalResourceByOwner(params.dispatch)
if (!resource || resource.ownership_state !== 'owned') {
const ownership = resource?.ownership_state ?? 'unproven'
return unknownReceipt(
params.dispatch,
db.markWorkerStopUnknown(
params.dispatch,
`The worker terminal is ${ownership}; no terminal was closed.`
),
'none'
)
}
try {
const close = await runtime.closeTerminal(handle)
if (!close.ptyKilled) {
@@ -3,6 +3,14 @@ import { OrcaRuntimeService } from '../../orca-runtime'
import { OrchestrationDb } from '../../orchestration/db'
import { ORCHESTRATION_METHODS } from './orchestration'
function deferred<T>(): { promise: Promise<T>; resolve: (value: T) => void } {
let resolve!: (value: T) => void
const promise = new Promise<T>((promiseResolve) => {
resolve = promiseResolve
})
return { promise, resolve }
}
describe('orchestration worker recovery', () => {
let db: OrchestrationDb
let runtime: OrcaRuntimeService
@@ -64,7 +72,8 @@ describe('orchestration worker recovery', () => {
processIncarnation: 'runtime:pty:1',
worktreeId: 'repo::worktree',
setupState: 'not_applicable',
effects: [{ kind: 'terminal', action: 'created', id: 'term_worker' }]
effects: [{ kind: 'terminal', action: 'created', id: 'term_worker' }],
terminalOwnership: 'created'
})
if (ready) {
db.markWorkerDispatchReady(started.dispatch.id)
@@ -105,6 +114,39 @@ describe('orchestration worker recovery', () => {
expect(db.getTask(task.id)?.status).toBe('blocked')
})
it('keeps an in-flight stop fenced during runtime-epoch reconciliation', async () => {
const { dispatch } = createWorker('previous_runtime')
const pendingObservation = deferred<Awaited<ReturnType<OrcaRuntimeService['showTerminal']>>>()
vi.mocked(runtime.showTerminal)
.mockReturnValueOnce(pendingObservation.promise)
.mockResolvedValue({
handle: 'term_worker',
worktreeId: 'repo::worktree',
connected: true,
status: 'running'
} as never)
const stop = call('orchestration.workerStop', { dispatch: dispatch.id })
await vi.waitFor(() => expect(runtime.showTerminal).toHaveBeenCalledTimes(1))
await expect(
call('orchestration.workerShow', { dispatch: dispatch.id })
).resolves.toMatchObject({ worker: { state: 'stopping' } })
await expect(call('orchestration.workerAbandon', { dispatch: dispatch.id })).rejects.toThrow(
'is stopping; wait for worker-stop to settle before abandoning'
)
pendingObservation.resolve({
handle: 'term_worker',
worktreeId: 'repo::worktree',
connected: true,
status: 'running'
} as never)
await expect(stop).resolves.toMatchObject({
state: 'stopped',
processAction: 'closed_agent_terminal'
})
})
it('does not adopt or stop a same-looking pane with a new process incarnation', async () => {
const { task, dispatch } = createWorker()
vi.mocked(runtime.getTerminalProcessIncarnation).mockReturnValue('runtime:pty:2')
@@ -184,7 +226,7 @@ describe('orchestration worker recovery', () => {
it('turns an interrupted stop into unknown after runtime restart', async () => {
const { task, dispatch } = createWorker('previous_runtime')
db.beginWorkerStop(dispatch.id)
db.beginWorkerStop(dispatch.id, 'previous_runtime')
await expect(
call('orchestration.workerShow', { dispatch: dispatch.id })
@@ -213,7 +255,7 @@ describe('orchestration worker recovery', () => {
}
})
db.markWorkerStartUnknown(started.dispatch.id, 'remote_attach', 'response lost')
db.beginWorkerStop(started.dispatch.id)
db.beginWorkerStop(started.dispatch.id, runtime.getRuntimeId())
db.markWorkerStopUnknown(started.dispatch.id, 'stop response lost')
vi.spyOn(runtime, 'resolveOrchestrationWorkerServer').mockReturnValue({
environmentId: 'environment_windows',