mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 00:02:31 +00:00
fix(orchestration): expose unsupervised dispatch lanes (#15105)
This commit is contained in:
@@ -270,6 +270,8 @@ Recovery is conditional, never a fixed destructive sequence:
|
||||
|
||||
Low-level `worktree create`, `terminal create`, and `dispatch --inject` remain valid recipes for custom argv or topology that `worker-start` does not express.
|
||||
|
||||
`dispatch --inject` deliberately keeps an operator-started terminal unsupervised: it never creates a `worker_dispatches` row and `worker-stop`/`worker-abandon` never close that process. The dispatch context is still authoritative, so `worker-show`, `worker-read`, and `worker-list` report it as `unsupervised`; settled `worker-retain` and `worker-release` report `retained` with `no_owned_resource` and take no process action. Use `worker-start --terminal <handle>` when supervision and worker lifecycle state are required.
|
||||
|
||||
## Gates And Legacy Inspection
|
||||
|
||||
```bash
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -42,7 +42,10 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['orchestration', 'worker-show'],
|
||||
summary: 'Inspect one supervised worker Dispatch',
|
||||
usage: 'orca orchestration worker-show --dispatch <dispatch_id> [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'dispatch']
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'dispatch'],
|
||||
notes: [
|
||||
'A Dispatch created by orchestration dispatch is shown as unsupervised and reports the exact adopted terminal when its identity is still provable.'
|
||||
]
|
||||
},
|
||||
{
|
||||
path: ['orchestration', 'worker-read'],
|
||||
@@ -52,6 +55,7 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'dispatch', 'source', 'cursor', 'limit'],
|
||||
notes: [
|
||||
'The default auto source uses an exact hook-reported transcript when available and otherwise returns labeled terminal output.',
|
||||
'A Dispatch created by orchestration dispatch reads from its adopted terminal with worker status unsupervised.',
|
||||
'A returned cursor is pinned to the exact source; start a fresh read if Orca reports source_changed.'
|
||||
]
|
||||
},
|
||||
@@ -82,6 +86,7 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'dispatch', 'retry-request'],
|
||||
notes: [
|
||||
'Post-completion cleanup for a settled (succeeded or failed) worker; closes only the exact coordinator-owned agent terminal of that worker.',
|
||||
'A settled Dispatch created by orchestration dispatch has no owned terminal resource and is reported retained without process action.',
|
||||
'An inspectable output archive is preserved before the terminal closes, so worker-read still returns output afterwards.',
|
||||
'Never closes setup terminals, configured tabs, reused or pre-existing terminals, user-taken-over terminals, or unproven identities.',
|
||||
'Idempotent: repeating the call reports already_released. Only release_unknown exits 1; retained, release_pending, and already_released exit 0.'
|
||||
@@ -95,6 +100,7 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'dispatch', 'retry-request'],
|
||||
notes: [
|
||||
'Records a durable user-requested exception; a later explicit worker-release clears it and releases the terminal.',
|
||||
'A settled Dispatch created by orchestration dispatch has no owned terminal resource and is reported retained without process action.',
|
||||
'Performs no process or filesystem action.'
|
||||
]
|
||||
},
|
||||
@@ -105,7 +111,8 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [
|
||||
'orca orchestration worker-list [--run <run_id>] [--terminal-state <active|reclaimable|retained|release_pending|release_unknown|released>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'run', 'terminal-state'],
|
||||
notes: [
|
||||
'Terminal state is process accounting and is reported separately from Task status; a completed Task can still own a live terminal.'
|
||||
'Terminal state is process accounting and is reported separately from Task status; a completed Task can still own a live terminal.',
|
||||
'Context-only Dispatches created by orchestration dispatch are included as unsupervised with terminal state retained.'
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -9,6 +9,19 @@ export type ContextOnlyDispatchReleaseResult = {
|
||||
releasedCurrentTask: boolean
|
||||
}
|
||||
|
||||
export function contextOnlyAbandonWarning(result: {
|
||||
state: string
|
||||
alreadySettled: boolean
|
||||
releasedCurrentTask: boolean
|
||||
}): string {
|
||||
if (result.alreadySettled) {
|
||||
return `Dispatch was already ${result.state}; no state or process changed.`
|
||||
}
|
||||
return result.releasedCurrentTask
|
||||
? 'The assignment was abandoned; its unsupervised terminal process was retained.'
|
||||
: 'The superseded assignment was abandoned without changing the current Task or terminal process.'
|
||||
}
|
||||
|
||||
export function releaseContextOnlyDispatch(
|
||||
db: Database.Database,
|
||||
dispatch: DispatchContextRow,
|
||||
|
||||
@@ -160,7 +160,7 @@ describe('Task/Dispatch invariant transactions', () => {
|
||||
sqliteFor(db).prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(task.id)
|
||||
const second = db.createDispatchContext(task.id, 'term_second')
|
||||
|
||||
expect(db.beginWorkerStop(second.id)).toMatchObject({
|
||||
expect(db.beginWorkerStop(second.id, 'runtime_test')).toMatchObject({
|
||||
disposition: 'context_only',
|
||||
releasedCurrentTask: false
|
||||
})
|
||||
|
||||
@@ -228,7 +228,7 @@ describe('Task/Dispatch lifecycle guards', () => {
|
||||
|
||||
const released =
|
||||
operation === 'stop'
|
||||
? database.beginWorkerStop(contextOnly.id)
|
||||
? database.beginWorkerStop(contextOnly.id, 'runtime_test')
|
||||
: database.abandonWorkerDispatch(contextOnly.id)
|
||||
expect(released).toMatchObject({
|
||||
disposition: 'context_only',
|
||||
@@ -257,7 +257,9 @@ describe('Task/Dispatch lifecycle guards', () => {
|
||||
const released = startWorker(database, task.id, `${operation}_released`)
|
||||
|
||||
if (operation === 'stop') {
|
||||
expect(database.beginWorkerStop(released.dispatchId).disposition).toBe('stopping')
|
||||
expect(database.beginWorkerStop(released.dispatchId, 'runtime_test').disposition).toBe(
|
||||
'stopping'
|
||||
)
|
||||
expect(database.settleWorkerStop(released.dispatchId).state).toBe('stopped')
|
||||
} else {
|
||||
expect(database.abandonWorkerDispatch(released.dispatchId).disposition).toBe('abandoned')
|
||||
@@ -286,7 +288,9 @@ describe('Task/Dispatch lifecycle guards', () => {
|
||||
sqliteFor(database).prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(task.id)
|
||||
const abandoned = startWorker(database, task.id, 'interleaved_abandoned')
|
||||
|
||||
expect(database.beginWorkerStop(stopping.dispatchId).disposition).toBe('stopping')
|
||||
expect(database.beginWorkerStop(stopping.dispatchId, 'runtime_test').disposition).toBe(
|
||||
'stopping'
|
||||
)
|
||||
expect(database.abandonWorkerDispatch(abandoned.dispatchId).disposition).toBe('abandoned')
|
||||
expect(database.getTask(task.id)?.status).toBe('dispatched')
|
||||
|
||||
@@ -305,7 +309,9 @@ describe('Task/Dispatch lifecycle guards', () => {
|
||||
database.markWorkerStartUnknown(uncertain.dispatch.id, 'agent_readiness', 'outcome unknown')
|
||||
expect(database.getTask(task.id)?.status).toBe('blocked')
|
||||
|
||||
expect(database.beginWorkerStop(uncertain.dispatch.id).disposition).toBe('stopping')
|
||||
expect(database.beginWorkerStop(uncertain.dispatch.id, 'runtime_test').disposition).toBe(
|
||||
'stopping'
|
||||
)
|
||||
expect(database.settleWorkerStop(uncertain.dispatch.id).state).toBe('stopped')
|
||||
expect(database.getTask(task.id)?.status).toBe('dispatched')
|
||||
expect(
|
||||
|
||||
@@ -39,6 +39,12 @@ export function abandonWorkerDispatch(
|
||||
this.db.exec('COMMIT')
|
||||
return { disposition: 'stale', worker }
|
||||
}
|
||||
if (worker.state === 'stopping') {
|
||||
throw new OrchestrationError(
|
||||
'dispatch_inactive',
|
||||
`Dispatch ${dispatchId} is stopping; wait for worker-stop to settle before abandoning.`
|
||||
)
|
||||
}
|
||||
if (worker.state === 'succeeded') {
|
||||
throw new OrchestrationError(
|
||||
'dispatch_inactive',
|
||||
|
||||
@@ -28,7 +28,8 @@ export function isDispatchProcessCurrent(
|
||||
|
||||
export function beginWorkerStop(
|
||||
this: OrchestrationDb,
|
||||
dispatchId: string
|
||||
dispatchId: string,
|
||||
runtimeEpoch: string
|
||||
):
|
||||
| { disposition: 'stopping'; worker: WorkerDispatchRow; dispatch: DispatchContextRow }
|
||||
| { disposition: 'already_settled'; worker: WorkerDispatchRow; dispatch: DispatchContextRow }
|
||||
@@ -61,10 +62,11 @@ export function beginWorkerStop(
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE worker_dispatches
|
||||
SET state = 'stopping', stage = 'stop_requested', updated_at = datetime('now')
|
||||
SET state = 'stopping', stage = 'stop_requested',
|
||||
runtime_epoch = COALESCE(?, runtime_epoch), updated_at = datetime('now')
|
||||
WHERE dispatch_id = ? AND state IN ('ready', 'start_unknown')`
|
||||
)
|
||||
.run(dispatchId)
|
||||
.run(runtimeEpoch, dispatchId)
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE dispatch_contexts
|
||||
|
||||
@@ -150,6 +150,17 @@ export function retainWorkerTerminalResource(
|
||||
| { disposition: 'no_owned_resource'; resource: null } {
|
||||
this.db.exec('BEGIN IMMEDIATE')
|
||||
try {
|
||||
const dispatch = this.getDispatchContextById(dispatchId)
|
||||
if (!dispatch) {
|
||||
throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`)
|
||||
}
|
||||
const worker = this.getWorkerDispatch(dispatchId)
|
||||
if (!worker && !['completed', 'failed', 'circuit_broken'].includes(dispatch.status)) {
|
||||
throw new OrchestrationError(
|
||||
'dispatch_inactive',
|
||||
`Dispatch ${dispatchId} is ${dispatch.status}; only a settled dispatch can retain.`
|
||||
)
|
||||
}
|
||||
const resource = this.getWorkerTerminalResourceByOwner(dispatchId)
|
||||
if (!resource) {
|
||||
this.db.exec('COMMIT')
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { DispatchStatus, WorkerDispatchState } from '../../types'
|
||||
import type { DispatchStatus } from '../../types'
|
||||
import { deriveWorkerTerminalListState } from '../../worker-terminal-ownership'
|
||||
import type {
|
||||
WorkerDispatchListState,
|
||||
WorkerTerminalResourceRow,
|
||||
WorkerTerminalListState
|
||||
} from '../../worker-terminal-ownership'
|
||||
@@ -16,7 +17,11 @@ export function markWorkerTerminalUserOwned(this: OrchestrationDb, paneKey: stri
|
||||
.prepare(
|
||||
`SELECT id, owner_dispatch_id, pane_key FROM worker_terminal_resources
|
||||
WHERE pane_key = ? AND ownership_state = 'owned'
|
||||
AND release_state IN ('not_requested', 'retained', 'requested')`
|
||||
AND release_state IN ('not_requested', 'retained', 'requested')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM worker_dispatches w
|
||||
WHERE w.dispatch_id = owner_dispatch_id AND w.state = 'stopping'
|
||||
)`
|
||||
)
|
||||
.all(paneKey) as { id: string; owner_dispatch_id: string; pane_key: string }[]
|
||||
const candidates =
|
||||
@@ -28,6 +33,10 @@ export function markWorkerTerminalUserOwned(this: OrchestrationDb, paneKey: stri
|
||||
`SELECT id, owner_dispatch_id, pane_key FROM worker_terminal_resources
|
||||
WHERE ownership_state = 'owned'
|
||||
AND release_state IN ('not_requested', 'retained', 'requested')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM worker_dispatches w
|
||||
WHERE w.dispatch_id = owner_dispatch_id AND w.state = 'stopping'
|
||||
)
|
||||
AND pane_key IS NOT NULL`
|
||||
)
|
||||
.all() as { id: string; owner_dispatch_id: string; pane_key: string }[]
|
||||
@@ -37,7 +46,11 @@ export function markWorkerTerminalUserOwned(this: OrchestrationDb, paneKey: stri
|
||||
SET ownership_state = 'user_owned', release_state = 'retained',
|
||||
retained_reason = 'user_takeover', updated_at = datetime('now')
|
||||
WHERE id = ? AND ownership_state = 'owned'
|
||||
AND release_state IN ('not_requested', 'retained', 'requested')`
|
||||
AND release_state IN ('not_requested', 'retained', 'requested')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM worker_dispatches w
|
||||
WHERE w.dispatch_id = owner_dispatch_id AND w.state = 'stopping'
|
||||
)`
|
||||
)
|
||||
let changed = 0
|
||||
for (const candidate of candidates) {
|
||||
@@ -76,7 +89,7 @@ export function listWorkerTerminalResources(
|
||||
dispatchId: string
|
||||
taskId: string
|
||||
runId: string
|
||||
workerState: WorkerDispatchState
|
||||
workerState: WorkerDispatchListState
|
||||
dispatchStatus: DispatchStatus
|
||||
agentTerminalHandle: string | null
|
||||
terminalState: WorkerTerminalListState | null
|
||||
@@ -84,16 +97,18 @@ export function listWorkerTerminalResources(
|
||||
}[] {
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`SELECT w.dispatch_id, w.state AS worker_state, w.agent_terminal_handle,
|
||||
`SELECT d.id AS dispatch_id,
|
||||
COALESCE(w.state, 'unsupervised') AS worker_state,
|
||||
COALESCE(w.agent_terminal_handle, d.assignee_handle) AS agent_terminal_handle,
|
||||
d.task_id, d.run_id, d.status AS dispatch_status
|
||||
FROM worker_dispatches w
|
||||
JOIN dispatch_contexts d ON d.id = w.dispatch_id
|
||||
FROM dispatch_contexts d
|
||||
LEFT JOIN worker_dispatches w ON w.dispatch_id = d.id
|
||||
${params.runId ? 'WHERE d.run_id = ?' : ''}
|
||||
ORDER BY w.created_at ASC`
|
||||
ORDER BY COALESCE(w.created_at, d.created_at) ASC`
|
||||
)
|
||||
.all(...(params.runId ? [params.runId] : [])) as {
|
||||
dispatch_id: string
|
||||
worker_state: WorkerDispatchState
|
||||
worker_state: WorkerDispatchListState
|
||||
agent_terminal_handle: string | null
|
||||
task_id: string
|
||||
run_id: string
|
||||
|
||||
@@ -20,10 +20,21 @@ export function requestWorkerTerminalRelease(
|
||||
} {
|
||||
this.db.exec('BEGIN IMMEDIATE')
|
||||
try {
|
||||
const dispatch = this.getDispatchContextById(dispatchId)
|
||||
const worker = this.getWorkerDispatch(dispatchId)
|
||||
if (!worker) {
|
||||
if (!dispatch) {
|
||||
throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`)
|
||||
}
|
||||
if (!worker) {
|
||||
if (!['completed', 'failed', 'circuit_broken'].includes(dispatch.status)) {
|
||||
throw new OrchestrationError(
|
||||
'dispatch_inactive',
|
||||
`Dispatch ${dispatchId} is ${dispatch.status}; only a settled dispatch can release.`
|
||||
)
|
||||
}
|
||||
this.db.exec('COMMIT')
|
||||
return { disposition: 'retained', resource: null, reason: 'no_owned_resource' }
|
||||
}
|
||||
if (!WORKER_SETTLED_STATES.includes(worker.state)) {
|
||||
// Why: release is post-completion cleanup only; recording intent for an unsettled or
|
||||
// uncertain worker would let recovery close a terminal the coordinator never reviewed.
|
||||
|
||||
@@ -54,6 +54,29 @@ describe('OrchestrationDb worker Dispatch state', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('retains an active supervised worker terminal', () => {
|
||||
const d = createDb()
|
||||
const task = d.createTask({ spec: 'retain active worker' })
|
||||
const started = d.createStartingWorkerDispatch({ taskId: task.id, startOptions: {} })
|
||||
d.prepareStartingWorkerAuthority({
|
||||
dispatchId: started.dispatch.id,
|
||||
handle: 'term_worker',
|
||||
paneKey: 'tab_worker:leaf_worker',
|
||||
processIncarnation: 'runtime:pty:1',
|
||||
worktreeId: 'repo::worktree',
|
||||
setupState: 'not_applicable',
|
||||
effects: [],
|
||||
terminalOwnership: 'created'
|
||||
})
|
||||
d.markWorkerDispatchReady(started.dispatch.id)
|
||||
|
||||
expect(d.retainWorkerTerminalResource(started.dispatch.id)).toMatchObject({
|
||||
disposition: 'retained',
|
||||
resource: { release_state: 'retained', retained_reason: 'user_requested' }
|
||||
})
|
||||
expect(d.getWorkerDispatch(started.dispatch.id)?.state).toBe('ready')
|
||||
})
|
||||
|
||||
it('requeues an active Task before settling a worker whose terminal is missing', () => {
|
||||
const d = createDb()
|
||||
const task = d.createTask({ spec: 'recover missing worker' })
|
||||
@@ -237,7 +260,7 @@ describe('OrchestrationDb worker Dispatch state', () => {
|
||||
})
|
||||
d.markWorkerDispatchReady(started.dispatch.id)
|
||||
|
||||
expect(d.beginWorkerStop(started.dispatch.id).disposition).toBe('stopping')
|
||||
expect(d.beginWorkerStop(started.dispatch.id, 'runtime_test').disposition).toBe('stopping')
|
||||
expect(
|
||||
d.settleWorkerReport({
|
||||
taskId: task.id,
|
||||
@@ -256,7 +279,7 @@ describe('OrchestrationDb worker Dispatch state', () => {
|
||||
const started = d.createStartingWorkerDispatch({ taskId: task.id, startOptions: {} })
|
||||
d.markWorkerStartUnknown(started.dispatch.id, 'agent_readiness', 'connection lost')
|
||||
|
||||
expect(d.beginWorkerStop(started.dispatch.id)).toMatchObject({
|
||||
expect(d.beginWorkerStop(started.dispatch.id, 'runtime_test')).toMatchObject({
|
||||
disposition: 'stopping',
|
||||
worker: { state: 'stopping' }
|
||||
})
|
||||
@@ -356,7 +379,7 @@ describe('OrchestrationDb worker Dispatch state', () => {
|
||||
})
|
||||
).toMatchObject({ action: 'settled' })
|
||||
|
||||
expect(d.beginWorkerStop(started.dispatch.id)).toMatchObject({
|
||||
expect(d.beginWorkerStop(started.dispatch.id, 'runtime_test')).toMatchObject({
|
||||
disposition: 'already_settled',
|
||||
worker: { state: 'succeeded' }
|
||||
})
|
||||
|
||||
@@ -58,6 +58,8 @@ export type WorkerTerminalListState =
|
||||
| 'release_unknown'
|
||||
| 'released'
|
||||
|
||||
export type WorkerDispatchListState = WorkerDispatchState | 'unsupervised'
|
||||
|
||||
export type WorkerTerminalArchiveRow = {
|
||||
dispatch_id: string
|
||||
resource_id: string
|
||||
@@ -77,7 +79,7 @@ export const WORKER_RELEASABLE_STATES: readonly WorkerDispatchState[] = ['succee
|
||||
|
||||
// Process accounting for worker-list; deliberately independent of Task/Dispatch outcome.
|
||||
export function deriveWorkerTerminalListState(params: {
|
||||
workerState: WorkerDispatchState
|
||||
workerState: WorkerDispatchListState
|
||||
agentTerminalHandle: string | null
|
||||
resource: WorkerTerminalResourceRow | null
|
||||
}): WorkerTerminalListState | null {
|
||||
@@ -97,8 +99,13 @@ export function deriveWorkerTerminalListState(params: {
|
||||
if (resource.ownership_state !== 'owned' || resource.release_state === 'retained') {
|
||||
return 'retained'
|
||||
}
|
||||
if (WORKER_RELEASABLE_STATES.includes(params.workerState)) {
|
||||
if (
|
||||
params.workerState !== 'unsupervised' &&
|
||||
WORKER_RELEASABLE_STATES.includes(params.workerState)
|
||||
) {
|
||||
return 'reclaimable'
|
||||
}
|
||||
return WORKER_SETTLED_STATES.includes(params.workerState) ? 'retained' : 'active'
|
||||
return params.workerState !== 'unsupervised' && WORKER_SETTLED_STATES.includes(params.workerState)
|
||||
? 'retained'
|
||||
: 'active'
|
||||
}
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { OrcaRuntimeService } from '../../orca-runtime'
|
||||
import { OrchestrationDb } from '../../orchestration/db'
|
||||
import { ORCHESTRATION_METHODS } from './orchestration'
|
||||
|
||||
describe('manual Dispatch observation', () => {
|
||||
let db: OrchestrationDb | undefined
|
||||
|
||||
afterEach(() => db?.close())
|
||||
|
||||
it('covers the real dispatch --inject entry path before observing the lane', async () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const runtime = new OrcaRuntimeService()
|
||||
runtime.setOrchestrationDb(db)
|
||||
const coordinatorPaneKey = 'tab_coord:leaf_coord'
|
||||
const workerPaneKey = 'tab_worker:leaf_worker'
|
||||
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) =>
|
||||
handle === 'term_coord' ? coordinatorPaneKey : workerPaneKey
|
||||
)
|
||||
vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockReturnValue({
|
||||
terminalHandle: 'term_worker',
|
||||
paneKey: workerPaneKey,
|
||||
processIncarnation: 'runtime_test:term_worker:1'
|
||||
} as never)
|
||||
vi.spyOn(runtime, 'isTerminalRunningAgent').mockResolvedValue(true)
|
||||
vi.spyOn(runtime, 'sendTerminalAgentPrompt').mockResolvedValue({
|
||||
handle: 'term_worker',
|
||||
accepted: true,
|
||||
bytesWritten: 1
|
||||
})
|
||||
vi.spyOn(runtime, 'showTerminal').mockResolvedValue({
|
||||
handle: 'term_worker',
|
||||
connected: true,
|
||||
status: 'running'
|
||||
} as never)
|
||||
vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockReturnValue('runtime_test:term_worker:1')
|
||||
vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue({
|
||||
status: 'live',
|
||||
ptyIds: ['runtime_test:term_worker:1']
|
||||
})
|
||||
vi.spyOn(runtime, 'getTerminalOrchestrationCliCommand').mockReturnValue('orca')
|
||||
const run = db.createRun({
|
||||
objective: 'STA-3848 repro',
|
||||
coordinatorHandle: 'term_coord',
|
||||
coordinatorPaneKey
|
||||
})
|
||||
const task = db.createTask({ spec: 'injected lane', runId: run.id })
|
||||
const dispatchMethod = ORCHESTRATION_METHODS.find(
|
||||
(candidate) => candidate.name === 'orchestration.dispatch'
|
||||
)
|
||||
if (!dispatchMethod) {
|
||||
throw new Error('Missing method orchestration.dispatch')
|
||||
}
|
||||
|
||||
const result = (await dispatchMethod.handler(
|
||||
dispatchMethod.params?.parse({
|
||||
task: task.id,
|
||||
to: 'term_worker',
|
||||
from: 'term_coord',
|
||||
run: run.id,
|
||||
inject: true
|
||||
}),
|
||||
{ runtime, legacyCoordinatorRunId: run.id }
|
||||
)) as { dispatch: { id: string } }
|
||||
|
||||
expect(db.getWorkerDispatch(result.dispatch.id)).toBeUndefined()
|
||||
expect(db.getDispatchContextById(result.dispatch.id)).toMatchObject({
|
||||
assignee_handle: 'term_worker',
|
||||
assignee_pane_key: workerPaneKey,
|
||||
process_incarnation: 'runtime_test:term_worker:1',
|
||||
capability_hash: expect.any(String)
|
||||
})
|
||||
|
||||
const workerShowMethod = ORCHESTRATION_METHODS.find(
|
||||
(candidate) => candidate.name === 'orchestration.workerShow'
|
||||
)
|
||||
if (!workerShowMethod) {
|
||||
throw new Error('Missing method orchestration.workerShow')
|
||||
}
|
||||
await expect(
|
||||
workerShowMethod.handler(workerShowMethod.params?.parse({ dispatch: result.dispatch.id }), {
|
||||
runtime
|
||||
})
|
||||
).resolves.toMatchObject({
|
||||
worker: { state: 'unsupervised', stage: 'injected' },
|
||||
observation: { status: 'live', exactWorker: true }
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps context-only reads truthful without supervising the operator pane', async () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const runtime = new OrcaRuntimeService()
|
||||
runtime.setOrchestrationDb(db)
|
||||
vi.spyOn(runtime, 'showTerminal').mockResolvedValue({
|
||||
handle: 'term_worker',
|
||||
connected: true,
|
||||
status: 'running'
|
||||
} as never)
|
||||
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker')
|
||||
vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockReturnValue('runtime_test:term_worker:1')
|
||||
vi.spyOn(runtime, 'readTerminal').mockResolvedValue({
|
||||
handle: 'term_worker',
|
||||
status: 'running',
|
||||
tail: ['injected output'],
|
||||
truncated: false,
|
||||
nextCursor: null
|
||||
})
|
||||
const run = db.createRun({
|
||||
objective: 'STA-3848 repro',
|
||||
coordinatorHandle: 'term_coord',
|
||||
coordinatorPaneKey: 'tab_coord:leaf_coord'
|
||||
})
|
||||
const task = db.createTask({ spec: 'injected lane', runId: run.id })
|
||||
const dispatch = db.createDispatchContext(
|
||||
task.id,
|
||||
'term_worker',
|
||||
'tab_worker:leaf_worker',
|
||||
'launch-hash',
|
||||
'runtime_test:term_worker:1'
|
||||
)
|
||||
db.mintDispatchCapability({
|
||||
dispatchId: dispatch.id,
|
||||
paneKey: 'tab_worker:leaf_worker',
|
||||
processIncarnation: 'runtime_test:term_worker:1'
|
||||
})
|
||||
const context = { runtime }
|
||||
const call = async (name: string, params: Record<string, unknown>) => {
|
||||
const method = ORCHESTRATION_METHODS.find((candidate) => candidate.name === name)
|
||||
if (!method) {
|
||||
throw new Error(`Missing method ${name}`)
|
||||
}
|
||||
return method.handler(method.params?.parse(params), context)
|
||||
}
|
||||
|
||||
const dispatchShow = (await call('orchestration.dispatchShow', { task: task.id })) as {
|
||||
dispatch: {
|
||||
id: string
|
||||
assignee_handle: string
|
||||
assignee_pane_key: string
|
||||
process_incarnation: string
|
||||
}
|
||||
}
|
||||
expect(dispatchShow.dispatch).toMatchObject({
|
||||
id: dispatch.id,
|
||||
assignee_handle: 'term_worker',
|
||||
assignee_pane_key: 'tab_worker:leaf_worker',
|
||||
process_incarnation: 'runtime_test:term_worker:1'
|
||||
})
|
||||
expect(db.getWorkerDispatch(dispatch.id)).toBeUndefined()
|
||||
|
||||
const workerList = (await call('orchestration.workerList', { run: run.id })) as {
|
||||
workers: {
|
||||
dispatchId: string
|
||||
workerState: string
|
||||
terminalState: string | null
|
||||
agentTerminalHandle: string | null
|
||||
}[]
|
||||
}
|
||||
expect(workerList.workers).toEqual([
|
||||
expect.objectContaining({
|
||||
dispatchId: dispatch.id,
|
||||
workerState: 'unsupervised',
|
||||
terminalState: 'retained',
|
||||
agentTerminalHandle: 'term_worker'
|
||||
})
|
||||
])
|
||||
|
||||
await expect(
|
||||
call('orchestration.workerShow', { dispatch: dispatch.id })
|
||||
).resolves.toMatchObject({
|
||||
worker: { state: 'unsupervised', stage: 'injected', agent_terminal_handle: 'term_worker' },
|
||||
observation: { status: 'live', exactWorker: true }
|
||||
})
|
||||
await expect(
|
||||
call('orchestration.workerRead', { dispatch: dispatch.id, source: 'terminal' })
|
||||
).resolves.toMatchObject({
|
||||
dispatchId: dispatch.id,
|
||||
status: { worker: 'unsupervised' },
|
||||
terminal: { tail: ['injected output'] }
|
||||
})
|
||||
|
||||
await expect(call('orchestration.workerRetain', { dispatch: dispatch.id })).rejects.toThrow(
|
||||
/only a settled dispatch can retain/
|
||||
)
|
||||
|
||||
expect(
|
||||
db.settleWorkerReport({
|
||||
taskId: task.id,
|
||||
dispatchId: dispatch.id,
|
||||
outcome: 'succeeded',
|
||||
result: 'done'
|
||||
})
|
||||
).toEqual({ action: 'settled', outcome: 'succeeded', duplicate: false })
|
||||
await expect(
|
||||
call('orchestration.workerRetain', { dispatch: dispatch.id })
|
||||
).resolves.toMatchObject({
|
||||
state: 'retained',
|
||||
reason: 'no_owned_resource',
|
||||
processAction: 'none'
|
||||
})
|
||||
await expect(
|
||||
call('orchestration.workerRelease', { dispatch: dispatch.id })
|
||||
).resolves.toMatchObject({
|
||||
state: 'retained',
|
||||
reason: 'no_owned_resource',
|
||||
processAction: 'none'
|
||||
})
|
||||
})
|
||||
|
||||
it.each([
|
||||
['orchestration.workerStop', 'stopped'],
|
||||
['orchestration.workerAbandon', 'abandoned']
|
||||
] as const)('%s fences the assignment without closing the operator pane', async (name, state) => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const runtime = new OrcaRuntimeService()
|
||||
runtime.setOrchestrationDb(db)
|
||||
const closeTerminal = vi.spyOn(runtime, 'closeTerminal')
|
||||
const task = db.createTask({ spec: 'operator-owned lane' })
|
||||
const dispatch = db.createDispatchContext(
|
||||
task.id,
|
||||
'term_worker',
|
||||
'tab_worker:leaf_worker',
|
||||
'launch-hash',
|
||||
'runtime_test:term_worker:1'
|
||||
)
|
||||
const method = ORCHESTRATION_METHODS.find((candidate) => candidate.name === name)
|
||||
|
||||
if (!method) {
|
||||
throw new Error(`Missing method ${name}`)
|
||||
}
|
||||
const result = await method.handler(method.params?.parse({ dispatch: dispatch.id }), {
|
||||
runtime
|
||||
})
|
||||
|
||||
expect(result).toMatchObject({ state, processAction: 'none' })
|
||||
expect(closeTerminal).not.toHaveBeenCalled()
|
||||
expect(db.getWorkerDispatch(dispatch.id)).toBeUndefined()
|
||||
expect(db.getDispatchContextById(dispatch.id)).toMatchObject({
|
||||
status: 'failed',
|
||||
last_failure: state
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
ORCHESTRATION_WORKER_READ_SOURCES,
|
||||
type OrchestrationWorkerReadResult
|
||||
} from '../../../../shared/orchestration-worker-output'
|
||||
import { contextOnlyAbandonWarning } from '../../orchestration/context-only-dispatch-release'
|
||||
import { OrchestrationError } from '../../orchestration/orchestration-error'
|
||||
import { defineMethod, type RpcMethod } from '../core'
|
||||
import { OptionalFiniteNumber, requiredString } from '../schemas'
|
||||
@@ -10,7 +11,8 @@ import {
|
||||
callFederatedWorkerShow,
|
||||
exposeWorker,
|
||||
inspectWorkerTerminal,
|
||||
resolvePinnedFederatedServer
|
||||
resolvePinnedFederatedServer,
|
||||
showContextOnlyWorker
|
||||
} from './orchestration-worker-observation'
|
||||
import { readArchivedWorkerOutput } from './orchestration-worker-archive-read'
|
||||
import { readLegacyFederatedTerminal } from './orchestration-worker-legacy-federated-read'
|
||||
@@ -32,7 +34,7 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const dispatch = db.getDispatchContextById(params.dispatch)
|
||||
let worker = db.getWorkerDispatch(params.dispatch)
|
||||
if (!dispatch || !worker) {
|
||||
if (!dispatch) {
|
||||
throw new OrchestrationError(
|
||||
'dispatch_not_found',
|
||||
`Worker Dispatch ${params.dispatch} was not found.`
|
||||
@@ -40,6 +42,12 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
|
||||
}
|
||||
const federated = db.getFederatedDispatch(params.dispatch)
|
||||
if (federated) {
|
||||
if (!worker) {
|
||||
throw new OrchestrationError(
|
||||
'dispatch_not_found',
|
||||
`Federated Worker Dispatch ${params.dispatch} has no worker record.`
|
||||
)
|
||||
}
|
||||
const server = resolvePinnedFederatedServer(runtime, federated)
|
||||
runtime.ensureOrchestrationFederationRelay(dispatch.run_id)
|
||||
const remote = await callFederatedWorkerShow(runtime, federated)
|
||||
@@ -106,6 +114,9 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!worker) {
|
||||
return showContextOnlyWorker(runtime, db, dispatch)
|
||||
}
|
||||
if (worker.runtime_epoch && worker.runtime_epoch !== runtime.getRuntimeId()) {
|
||||
if (worker.state === 'starting') {
|
||||
worker = db.markWorkerStartUnknown(
|
||||
@@ -177,8 +188,16 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
|
||||
})
|
||||
}
|
||||
}
|
||||
const dispatch = db.getDispatchContextById(params.dispatch)
|
||||
const worker = db.getWorkerDispatch(params.dispatch)
|
||||
if (!worker?.agent_terminal_handle) {
|
||||
const terminalHandle = worker?.agent_terminal_handle ?? dispatch?.assignee_handle
|
||||
if (!dispatch) {
|
||||
throw new OrchestrationError(
|
||||
'dispatch_not_found',
|
||||
`Dispatch ${params.dispatch} was not found.`
|
||||
)
|
||||
}
|
||||
if (!terminalHandle) {
|
||||
throw new OrchestrationError(
|
||||
'dispatch_not_found',
|
||||
`Worker Dispatch ${params.dispatch} has no agent terminal.`
|
||||
@@ -189,7 +208,7 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
|
||||
return readArchivedWorkerOutput({
|
||||
db,
|
||||
dispatchId: params.dispatch,
|
||||
workerState: worker.state,
|
||||
workerState: worker?.state ?? 'unsupervised',
|
||||
resource,
|
||||
source: params.source,
|
||||
cursor: params.cursor,
|
||||
@@ -206,8 +225,8 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
|
||||
const output = await readExactWorkerOutput({
|
||||
runtime,
|
||||
dispatchId: params.dispatch,
|
||||
terminalHandle: worker.agent_terminal_handle,
|
||||
workerState: worker.state,
|
||||
terminalHandle,
|
||||
workerState: worker?.state ?? 'unsupervised',
|
||||
terminalStatus:
|
||||
observation.status === 'exited'
|
||||
? 'exited'
|
||||
@@ -220,7 +239,7 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
|
||||
: observation.status === 'exited'
|
||||
? 'exited'
|
||||
: 'live',
|
||||
attachedAt: worker.created_at,
|
||||
attachedAt: worker?.created_at ?? dispatch.dispatched_at ?? dispatch.created_at,
|
||||
source: params.source,
|
||||
cursor: params.cursor,
|
||||
limit: params.limit
|
||||
@@ -273,16 +292,3 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [
|
||||
}
|
||||
})
|
||||
]
|
||||
|
||||
function contextOnlyAbandonWarning(result: {
|
||||
state: string
|
||||
alreadySettled: boolean
|
||||
releasedCurrentTask: boolean
|
||||
}): string {
|
||||
if (result.alreadySettled) {
|
||||
return `Dispatch was already ${result.state}; no state or process changed.`
|
||||
}
|
||||
return result.releasedCurrentTask
|
||||
? 'The assignment was abandoned; its unsupervised terminal process was retained.'
|
||||
: 'The superseded assignment was abandoned without changing the current Task or terminal process.'
|
||||
}
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
import type { OrcaRuntimeService } from '../../orca-runtime'
|
||||
import type { OrchestrationDb } from '../../orchestration/db'
|
||||
import { OrchestrationError } from '../../orchestration/orchestration-error'
|
||||
import type { FederatedDispatchRow, WorkerDispatchRow } from '../../orchestration/types'
|
||||
import type {
|
||||
DispatchContextRow,
|
||||
FederatedDispatchRow,
|
||||
WorkerDispatchRow
|
||||
} from '../../orchestration/types'
|
||||
|
||||
export async function inspectWorkerTerminal(
|
||||
runtime: OrcaRuntimeService,
|
||||
@@ -15,17 +19,19 @@ export async function inspectWorkerTerminal(
|
||||
reason?: string
|
||||
}> {
|
||||
const worker = db.getWorkerDispatch(dispatchId)
|
||||
if (!worker?.agent_terminal_handle) {
|
||||
const terminalHandle =
|
||||
worker?.agent_terminal_handle ?? db.getDispatchContextById(dispatchId)?.assignee_handle
|
||||
if (!terminalHandle) {
|
||||
return { terminal: null, exact: false, status: 'unattached' }
|
||||
}
|
||||
const terminal = await runtime.showTerminal(worker.agent_terminal_handle).catch(() => null)
|
||||
const terminal = await runtime.showTerminal(terminalHandle).catch(() => null)
|
||||
if (!terminal) {
|
||||
return { terminal: null, exact: false, status: 'missing' }
|
||||
}
|
||||
const exact = db.isDispatchProcessCurrent({
|
||||
dispatchId,
|
||||
paneKey: runtime.getTerminalPaneKey(worker.agent_terminal_handle),
|
||||
processIncarnation: runtime.getTerminalProcessIncarnation(worker.agent_terminal_handle)
|
||||
paneKey: runtime.getTerminalPaneKey(terminalHandle),
|
||||
processIncarnation: runtime.getTerminalProcessIncarnation(terminalHandle)
|
||||
})
|
||||
if (!exact) {
|
||||
return { terminal, exact, status: 'identity_changed' }
|
||||
@@ -33,7 +39,7 @@ export async function inspectWorkerTerminal(
|
||||
// Why: the aggregate inventory only iterates registered providers, so a dropped
|
||||
// relay clears `connected` for every remote PTY at once. Lost contact is not a
|
||||
// death certificate, and the verdict is the only field that can tell them apart.
|
||||
const verdict = runtime.getTerminalLivenessVerdict?.(worker.agent_terminal_handle) ?? null
|
||||
const verdict = runtime.getTerminalLivenessVerdict?.(terminalHandle) ?? null
|
||||
if (verdict?.status === 'unverifiable') {
|
||||
return { terminal, exact, status: 'unverifiable', reason: verdict.reason }
|
||||
}
|
||||
@@ -47,6 +53,43 @@ export async function inspectWorkerTerminal(
|
||||
}
|
||||
}
|
||||
|
||||
export function exposeContextOnlyWorker(dispatch: DispatchContextRow) {
|
||||
return {
|
||||
dispatch_id: dispatch.id,
|
||||
runtime_epoch: null,
|
||||
state: 'unsupervised' as const,
|
||||
stage: dispatch.capability_hash ? 'injected' : 'context_only',
|
||||
worktree_id: null,
|
||||
agent_terminal_handle: dispatch.assignee_handle,
|
||||
setup_state: 'not_applicable',
|
||||
effects: [],
|
||||
residualResources: [],
|
||||
startOptions: {},
|
||||
last_error: dispatch.last_failure,
|
||||
created_at: dispatch.created_at,
|
||||
updated_at: dispatch.completed_at ?? dispatch.created_at
|
||||
}
|
||||
}
|
||||
|
||||
export async function showContextOnlyWorker(
|
||||
runtime: OrcaRuntimeService,
|
||||
db: OrchestrationDb,
|
||||
dispatch: DispatchContextRow
|
||||
) {
|
||||
const observation = await inspectWorkerTerminal(runtime, db, dispatch.id)
|
||||
return {
|
||||
dispatch,
|
||||
worker: exposeContextOnlyWorker(dispatch),
|
||||
terminal: observation.exact ? observation.terminal : null,
|
||||
observation: {
|
||||
status: observation.status,
|
||||
exactWorker: observation.exact,
|
||||
...(observation.reason ? { reason: observation.reason } : {})
|
||||
},
|
||||
terminalResource: null
|
||||
}
|
||||
}
|
||||
|
||||
export function exposeWorker(worker: WorkerDispatchRow) {
|
||||
return {
|
||||
...worker,
|
||||
|
||||
@@ -320,7 +320,7 @@ describe('orchestration worker release', () => {
|
||||
setup()
|
||||
const { dispatchId } = await startWorker()
|
||||
if (state === 'stopped') {
|
||||
db.beginWorkerStop(dispatchId)
|
||||
db.beginWorkerStop(dispatchId, runtime.getRuntimeId())
|
||||
db.settleWorkerStop(dispatchId)
|
||||
} else {
|
||||
db.abandonWorkerDispatch(dispatchId)
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { z } from 'zod'
|
||||
import { OrchestrationError } from '../../orchestration/orchestration-error'
|
||||
import type { WorkerTerminalListState } from '../../orchestration/worker-terminal-ownership'
|
||||
import { defineMethod, type RpcMethod } from '../core'
|
||||
import { requiredString } from '../schemas'
|
||||
@@ -99,12 +98,6 @@ export const ORCHESTRATION_WORKER_RELEASE_METHODS: RpcMethod[] = [
|
||||
params: WorkerDispatchParams,
|
||||
handler: (params, { runtime }) => {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
if (!db.getWorkerDispatch(params.dispatch)) {
|
||||
throw new OrchestrationError(
|
||||
'dispatch_not_found',
|
||||
`Worker Dispatch ${params.dispatch} was not found.`
|
||||
)
|
||||
}
|
||||
const retained = db.retainWorkerTerminalResource(params.dispatch)
|
||||
if (retained.disposition === 'already_released') {
|
||||
return {
|
||||
|
||||
@@ -28,6 +28,7 @@ describe('federated worker stop capability', () => {
|
||||
})
|
||||
const runtime = {
|
||||
getOrchestrationDb: () => db,
|
||||
getRuntimeId: () => 'runtime_current',
|
||||
resolveOrchestrationWorkerServer: () => ({
|
||||
environmentId: 'environment_linux',
|
||||
name: 'linux',
|
||||
@@ -48,6 +49,7 @@ describe('federated worker stop capability', () => {
|
||||
}
|
||||
})
|
||||
).resolves.toMatchObject({ state: 'stop_unknown', processAction: 'none' })
|
||||
expect(db.beginWorkerStop).toHaveBeenCalledWith('ctx_remote', 'runtime_current')
|
||||
expect(markWorkerStopUnknown).toHaveBeenCalledWith(
|
||||
'ctx_remote',
|
||||
'Connected server linux cannot prove the worker stop outcome.'
|
||||
|
||||
@@ -55,7 +55,8 @@ describe('worker-stop against a terminal we lost contact with', () => {
|
||||
processIncarnation: 'runtime:pty:1',
|
||||
worktreeId: 'repo::worktree',
|
||||
setupState: 'not_applicable',
|
||||
effects: [{ kind: 'terminal', action: 'created', id: 'term_worker' }]
|
||||
effects: [{ kind: 'terminal', action: 'created', id: 'term_worker' }],
|
||||
terminalOwnership: 'created'
|
||||
})
|
||||
db.markWorkerDispatchReady(started.dispatch.id)
|
||||
return started.dispatch
|
||||
@@ -141,6 +142,94 @@ describe('worker-stop against a terminal we lost contact with', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('does not close a pane after user input transfers ownership', async () => {
|
||||
vi.spyOn(runtime, 'showTerminal').mockResolvedValue({
|
||||
handle: 'term_worker',
|
||||
worktreeId: 'repo::worktree',
|
||||
connected: true,
|
||||
status: 'running'
|
||||
} as never)
|
||||
vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue({
|
||||
status: 'live',
|
||||
ptyIds: ['runtime:pty:1']
|
||||
})
|
||||
const closeTerminal = vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({
|
||||
handle: 'term_worker',
|
||||
tabId: 'tab_worker',
|
||||
ptyKilled: true
|
||||
})
|
||||
const dispatch = createWorker()
|
||||
|
||||
await expect(
|
||||
call('orchestration.workerTerminalUserInput', {
|
||||
paneKey: 'tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'
|
||||
})
|
||||
).resolves.toMatchObject({ changed: 1 })
|
||||
|
||||
await expect(
|
||||
call('orchestration.workerStop', { dispatch: dispatch.id })
|
||||
).resolves.toMatchObject({
|
||||
state: 'stop_unknown',
|
||||
processAction: 'none',
|
||||
lastError: 'The worker terminal is user_owned; no terminal was closed.'
|
||||
})
|
||||
expect(closeTerminal).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not close a legacy worker terminal without an ownership record', async () => {
|
||||
vi.spyOn(runtime, 'showTerminal').mockResolvedValue({
|
||||
handle: 'term_worker',
|
||||
worktreeId: 'repo::worktree',
|
||||
connected: true,
|
||||
status: 'running'
|
||||
} as never)
|
||||
vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue({
|
||||
status: 'live',
|
||||
ptyIds: ['runtime:pty:1']
|
||||
})
|
||||
const closeTerminal = vi.spyOn(runtime, 'closeTerminal')
|
||||
const dispatch = createWorker()
|
||||
const resource = db.getWorkerTerminalResourceByOwner(dispatch.id)
|
||||
if (!resource) {
|
||||
throw new Error('Expected worker terminal resource')
|
||||
}
|
||||
;(db as unknown as { db: { prepare: (sql: string) => { run: (id: string) => void } } }).db
|
||||
.prepare('DELETE FROM worker_terminal_resources WHERE id = ?')
|
||||
.run(resource.id)
|
||||
|
||||
await expect(
|
||||
call('orchestration.workerStop', { dispatch: dispatch.id })
|
||||
).resolves.toMatchObject({
|
||||
state: 'stop_unknown',
|
||||
processAction: 'none',
|
||||
lastError: 'The worker terminal is unproven; no terminal was closed.'
|
||||
})
|
||||
expect(closeTerminal).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('linearizes stop before a concurrent user takeover', () => {
|
||||
const dispatch = createWorker()
|
||||
expect(db.beginWorkerStop(dispatch.id, runtime.getRuntimeId()).disposition).toBe('stopping')
|
||||
|
||||
expect(db.markWorkerTerminalUserOwned('tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb')).toBe(
|
||||
0
|
||||
)
|
||||
expect(db.getWorkerTerminalResourceByOwner(dispatch.id)).toMatchObject({
|
||||
ownership_state: 'owned',
|
||||
release_state: 'not_requested'
|
||||
})
|
||||
})
|
||||
|
||||
it('does not let abandon overwrite a stop in progress', () => {
|
||||
const dispatch = createWorker()
|
||||
expect(db.beginWorkerStop(dispatch.id, runtime.getRuntimeId()).disposition).toBe('stopping')
|
||||
|
||||
expect(() => db.abandonWorkerDispatch(dispatch.id)).toThrow(
|
||||
'is stopping; wait for worker-stop to settle before abandoning'
|
||||
)
|
||||
expect(db.getWorkerDispatch(dispatch.id)?.state).toBe('stopping')
|
||||
})
|
||||
|
||||
it('still reports a locally observed exit as exited', async () => {
|
||||
const dispatch = createWorker()
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ export const ORCHESTRATION_WORKER_STOP_METHODS: RpcMethod[] = [
|
||||
)
|
||||
}
|
||||
const server = resolvePinnedFederatedServer(runtime, federated)
|
||||
const begun = db.beginWorkerStop(params.dispatch)
|
||||
const begun = db.beginWorkerStop(params.dispatch, runtime.getRuntimeId())
|
||||
if (begun.disposition === 'already_settled') {
|
||||
return settledReceipt(params.dispatch, begun.worker.state)
|
||||
}
|
||||
@@ -97,7 +97,7 @@ export const ORCHESTRATION_WORKER_STOP_METHODS: RpcMethod[] = [
|
||||
}
|
||||
}
|
||||
|
||||
const begun = db.beginWorkerStop(params.dispatch)
|
||||
const begun = db.beginWorkerStop(params.dispatch, runtime.getRuntimeId())
|
||||
if (begun.disposition === 'already_settled') {
|
||||
return settledReceipt(params.dispatch, begun.worker.state)
|
||||
}
|
||||
@@ -137,6 +137,18 @@ export const ORCHESTRATION_WORKER_STOP_METHODS: RpcMethod[] = [
|
||||
'none'
|
||||
)
|
||||
}
|
||||
const resource = db.getWorkerTerminalResourceByOwner(params.dispatch)
|
||||
if (!resource || resource.ownership_state !== 'owned') {
|
||||
const ownership = resource?.ownership_state ?? 'unproven'
|
||||
return unknownReceipt(
|
||||
params.dispatch,
|
||||
db.markWorkerStopUnknown(
|
||||
params.dispatch,
|
||||
`The worker terminal is ${ownership}; no terminal was closed.`
|
||||
),
|
||||
'none'
|
||||
)
|
||||
}
|
||||
try {
|
||||
const close = await runtime.closeTerminal(handle)
|
||||
if (!close.ptyKilled) {
|
||||
|
||||
@@ -3,6 +3,14 @@ import { OrcaRuntimeService } from '../../orca-runtime'
|
||||
import { OrchestrationDb } from '../../orchestration/db'
|
||||
import { ORCHESTRATION_METHODS } from './orchestration'
|
||||
|
||||
function deferred<T>(): { promise: Promise<T>; resolve: (value: T) => void } {
|
||||
let resolve!: (value: T) => void
|
||||
const promise = new Promise<T>((promiseResolve) => {
|
||||
resolve = promiseResolve
|
||||
})
|
||||
return { promise, resolve }
|
||||
}
|
||||
|
||||
describe('orchestration worker recovery', () => {
|
||||
let db: OrchestrationDb
|
||||
let runtime: OrcaRuntimeService
|
||||
@@ -64,7 +72,8 @@ describe('orchestration worker recovery', () => {
|
||||
processIncarnation: 'runtime:pty:1',
|
||||
worktreeId: 'repo::worktree',
|
||||
setupState: 'not_applicable',
|
||||
effects: [{ kind: 'terminal', action: 'created', id: 'term_worker' }]
|
||||
effects: [{ kind: 'terminal', action: 'created', id: 'term_worker' }],
|
||||
terminalOwnership: 'created'
|
||||
})
|
||||
if (ready) {
|
||||
db.markWorkerDispatchReady(started.dispatch.id)
|
||||
@@ -105,6 +114,39 @@ describe('orchestration worker recovery', () => {
|
||||
expect(db.getTask(task.id)?.status).toBe('blocked')
|
||||
})
|
||||
|
||||
it('keeps an in-flight stop fenced during runtime-epoch reconciliation', async () => {
|
||||
const { dispatch } = createWorker('previous_runtime')
|
||||
const pendingObservation = deferred<Awaited<ReturnType<OrcaRuntimeService['showTerminal']>>>()
|
||||
vi.mocked(runtime.showTerminal)
|
||||
.mockReturnValueOnce(pendingObservation.promise)
|
||||
.mockResolvedValue({
|
||||
handle: 'term_worker',
|
||||
worktreeId: 'repo::worktree',
|
||||
connected: true,
|
||||
status: 'running'
|
||||
} as never)
|
||||
|
||||
const stop = call('orchestration.workerStop', { dispatch: dispatch.id })
|
||||
await vi.waitFor(() => expect(runtime.showTerminal).toHaveBeenCalledTimes(1))
|
||||
await expect(
|
||||
call('orchestration.workerShow', { dispatch: dispatch.id })
|
||||
).resolves.toMatchObject({ worker: { state: 'stopping' } })
|
||||
await expect(call('orchestration.workerAbandon', { dispatch: dispatch.id })).rejects.toThrow(
|
||||
'is stopping; wait for worker-stop to settle before abandoning'
|
||||
)
|
||||
|
||||
pendingObservation.resolve({
|
||||
handle: 'term_worker',
|
||||
worktreeId: 'repo::worktree',
|
||||
connected: true,
|
||||
status: 'running'
|
||||
} as never)
|
||||
await expect(stop).resolves.toMatchObject({
|
||||
state: 'stopped',
|
||||
processAction: 'closed_agent_terminal'
|
||||
})
|
||||
})
|
||||
|
||||
it('does not adopt or stop a same-looking pane with a new process incarnation', async () => {
|
||||
const { task, dispatch } = createWorker()
|
||||
vi.mocked(runtime.getTerminalProcessIncarnation).mockReturnValue('runtime:pty:2')
|
||||
@@ -184,7 +226,7 @@ describe('orchestration worker recovery', () => {
|
||||
|
||||
it('turns an interrupted stop into unknown after runtime restart', async () => {
|
||||
const { task, dispatch } = createWorker('previous_runtime')
|
||||
db.beginWorkerStop(dispatch.id)
|
||||
db.beginWorkerStop(dispatch.id, 'previous_runtime')
|
||||
|
||||
await expect(
|
||||
call('orchestration.workerShow', { dispatch: dispatch.id })
|
||||
@@ -213,7 +255,7 @@ describe('orchestration worker recovery', () => {
|
||||
}
|
||||
})
|
||||
db.markWorkerStartUnknown(started.dispatch.id, 'remote_attach', 'response lost')
|
||||
db.beginWorkerStop(started.dispatch.id)
|
||||
db.beginWorkerStop(started.dispatch.id, runtime.getRuntimeId())
|
||||
db.markWorkerStopUnknown(started.dispatch.id, 'stop response lost')
|
||||
vi.spyOn(runtime, 'resolveOrchestrationWorkerServer').mockReturnValue({
|
||||
environmentId: 'environment_windows',
|
||||
|
||||
Reference in New Issue
Block a user