fix(agent-status): preserve readiness uncertainty custody

This commit is contained in:
Brennan Benson
2026-09-15 11:16:26 -07:00
parent 56e7df996d
commit 47ba36daff
11 changed files with 446 additions and 37 deletions
@@ -23,7 +23,7 @@ type FakeWaiter = {
function createFakeRuntime(
initial: Partial<FakePane>,
readinessOptions: { resolveUnknown?: boolean } = {}
readinessOptions: { resolveUnknown?: boolean; resolveBusy?: boolean } = {}
) {
const pane: FakePane = {
lastAgentStatus: null,
@@ -61,6 +61,12 @@ function createFakeRuntime(
readiness: { state: 'unknown' as const }
})
}
if (readinessOptions.resolveBusy && waitOptions?.timeoutMs !== undefined) {
return Promise.resolve({
satisfied: false,
readiness: { state: 'busy' as const }
})
}
return new Promise((resolve, reject) => {
const waiter: FakeWaiter = {
resolve,
@@ -210,6 +216,22 @@ describe('createRuntimeAutomationRunTerminalObserver', () => {
await run.promise
})
it('accepts a title-only working observation before the pane returns to ready', async () => {
const runtime = createFakeRuntime(
{ lastAgentStatus: null, paneTitle: '✳ Claude — working' },
{ resolveBusy: true }
)
const run = observe(runtime)
await vi.advanceTimersByTimeAsync(1_000)
expect(run.settled).toEqual([])
runtime.setPane({ paneTitle: '✳ Claude — idle' })
await vi.advanceTimersByTimeAsync(10)
expect(run.settled[0]?.status).toBe('completed')
await run.promise
})
it('stops re-arming the tui-idle wait instead of looping for the process lifetime', async () => {
const runtime = createFakeRuntime({ lastAgentStatus: 'working' })
const run = observe(runtime)
@@ -137,6 +137,29 @@ export function markRemoteAttachmentReady(
return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow
}
/** Preserve a prepared attachment and its capability when readiness remains unverifiable. */
export function markRemoteAttachmentStartUnknown(
this: OrchestrationDb,
dispatchId: string,
stage: string,
reason: string
): RemoteDispatchAttachmentRow {
const result = this.db
.prepare(
`UPDATE remote_dispatch_attachments
SET state = 'start_unknown', stage = ?, last_error = ?, updated_at = datetime('now')
WHERE dispatch_id = ? AND state = 'starting'`
)
.run(stage, reason, dispatchId)
if (result.changes !== 1) {
throw new OrchestrationError(
'dispatch_inactive',
`Remote Dispatch ${dispatchId} is not starting.`
)
}
return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow
}
export function failRemoteAttachment(
this: OrchestrationDb,
dispatchId: string,
@@ -209,6 +232,7 @@ export function isRemoteAttachmentProcessCurrent(
export type RemoteDispatchAttachmentAuthorityMethods = {
prepareRemoteAttachmentAuthority: typeof prepareRemoteAttachmentAuthority
markRemoteAttachmentReady: typeof markRemoteAttachmentReady
markRemoteAttachmentStartUnknown: typeof markRemoteAttachmentStartUnknown
failRemoteAttachment: typeof failRemoteAttachment
verifyRemoteAttachmentAuthority: typeof verifyRemoteAttachmentAuthority
isRemoteAttachmentProcessCurrent: typeof isRemoteAttachmentProcessCurrent
@@ -218,6 +242,7 @@ export function attachRemoteDispatchAttachmentAuthority(ctor: { prototype: objec
Object.assign(ctor.prototype, {
prepareRemoteAttachmentAuthority,
markRemoteAttachmentReady,
markRemoteAttachmentStartUnknown,
failRemoteAttachment,
verifyRemoteAttachmentAuthority,
isRemoteAttachmentProcessCurrent
@@ -3,6 +3,7 @@ import type { OrchestrationWorkerLaunchReceipt } from '../worker/worker-launch-p
export type RemoteFederatedWorkerStartReceipt = {
dispatchId: string
state: string
stage?: string
runtimeEpoch?: string
worktreeId?: string
terminalHandle?: string
@@ -52,6 +52,7 @@ describe('federated worker start receipt validation', () => {
}
})
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the RPC receipt is checked by toMatchObject below.
const result = (await startFederatedWorker({
params: {
task: task.id,
@@ -88,4 +89,83 @@ describe('federated worker start receipt validation', () => {
})
}
})
it('retains remote terminal custody when readiness is unknown before prompt delivery', async () => {
const db = new OrchestrationDb(':memory:')
const runtime = new OrcaRuntimeService()
runtime.setOrchestrationDb(db)
databases.push(db)
const run = db.createRun({
objective: 'federated readiness',
coordinatorHandle: 'term_coord',
coordinatorPaneKey: 'tab_coord:leaf_coord'
})
const task = db.createTask({ spec: 'remote work', runId: run.id })
vi.spyOn(runtime, 'resolveOrchestrationWorkerServer').mockReturnValue({
environmentId: 'environment_remote',
name: 'remote',
peerFingerprint: 'remote_peer',
pairingRevision: 73
})
vi.spyOn(runtime, 'callOrchestrationWorkerServer').mockImplementation(
async (_environmentId, method, params) => {
if (method === 'status.get') {
return {
capabilities: [
ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY,
ORCHESTRATION_FEDERATION_RUNTIME_CAPABILITY
]
}
}
return {
dispatchId: (params as { dispatchId: string }).dispatchId,
state: 'outcome_unknown',
runtimeEpoch: 'epoch_remote',
worktreeId: 'worktree_remote',
terminalHandle: 'term_remote',
setup: { state: 'running' },
effects: [{ kind: 'terminal', action: 'created', id: 'term_remote' }],
residualResources: [{ kind: 'terminal', id: 'term_remote' }],
failedStage: 'agent_readiness',
lastError: 'Agent startup readiness could not be verified (unknown).'
}
}
)
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the RPC receipt is checked by toMatchObject below.
const result = (await startFederatedWorker({
params: {
task: task.id,
from: 'term_coord',
on: 'remote',
worktree: 'id:worktree_remote',
agent: 'codex'
},
runtime,
db,
runId: run.id,
task,
orchestrationMutation: {
callerFingerprint: 'caller',
requestId: 'remote_unknown',
method: 'orchestration.workerStart',
payloadHash: 'payload'
}
})) as { dispatchId: string; state: string; terminalHandle?: string }
expect(result).toMatchObject({
state: 'outcome_unknown',
terminalHandle: 'term_remote'
})
expect(db.getWorkerDispatch(result.dispatchId)).toMatchObject({
state: 'start_unknown',
agent_terminal_handle: 'term_remote',
worktree_id: 'worktree_remote'
})
expect(db.getFederatedDispatch(result.dispatchId)).toMatchObject({
remote_runtime_epoch: 'epoch_remote',
remote_worktree_id: 'worktree_remote',
remote_terminal_handle: 'term_remote'
})
})
})
@@ -1,4 +1,6 @@
import type { OrchestrationWorkerLaunchReceipt } from '../worker/worker-launch-preferences'
import type { OrchestrationDb } from '../../../../orchestration/db'
import type { RemoteFederatedWorkerStartReceipt } from './federated-attach-receipt'
export type RemoteStartReceipt = {
dispatchId: string
@@ -29,7 +31,13 @@ export function federatedUnknownReceipt(
worker: { dispatch_id: string; state: string; stage: string; last_error: string | null },
taskId: string,
serverName: string,
launch: OrchestrationWorkerLaunchReceipt
launch: OrchestrationWorkerLaunchReceipt,
details?: {
setup?: { state: string }
terminalHandle?: string
effects?: unknown[]
residualResources?: unknown[]
}
): unknown {
return {
taskId,
@@ -38,13 +46,60 @@ export function federatedUnknownReceipt(
stage: worker.stage,
server: { name: serverName },
launch,
...(details?.setup ? { setup: details.setup } : {}),
...(details?.terminalHandle ? { terminalHandle: details.terminalHandle } : {}),
...(details?.effects ? { effects: details.effects } : { effects: [] }),
...(details?.residualResources
? { residualResources: details.residualResources }
: { residualResources: [] }),
failedStage: worker.stage,
lastError: worker.last_error,
effects: [],
residualResources: [],
nextCommands: [
`orca orchestration worker-show --dispatch ${worker.dispatch_id} --json`,
...(details?.terminalHandle
? [`orca terminal read --terminal ${details.terminalHandle} --screen`]
: []),
`orca orchestration worker-abandon --dispatch ${worker.dispatch_id} --json`
]
}
}
export function preserveFederatedWorkerStartUnknown(args: {
db: OrchestrationDb
dispatchId: string
taskId: string
serverName: string
launch: OrchestrationWorkerLaunchReceipt
remote: RemoteFederatedWorkerStartReceipt
}): unknown {
const stage = args.remote.failedStage ?? args.remote.stage ?? 'remote_attach'
if (args.remote.runtimeEpoch && args.remote.worktreeId && args.remote.terminalHandle) {
args.db.updateFederatedDispatchResources({
dispatchId: args.dispatchId,
remoteRuntimeEpoch: args.remote.runtimeEpoch,
worktreeId: args.remote.worktreeId,
terminalHandle: args.remote.terminalHandle
})
}
args.db.recordWorkerStage({
dispatchId: args.dispatchId,
stage,
worktreeId: args.remote.worktreeId,
terminalHandle: args.remote.terminalHandle,
setupState: args.remote.setup?.state,
effects: args.remote.effects,
residualResources: args.remote.residualResources
})
const worker = args.db.markWorkerStartUnknown(
args.dispatchId,
stage,
args.remote.lastError ?? 'The worker server reported an unknown start outcome.',
args.remote.effects
)
return federatedUnknownReceipt(worker, args.taskId, args.serverName, args.launch, {
setup: args.remote.setup,
terminalHandle: args.remote.terminalHandle,
effects: args.remote.effects,
residualResources: args.remote.residualResources
})
}
@@ -29,7 +29,8 @@ import {
import { isWorkerStartTimeoutWithinTimerLimit } from '../../../../../../shared/orchestration-timing-budgets'
import {
federatedUnknownReceipt,
isKnownRemoteStartFailure
isKnownRemoteStartFailure,
preserveFederatedWorkerStartUnknown
} from './federated-worker-start-receipts'
import { parseTaskDeps } from '../worker/task-deps-argument'
@@ -239,12 +240,14 @@ export async function startFederatedWorker(args: {
}
}
if (remote.state === 'outcome_unknown') {
const worker = db.markWorkerStartUnknown(
started.dispatch.id,
remote.failedStage ?? 'remote_attach',
remote.lastError ?? 'The worker server reported an unknown start outcome.'
)
return federatedUnknownReceipt(worker, taskForRemote.id, server.name, launch)
return preserveFederatedWorkerStartUnknown({
db,
dispatchId: started.dispatch.id,
taskId: taskForRemote.id,
serverName: server.name,
launch,
remote
})
}
const worker = db.failWorkerStart(
started.dispatch.id,
@@ -0,0 +1,123 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { RuntimeRpcResponse } from '../../../../../../shared/runtime-rpc-envelope'
import { OrcaRuntimeService } from '../../../../orca-runtime'
import { OrchestrationDb } from '../../../../orchestration/db'
import type { OrchestrationEnvironmentTransport } from '../../../../orchestration/environment-transport'
import { RpcDispatcher } from '../../../dispatcher'
import { ORCHESTRATION_METHODS } from '../../orchestration'
import { createFederationWorkerStartRequest } from './federation-request.test-support'
import { configureFederationWorkerRuntime } from './federation-runtime.test-support'
describe('federated readiness custody', () => {
let homeDb: OrchestrationDb
let workerDb: OrchestrationDb
let homeRuntime: OrcaRuntimeService
let workerRuntime: OrcaRuntimeService
let homeDispatcher: RpcDispatcher
beforeEach(() => {
homeDb = new OrchestrationDb(':memory:')
workerDb = new OrchestrationDb(':memory:')
workerRuntime = new OrcaRuntimeService()
workerRuntime.setOrchestrationDb(workerDb)
configureFederationWorkerRuntime(workerRuntime)
const workerDispatcher = new RpcDispatcher({
runtime: workerRuntime,
methods: ORCHESTRATION_METHODS
})
const transport: OrchestrationEnvironmentTransport = {
resolve: () => ({
environmentId: 'environment_windows',
name: 'windows',
peerFingerprint: 'windows_peer_fingerprint',
pairingRevision: 73
}),
call: async (_selector, method, params, _timeoutMs, envelope) => {
if (method === 'status.get') {
return {
id: 'status',
ok: true,
result: workerRuntime.getStatus(),
_meta: { runtimeId: workerRuntime.getRuntimeId() }
}
}
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: RpcDispatcher returns the transport envelope required by this adapter.
return (await workerDispatcher.dispatch({
id: `remote_${method}`,
authToken: 'run-home-device-token',
method,
params,
orchestrationContractVersion: envelope?.orchestrationContractVersion,
orchestrationRequestId: envelope?.orchestrationRequestId,
orchestrationCapability: envelope?.orchestrationCapability
})) as RuntimeRpcResponse<unknown>
}
}
homeRuntime = new OrcaRuntimeService(null, undefined, {
orchestrationEnvironmentTransport: transport
})
homeRuntime.setOrchestrationDb(homeDb)
homeDispatcher = new RpcDispatcher({ runtime: homeRuntime, methods: ORCHESTRATION_METHODS })
vi.spyOn(homeRuntime, 'getTerminalPaneKey').mockReturnValue(
'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
)
})
afterEach(() => {
homeRuntime.stopOrchestrationFederationRelay()
homeDb.close()
workerDb.close()
})
it('preserves paired-runtime custody when startup readiness is unsupported', async () => {
vi.mocked(workerRuntime.waitForTerminal).mockResolvedValueOnce({
handle: 'term_windows_worker',
condition: 'tui-idle',
satisfied: false,
status: 'running',
exitCode: null,
readiness: { state: 'unsupported', source: 'capability', agent: 'codex' }
})
const run = homeDb.createRun({
objective: 'Mac to Windows',
coordinatorHandle: 'term_coord',
coordinatorPaneKey: 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
})
const task = homeDb.createTask({ spec: 'Audit Windows behavior', runId: run.id })
const response = await homeDispatcher.dispatch(createFederationWorkerStartRequest(task.id))
expect(response).toMatchObject({
ok: true,
result: {
state: 'outcome_unknown',
stage: 'agent_readiness',
terminalHandle: 'term_windows_worker'
}
})
const dispatch = homeDb.getDispatchContext(task.id)
expect(dispatch).toBeDefined()
if (!dispatch) {
return
}
expect(homeDb.getTask(task.id)?.status).toBe('blocked')
expect(homeDb.getWorkerDispatch(dispatch.id)).toMatchObject({
state: 'start_unknown',
worktree_id: 'repo::windows-worktree',
agent_terminal_handle: 'term_windows_worker'
})
expect(homeDb.getFederatedDispatch(dispatch.id)).toMatchObject({
remote_runtime_epoch: workerRuntime.getRuntimeId(),
remote_worktree_id: 'repo::windows-worktree',
remote_terminal_handle: 'term_windows_worker'
})
expect(workerDb.getRemoteDispatchAttachment(dispatch.id)).toMatchObject({
state: 'start_unknown',
stage: 'agent_readiness',
worktree_id: 'repo::windows-worktree',
terminal_handle: 'term_windows_worker',
capability_hash: expect.any(String)
})
expect(workerRuntime.sendTerminalAgentPrompt).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,83 @@
import type { RuntimeTerminalWait } from '../../../../../../shared/runtime-types'
import { describeTerminalWaitBlockedReason } from '../../../../../../shared/terminal-wait-blocked-reason-legacy-alias'
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import type { OrchestrationDb } from '../../../../orchestration/db'
import type { WorkerSetupReceipt } from '../worker/worker-topology'
import type { OrchestrationWorkerLaunchReceipt } from '../worker/worker-launch-preferences'
import type { FederationEffect } from './federation-effects'
type FederatedReadinessOutcome =
| { state: 'ready'; capability: string }
| { state: 'failed'; stage: string; reason: string }
| { state: 'unknown'; receipt: unknown }
export function prepareFederatedReadinessOutcome(args: {
runtime: OrcaRuntimeService
db: OrchestrationDb
dispatchId: string
wait: RuntimeTerminalWait
worktreeId: string
terminalHandle: string
terminalOwnership: 'created' | 'external'
setup: WorkerSetupReceipt
launch: OrchestrationWorkerLaunchReceipt
effects: FederationEffect[]
}): FederatedReadinessOutcome {
const reason = args.wait.blockedReason
? `Agent startup blocked: ${describeTerminalWaitBlockedReason(args.wait.blockedReason)}`
: `Agent startup readiness could not be verified (${args.wait.readiness?.state ?? args.wait.status}); no prompt was submitted.`
if (!args.wait.satisfied) {
const setupFailed =
args.setup.startupPolicy === 'wait-for-setup' &&
(args.setup.state === 'failed' || args.wait.status === 'exited')
const knownAgentExit =
args.setup.startupPolicy !== 'wait-for-setup' && args.wait.status === 'exited'
if (setupFailed || knownAgentExit) {
return { state: 'failed', stage: setupFailed ? 'setup_wait' : 'agent_readiness', reason }
}
}
const authority = args.runtime.getOrchestrationDispatchAuthority(args.terminalHandle)
const paneKey = authority?.paneKey ?? args.runtime.getTerminalPaneKey(args.terminalHandle)
const processIncarnation =
authority?.processIncarnation ?? args.runtime.getTerminalProcessIncarnation(args.terminalHandle)
if (!paneKey || !processIncarnation) {
throw new Error('stable_pane_required')
}
const capability = args.db.prepareRemoteAttachmentAuthority({
dispatchId: args.dispatchId,
paneKey,
processIncarnation,
worktreeId: args.worktreeId,
terminalHandle: args.terminalHandle,
setupState: args.setup.state,
effects: args.effects,
hostScope: authority?.hostScope ? JSON.stringify(authority.hostScope) : null,
terminalOwnership: args.terminalOwnership
})
if (args.wait.satisfied) {
return { state: 'ready', capability }
}
const attachment = args.db.markRemoteAttachmentStartUnknown(
args.dispatchId,
'agent_readiness',
reason
)
return {
state: 'unknown',
receipt: {
dispatchId: args.dispatchId,
state: 'outcome_unknown',
stage: attachment.stage,
runtimeEpoch: args.runtime.getRuntimeId(),
worktreeId: args.worktreeId,
terminalHandle: args.terminalHandle,
setup: args.setup,
launch: args.launch,
effects: args.effects,
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: attachment authority persists residual_resources as a JSON array.
residualResources: JSON.parse(attachment.residual_resources) as unknown[]
}
}
}
@@ -1,5 +1,4 @@
import type { TuiAgent } from '../../../../../../shared/tui-agent'
import { describeTerminalWaitBlockedReason } from '../../../../../../shared/terminal-wait-blocked-reason-legacy-alias'
import { buildDispatchPreamble } from '../../../../orchestration/preamble'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import { defineMethod } from '../../../core'
@@ -24,6 +23,7 @@ import {
resolveWorkerStartReadinessTimeoutMs
} from '../../../../../../shared/orchestration-timing-budgets'
import { assertWorkerStartTaskSpecWithinPromptBudget } from '../worker/worker-start-prompt-budget'
import { prepareFederatedReadinessOutcome } from './federation-readiness-outcome'
export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
defineMethod({
@@ -217,34 +217,27 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
timeoutMs: readinessTimeoutMs
})
persistFederatedSetupWaitOutcome({ ...setupStage, wait })
if (!wait.satisfied) {
if (setup.state === 'failed') {
failedStage = 'setup_wait'
}
throw new Error(
wait.blockedReason
? `Agent startup blocked: ${describeTerminalWaitBlockedReason(wait.blockedReason)}`
: `Agent did not become ready (${wait.status}).`
)
}
const authority = runtime.getOrchestrationDispatchAuthority(terminalHandle)
const paneKey = authority?.paneKey ?? runtime.getTerminalPaneKey(terminalHandle)
const processIncarnation =
authority?.processIncarnation ?? runtime.getTerminalProcessIncarnation(terminalHandle)
if (!paneKey || !processIncarnation) {
throw new Error('stable_pane_required')
}
const capability = db.prepareRemoteAttachmentAuthority({
const readiness = prepareFederatedReadinessOutcome({
runtime,
db,
dispatchId: params.dispatchId,
paneKey,
processIncarnation,
worktreeId: worktree.id,
terminalHandle,
setupState: setup.state,
effects,
hostScope: authority?.hostScope ? JSON.stringify(authority.hostScope) : null,
terminalOwnership: params.terminal ? 'external' : 'created'
terminalOwnership: params.terminal ? 'external' : 'created',
wait,
setup,
launch: launch.receipt,
effects
})
if (readiness.state === 'failed') {
failedStage = readiness.stage
throw new Error(readiness.reason)
}
if (readiness.state === 'unknown') {
monitorFederatedSetup({ ...setupStage, runtime })
return readiness.receipt
}
const { capability } = readiness
failedStage = 'dispatch_input'
const prompt = await runtime.sendTerminalAgentPrompt(
terminalHandle,
@@ -125,6 +125,18 @@ describe('tui-idle evidence ranking', () => {
).resolves.toMatchObject({ satisfied: true })
})
it('reports an explicit provider working title as busy instead of unknown', async () => {
const pty = makeTuiIdlePty({ lastAgentStatus: 'working', lastOscTitle: '⠋ Codex' })
const { wait } = createWait({ pty, agent: 'codex' })
const result = wait.wait(HANDLE, { condition: 'tui-idle', timeoutMs: 100 })
await vi.advanceTimersByTimeAsync(100)
await expect(result).resolves.toMatchObject({
satisfied: false,
readiness: { state: 'busy', source: 'title', agent: 'codex' }
})
})
it('accepts a provider-specific ready screen when launch metadata is absent', async () => {
const pty = makeTuiIdlePty({
preview: 'OpenAI Codex\nModel: gpt-5\nDirectory: /tmp/repo'
+13 -1
View File
@@ -1,4 +1,4 @@
import type { AgentStatus } from '../../shared/agent-detection'
import { detectAgentStatusFromTitle, type AgentStatus } from '../../shared/agent-detection'
import { isFreshNonDoneAgentStatus } from '../../shared/agent-status-freshness'
import type { AgentStatusState } from '../../shared/agent-status-types'
import { getAgentReadinessCapability } from '../../shared/agent-readiness-capabilities'
@@ -112,6 +112,18 @@ export function observeTuiIdle(input: TuiIdleSatisfactionInput): TuiIdleObservat
if (hasFreshWorkingFirstPartyStatus(input.firstPartyStatus)) {
return { state: 'busy', source: 'first-party', agent }
}
// A provider-owned working title is a positive busy observation for title-capable launch
// paths. Automation consumers use this edge to distinguish a real working turn from an
// unknown/unsupported pane; it never satisfies `tui-idle` and therefore cannot prove readiness.
if (
agent &&
supportsEvidence(agent, 'title') &&
[input.rendererTitle, input.record.lastOscTitle].some(
(title) => title && detectAgentStatusFromTitle(title) === 'working'
)
) {
return { state: 'busy', source: 'title', agent }
}
if (hasExplicitIdleTitle(input.record, input.rendererTitle) && supportsEvidence(agent, 'title')) {
return { state: 'ready', source: 'title', agent }
}