fix(agent-prompt): stop reporting delivered prompts as stalled (#16095)

Enter is written before verification runs, so `agent_prompt_stalled` can only
ever mean "turn start not observed" — never "prompt not delivered". Three of the
verifier's blind spots made that misreading routine, and the coordinator then
treated it as non-delivery and pasted the whole preamble a second time into a
worker already running it.

- Accept a hook-reported `working` recorded after the baseline. Hook rows reach
  the runtime through getAgentStatusSnapshot with no window involved, unlike the
  synthetic-title route that feeds workingSequence (suppressed for codex, absent
  for kimi, and gated on window visibility for everyone else).
- Accept pane output after Enter when the agent was already working: a
  `->working` edge is unreachable there, so the old predicate could never be
  satisfied by a follow-up prompt. An idle agent still owes a real turn start,
  so a swallowed Enter stays detectable.
- Give codex/kimi panes a longer effect window; their only turn-start proof is
  an out-of-process hook round-trip, not a TUI repaint.
- Coordinator dispatch no longer fails (and re-dispatches) a task whose prompt
  stalled; the dispatch stays active with its capability intact so the worker's
  own report settles it.
This commit is contained in:
Neil
2026-08-26 00:16:48 -07:00
parent 1c9fb84b77
commit f9f973c044
7 changed files with 415 additions and 16 deletions
@@ -402,15 +402,7 @@ describe('agent prompt submission runtime', () => {
it('does not treat an unchanged newer working status as submission evidence', async () => {
vi.useFakeTimers()
vi.setSystemTime(1_000)
const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => {
if (data === '\r') {
runtime.onPtyData(
'pty-prompt',
'\x1b]9999;{"state":"working","agentType":"aider"}\x07',
Date.now()
)
}
})
const { runtime, handle, writes } = await createPromptRuntime(() => undefined)
runtime.onPtyData('pty-prompt', '\x1b]0;Codex waiting for permission\x07', Date.now())
vi.setSystemTime(2_000)
runtime.onPtyData(
@@ -427,6 +419,77 @@ describe('agent prompt submission runtime', () => {
expect(writes.filter((data) => data === '\r')).toHaveLength(1)
})
// Why (#16095): a still-working agent can never produce a `→working` edge, so the old predicate
// was unsatisfiable for every follow-up prompt; pane output after Enter is the evidence left.
it('accepts pane output after Enter while the agent is already working', async () => {
vi.useFakeTimers()
vi.setSystemTime(1_000)
const { runtime, handle, writes } = await createPromptRuntime((runtime, data) => {
if (data === '\r') {
runtime.onPtyData('pty-prompt', 'queued for the current turn', Date.now())
}
})
runtime.onPtyData(
'pty-prompt',
'\x1b]9999;{"state":"working","agentType":"aider"}\x07',
Date.now()
)
const submission = runtime.sendTerminalAgentPrompt(handle, 'review this')
await vi.runAllTimersAsync()
await expect(submission).resolves.toMatchObject({ accepted: true })
expect(writes.filter((data) => data === '\r')).toHaveLength(1)
})
it('accepts a hook working status with no window and no title coverage', async () => {
vi.useFakeTimers()
vi.setSystemTime(1_000)
let handle = ''
let hookState: 'done' | 'working' = 'done'
const writes: string[] = []
// Why: hook rows reach the runtime through this provider, which has no window and no OSC title —
// the same path a headless `orca serve` host and a minimized desktop window take.
const runtime = new OrcaRuntimeService(makeStore() as never, undefined, {
getAgentStatusSnapshot: () => [
{
paneKey: 'prompt-pane',
terminalHandle: handle,
state: hookState,
prompt: '',
agentType: 'kimi',
connectionId: null,
receivedAt: Date.now(),
stateStartedAt: Date.now()
}
]
})
runtime.setPtyController({
spawn: vi.fn().mockResolvedValue({ id: 'pty-prompt' }),
write: (_ptyId, data) => {
writes.push(data)
if (data === '\r') {
vi.setSystemTime(3_000)
hookState = 'working'
}
return true
},
kill: () => true,
getForegroundProcess: async () => null
})
handle = (
await runtime.createTerminal(`path:${AGENT_PROMPT_TEST_WORKTREE_PATH}`, {
launchAgent: 'kimi'
})
).handle
const submission = runtime.sendTerminalAgentPrompt(handle, 'review this')
await vi.runAllTimersAsync()
await expect(submission).resolves.toMatchObject({ accepted: true })
expect(writes.filter((data) => data === '\r')).toHaveLength(1)
})
it('does not write Enter after the PTY generation changes during settlement', async () => {
vi.useFakeTimers()
const { runtime, handle, writes } = await createPromptRuntime(() => undefined)
@@ -1,7 +1,10 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
AGENT_PROMPT_EFFECT_TIMEOUT_MS,
AGENT_PROMPT_HOOK_EFFECT_TIMEOUT_MS,
type AgentPromptActivity,
isAgentPromptStalledError,
resolveAgentPromptEffectTimeoutMs,
verifyAgentPromptSubmission
} from './agent-prompt-submission-verification'
@@ -10,6 +13,8 @@ function activity(overrides: Partial<AgentPromptActivity> = {}): AgentPromptActi
generation: 1,
permissionSequence: 2,
workingSequence: 4,
explicitWorkingAt: null,
outputSequence: 7,
status: 'idle',
...overrides
}
@@ -127,6 +132,94 @@ describe('agent prompt submission verification', () => {
await rejected
})
it('accepts a hook working status recorded after the baseline', async () => {
vi.useFakeTimers()
let current = activity()
const verification = verifyAgentPromptSubmission({
baseline: current,
readActivity: () => current
})
// No workingSequence edge: the window-gated synthetic title never ran (hidden window/headless).
current = activity({ explicitWorkingAt: 2_000, status: 'working' })
await vi.advanceTimersByTimeAsync(50)
await expect(verification).resolves.toBeUndefined()
})
it('does not accept a hook working status that predates the baseline', async () => {
vi.useFakeTimers()
const current = activity({ explicitWorkingAt: 2_000, status: 'working' })
const verification = verifyAgentPromptSubmission({
baseline: current,
readActivity: () => current
})
const rejected = expect(verification).rejects.toThrow('agent_prompt_stalled')
await vi.advanceTimersByTimeAsync(AGENT_PROMPT_EFFECT_TIMEOUT_MS)
await rejected
})
it('accepts pane output after Enter when the agent was already working', async () => {
vi.useFakeTimers()
let current = activity({ status: 'working' })
const verification = verifyAgentPromptSubmission({
baseline: current,
readActivity: () => current
})
current = activity({ status: 'working', outputSequence: 8 })
await vi.advanceTimersByTimeAsync(50)
await expect(verification).resolves.toBeUndefined()
})
it('does not accept pane output when the agent was idle at submit', async () => {
vi.useFakeTimers()
let current = activity()
const verification = verifyAgentPromptSubmission({
baseline: current,
readActivity: () => current
})
const rejected = expect(verification).rejects.toThrow('agent_prompt_stalled')
current = activity({ outputSequence: 9 })
await vi.advanceTimersByTimeAsync(AGENT_PROMPT_EFFECT_TIMEOUT_MS)
await rejected
})
it('holds the longer hook window open past the default timeout', async () => {
vi.useFakeTimers()
let current = activity()
const verification = verifyAgentPromptSubmission({
baseline: current,
readActivity: () => current,
timeoutMs: AGENT_PROMPT_HOOK_EFFECT_TIMEOUT_MS
})
await vi.advanceTimersByTimeAsync(AGENT_PROMPT_EFFECT_TIMEOUT_MS + 1_000)
current = activity({ explicitWorkingAt: 9_000, status: 'working' })
await vi.advanceTimersByTimeAsync(50)
await expect(verification).resolves.toBeUndefined()
})
it('gives hook-observed agents the longer effect window', () => {
expect(resolveAgentPromptEffectTimeoutMs('codex')).toBe(AGENT_PROMPT_HOOK_EFFECT_TIMEOUT_MS)
expect(resolveAgentPromptEffectTimeoutMs('kimi')).toBe(AGENT_PROMPT_HOOK_EFFECT_TIMEOUT_MS)
expect(resolveAgentPromptEffectTimeoutMs('claude')).toBe(AGENT_PROMPT_EFFECT_TIMEOUT_MS)
expect(resolveAgentPromptEffectTimeoutMs(null)).toBe(AGENT_PROMPT_EFFECT_TIMEOUT_MS)
})
it('recognizes a stalled verdict from a message or a relayed error code', () => {
expect(isAgentPromptStalledError(new Error('agent_prompt_stalled'))).toBe(true)
expect(isAgentPromptStalledError({ code: 'agent_prompt_stalled' })).toBe(true)
expect(isAgentPromptStalledError(new Error('terminal_not_writable'))).toBe(false)
expect(isAgentPromptStalledError(null)).toBe(false)
})
it('rejects a replaced terminal generation', async () => {
const baseline = activity()
@@ -1,31 +1,66 @@
import type { TuiAgent } from '../../shared/tui-agent'
export const AGENT_PROMPT_EFFECT_TIMEOUT_MS = 5_000
// Why: these panes prove a turn start only through the out-of-process hook — kimi has no synthetic
// title profile and codex suppresses the hook-driven working frame (synthesizeWorkingTitle: false),
// so the first proof lags Enter by agent startup, not by one TUI repaint. Capped so the worst case
// (8s render gate + this wait) still fits a paired client's 30s request budget.
export const AGENT_PROMPT_HOOK_EFFECT_TIMEOUT_MS = 15_000
const AGENT_PROMPT_EFFECT_POLL_MS = 50
const HOOK_OBSERVED_TURN_START_AGENTS = new Set<TuiAgent>(['codex', 'kimi'])
/** The prompt bytes are written before verification, so this only ever means "not observed". */
export const AGENT_PROMPT_STALLED_ERROR = 'agent_prompt_stalled'
export type AgentPromptActivity = Readonly<{
generation: number
permissionSequence: number
workingSequence: number
/** Hook-reported `working` timestamp; reaches the runtime with no window and no title coverage. */
explicitWorkingAt: number | null
/** PTY bytes seen on this pane; delivery evidence when a turn-start edge cannot be observed. */
outputSequence: number
status: 'working' | 'permission' | 'idle' | null
}>
type AgentPromptVerificationOptions = {
baseline: AgentPromptActivity
readActivity: () => AgentPromptActivity
timeoutMs?: number
signal?: AbortSignal
}
export function resolveAgentPromptEffectTimeoutMs(agent: TuiAgent | null | undefined): number {
return agent && HOOK_OBSERVED_TURN_START_AGENTS.has(agent)
? AGENT_PROMPT_HOOK_EFFECT_TIMEOUT_MS
: AGENT_PROMPT_EFFECT_TIMEOUT_MS
}
export function isAgentPromptStalledError(error: unknown): boolean {
if (error instanceof Error && error.message === AGENT_PROMPT_STALLED_ERROR) {
return true
}
// Why: a relayed submission surfaces the same verdict as an RPC error code, not a message.
return (
typeof error === 'object' &&
error !== null &&
(error as { code?: unknown }).code === AGENT_PROMPT_STALLED_ERROR
)
}
export async function verifyAgentPromptSubmission(
options: AgentPromptVerificationOptions
): Promise<void> {
throwIfAgentPromptAborted(options.signal)
assertPromptNotBlocked(options.baseline, options.baseline)
const deadline = Date.now() + AGENT_PROMPT_EFFECT_TIMEOUT_MS
const deadline = Date.now() + (options.timeoutMs ?? AGENT_PROMPT_EFFECT_TIMEOUT_MS)
while (Date.now() < deadline) {
const current = options.readActivity()
assertSamePromptGeneration(options.baseline, current)
assertPromptNotBlocked(options.baseline, current)
if (agentPromptLifecycleChanged(options.baseline, current)) {
if (agentPromptEffectObserved(options.baseline, current)) {
return
}
await waitForAgentPromptPoll(options.signal)
@@ -34,17 +69,43 @@ export async function verifyAgentPromptSubmission(
const current = options.readActivity()
assertSamePromptGeneration(options.baseline, current)
assertPromptNotBlocked(options.baseline, current)
if (agentPromptLifecycleChanged(options.baseline, current)) {
if (agentPromptEffectObserved(options.baseline, current)) {
return
}
throw new Error('agent_prompt_stalled')
throw new Error(AGENT_PROMPT_STALLED_ERROR)
}
function agentPromptLifecycleChanged(
function agentPromptEffectObserved(
baseline: AgentPromptActivity,
current: AgentPromptActivity
): boolean {
return current.workingSequence > baseline.workingSequence
return (
current.workingSequence > baseline.workingSequence ||
observedHookWorkingAfterBaseline(baseline, current) ||
observedDeliveryEvidence(baseline, current)
)
}
// Why: hook status reaches the runtime directly, so it survives a hidden window and headless serve —
// the synthetic-title route that feeds workingSequence does not (#16095).
function observedHookWorkingAfterBaseline(
baseline: AgentPromptActivity,
current: AgentPromptActivity
): boolean {
return (
current.explicitWorkingAt !== null &&
current.explicitWorkingAt > (baseline.explicitWorkingAt ?? 0)
)
}
// Why: a `→working` edge is unreachable for an agent that is already working, so the honest proof
// that the prompt landed is the pane emitting bytes after Enter. An idle agent still owes a real
// turn start, which keeps a swallowed Enter detectable.
function observedDeliveryEvidence(
baseline: AgentPromptActivity,
current: AgentPromptActivity
): boolean {
return baseline.status === 'working' && current.outputSequence > baseline.outputSequence
}
function assertSamePromptGeneration(
+11
View File
@@ -104,6 +104,7 @@ import {
} from '../../shared/agent-prompt-injection'
import {
type AgentPromptActivity,
resolveAgentPromptEffectTimeoutMs,
verifyAgentPromptSubmission
} from './agent-prompt-submission-verification'
import {
@@ -19836,6 +19837,7 @@ export class OrcaRuntimeService {
await verifyAgentPromptSubmission({
baseline,
readActivity: () => this.getAgentPromptActivity(handle, ptyId),
timeoutMs: resolveAgentPromptEffectTimeoutMs(this.getPtyAgent(ptyId)),
signal: options.signal
})
return 1
@@ -19892,10 +19894,19 @@ export class OrcaRuntimeService {
generation: this.getPtyLifecycleGeneration(ptyId),
permissionSequence: this.agentPromptPermissionSequenceByPtyId.get(ptyId) ?? 0,
workingSequence: lifecycle?.workingSequence ?? 0,
// Why: hook status is the only turn-start signal agents without title coverage have, and it
// reaches here without the window-gated synthetic title frame (#16095).
explicitWorkingAt: explicit?.status === 'working' ? explicit.updatedAt : null,
outputSequence: this.getPtyOutputSequence(ptyId),
status
}
}
private getPtyAgent(ptyId: string): TuiAgent | null {
const pty = this.ptysById.get(ptyId)
return pty?.launchAgent ?? pty?.foregroundAgent ?? null
}
private assertAgentPromptPermissionSafe(
baseline: AgentPromptActivity,
current: AgentPromptActivity
@@ -0,0 +1,125 @@
import { afterEach, describe, expect, it } from 'vitest'
import { OrchestrationDb } from './db'
import type { CoordinatorRuntime } from './coordinator-runtime-contract'
import { dispatchTaskToWorker } from './coordinator-task-dispatch'
const WORKER_PANE_KEY = 'tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'
let db: OrchestrationDb
function createRuntime(promptError: Error | null): CoordinatorRuntime & { prompts: string[] } {
const prompts: string[] = []
return {
prompts,
async sendTerminalAgentPrompt(_handle: string, prompt: string) {
prompts.push(prompt)
if (promptError) {
throw promptError
}
return { accepted: true }
},
async listTerminals() {
return { terminals: [] }
},
async createTerminal() {
return { handle: 'term_a', worktreeId: 'wt1' }
},
async waitForTerminal(handle: string) {
return { handle, condition: 'exit' }
},
async probeWorktreeDrift() {
return null
},
getTerminalPaneKey() {
return WORKER_PANE_KEY
},
getOrchestrationDispatchAuthority() {
return {
paneKey: WORKER_PANE_KEY,
processIncarnation: 'incarnation-1',
launchTokenHash: null
}
}
}
}
async function dispatch(
runtime: CoordinatorRuntime,
taskId: string,
logs: string[]
): Promise<string> {
return dispatchTaskToWorker({
db,
runtime,
task: db.getTask(taskId)!,
targetHandle: 'term_a',
baseDrift: null,
coordinatorHandle: 'coord',
worktree: undefined,
onLog: (message) => logs.push(message),
onCircuitBroken: () => undefined
})
}
describe('coordinator dispatch with an unobserved prompt', () => {
afterEach(() => db?.close())
it('never re-pastes a preamble whose turn start was not observed', async () => {
db = new OrchestrationDb(':memory:')
const task = db.createTask({ spec: 'do the work' })
const runtime = createRuntime(new Error('agent_prompt_stalled'))
const logs: string[] = []
const result = await dispatch(runtime, task.id, logs)
expect(result).toBe('dispatched-unobserved')
expect(runtime.prompts).toHaveLength(1)
// The task stays dispatched, so the next coordinator tick cannot pick it up again.
expect(db.getTask(task.id)?.status).toBe('dispatched')
const ctx = db.getDispatchContext(task.id)
expect(ctx).toMatchObject({
status: 'dispatched',
failure_count: 0,
capability_revoked_at: null
})
expect(db.listTasks({ status: 'ready' })).toEqual([])
expect(logs.join('\n')).toContain('turn start was not observed')
})
it('lets a late worker report settle a dispatch whose prompt was unobserved', async () => {
db = new OrchestrationDb(':memory:')
const task = db.createTask({ spec: 'do the work' })
await dispatch(createRuntime(new Error('agent_prompt_stalled')), task.id, [])
const dispatchId = db.getDispatchContext(task.id)!.id
const minted = db.mintDispatchCapability({
dispatchId,
paneKey: WORKER_PANE_KEY,
processIncarnation: 'incarnation-1'
})
expect(
db.verifyDispatchCapability({
dispatchId,
capability: minted,
paneKey: WORKER_PANE_KEY,
processIncarnation: 'incarnation-1'
})
).toEqual({ valid: true })
db.completeDispatch(dispatchId)
expect(db.getDispatchContextById(dispatchId)?.status).toBe('completed')
})
it('still fails the dispatch when the prompt was never delivered', async () => {
db = new OrchestrationDb(':memory:')
const task = db.createTask({ spec: 'do the work' })
const runtime = createRuntime(new Error('terminal_not_writable'))
await expect(dispatch(runtime, task.id, [])).rejects.toThrow('terminal_not_writable')
expect(db.getTask(task.id)?.status).toBe('ready')
expect(db.getDispatchContext(task.id)).toMatchObject({
status: 'failed',
failure_count: 1,
last_failure: 'terminal_not_writable'
})
})
})
@@ -7,8 +7,10 @@ import {
DISPATCH_STALE_THRESHOLD,
parseAllowStaleBaseFromSpec
} from './coordinator-stale-base-flag'
import { isAgentPromptStalledError } from '../agent-prompt-submission-verification'
export type TaskDispatchResult = 'dispatched' | 'stale-base-refused'
/** `dispatched-unobserved`: the preamble landed but the worker's turn start was never observed. */
export type TaskDispatchResult = 'dispatched' | 'dispatched-unobserved' | 'stale-base-refused'
// Why: 10 min = documented heartbeat cadence (5 min) × 2, so one missed heartbeat is the earliest a dispatch can look stale.
const HUNG_THRESHOLD_MS = 10 * 60 * 1000
@@ -130,6 +132,17 @@ export async function dispatchTaskToWorker(params: {
try {
await runtime.sendTerminalAgentPrompt(targetHandle, preamble + gateContext)
} catch (err) {
// Why (#16095): Enter is written before submission is verified, so a stall is only ever an
// unobserved turn start — never proof the preamble is missing. Failing here would reset the
// task to 'ready' and paste the whole preamble a second time into a worker already running it,
// and would revoke the capability its worker_done needs.
if (isAgentPromptStalledError(err)) {
onLog(
`Dispatched task ${task.id} to ${targetHandle}; turn start was not observed. ` +
`The preamble is already in the pane, so the dispatch stays active instead of being resent.`
)
return 'dispatched-unobserved'
}
const updated = db.failDispatch(dispatch.id, err instanceof Error ? err.message : String(err))
if (updated?.status === 'circuit_broken') {
params.onCircuitBroken(task.id)
@@ -0,0 +1,33 @@
import { describe, expect, it } from 'vitest'
import { isUnknownWorkerStartOutcome } from './orchestration-worker-topology'
describe('worker start outcome classification', () => {
it('treats an explicit operation_unknown code as unknown at any stage', () => {
const error = Object.assign(new Error('relay dropped'), { code: 'operation_unknown' })
expect(isUnknownWorkerStartOutcome(error, 'dispatch_input')).toBe(true)
expect(isUnknownWorkerStartOutcome(error, 'worktree_create')).toBe(true)
})
it('treats a lost connection during worktree create as unknown', () => {
expect(isUnknownWorkerStartOutcome(new Error('connection reset'), 'worktree_create')).toBe(true)
expect(isUnknownWorkerStartOutcome(new Error('request timed out'), 'worktree_create')).toBe(
true
)
})
it('keeps a definite failure definite', () => {
expect(isUnknownWorkerStartOutcome(new Error('connection reset'), 'dispatch_input')).toBe(false)
expect(isUnknownWorkerStartOutcome(new Error('worktree exists'), 'worktree_create')).toBe(false)
})
// Characterization: a stalled prompt means the Enter effect was not observed, never that the
// preamble is missing — but worker-start still settles it as a definite failure (and revokes the
// dispatch capability). Pinned by orchestration-worker-start-prompt-contract.test.ts; changing it
// needs late-report acceptance for a revoked capability, which this classifier cannot express.
it('does not class a stalled dispatch prompt as unknown', () => {
expect(isUnknownWorkerStartOutcome(new Error('agent_prompt_stalled'), 'dispatch_input')).toBe(
false
)
})
})