fix(orchestration): fail worker-start promptly when the agent exits to its shell (#37563)

* fix(orchestration): fail worker-start promptly when the agent exits to its shell

* test(orchestration): keep the exit watch helper's void return
This commit is contained in:
Kelvin Amoaba
2026-11-03 02:15:53 +00:00
committed by GitHub
parent eb55a5a6d3
commit 94f2be1b3d
4 changed files with 473 additions and 18 deletions
@@ -15,6 +15,7 @@ import { TUI_AGENT_CONFIG } from '../../shared/tui-agent-config'
import { draftPasteReadySignalHasMarker } from '../../shared/draft-paste-ready-scanner'
import { nameOnlyIdleNeedsCorroboration } from './tui-idle-evidence'
import type { OrcaRuntimeService } from './orca-runtime'
import { failOnLaunchedAgentExit, type LaunchedAgentExitRuntime } from './launched-agent-exit-watch'
import { showsHoldAnchor } from './agent-state-rules/agent-state-text-anchors'
import { detectTerminalWaitBlockedReason } from './terminal-wait-detection'
@@ -28,13 +29,27 @@ export type LaunchedAgentReadinessRuntime = Pick<
OrcaRuntimeService,
'waitForTerminal' | 'waitForFreshWorkerComposer'
> &
Partial<Pick<OrcaRuntimeService, 'getPtyLivenessVerdict'>>
Partial<Pick<OrcaRuntimeService, 'getPtyLivenessVerdict'> & LaunchedAgentExitRuntime>
export type TerminalReadinessCreation = Pick<
RuntimeTerminalCreate,
'handle' | 'ptyId' | 'incarnationId' | 'isReattach' | 'agentSessionDisposition'
>
/** The PTY this creation spawned for `handle`; a reattached or adopted pane launched nothing new. */
function freshLaunchPtyId(
handle: string,
creation: TerminalReadinessCreation | undefined
): string | null {
return creation &&
creation.handle === handle &&
!creation.isReattach &&
creation.agentSessionDisposition !== 'adopted' &&
creation.ptyId
? creation.ptyId
: null
}
function readinessFailure(
runtime: LaunchedAgentReadinessRuntime,
handle: string,
@@ -44,25 +59,46 @@ function readinessFailure(
if (readTerminalStartFailure(error)) {
return error
}
const verdict = creation?.ptyId ? runtime.getPtyLivenessVerdict?.(creation.ptyId) : null
const ptyId = freshLaunchPtyId(handle, creation)
const verdict = ptyId ? runtime.getPtyLivenessVerdict?.(ptyId) : null
if (
!creation ||
creation.handle !== handle ||
creation.isReattach ||
creation.agentSessionDisposition === 'adopted' ||
!creation.incarnationId ||
!creation.ptyId ||
!ptyId ||
!creation?.incarnationId ||
verdict?.status !== 'exited' ||
verdict.incarnationId !== creation.incarnationId
) {
return error
}
return classifyTerminalCreationFailure(error, {
id: creation.ptyId,
id: ptyId,
incarnationId: creation.incarnationId
})
}
function observesLaunchedAgentExit(
runtime: LaunchedAgentReadinessRuntime
): runtime is LaunchedAgentReadinessRuntime & LaunchedAgentExitRuntime {
return (
runtime.readLaunchedAgentForeground !== undefined &&
runtime.subscribeToTerminalData !== undefined &&
runtime.readRecentTerminalOutput !== undefined
)
}
function waitWatchingForAgentExit(
runtime: LaunchedAgentReadinessRuntime,
handle: string,
agent: TuiAgent,
creation: TerminalReadinessCreation | undefined,
signal: AbortSignal | undefined,
wait: (signal?: AbortSignal) => Promise<RuntimeTerminalWait>
): Promise<RuntimeTerminalWait> {
const ptyId = freshLaunchPtyId(handle, creation)
return ptyId && observesLaunchedAgentExit(runtime)
? failOnLaunchedAgentExit(runtime, ptyId, agent, signal, wait)
: wait(signal)
}
/** Startup dialogs and hold anchors veto signals; Codex's provisional loading header discards input. */
export function readFreshComposerHold(
waitText: string,
@@ -82,30 +118,36 @@ export async function waitForWorkerStartComposer(
runtime: LaunchedAgentReadinessRuntime,
handle: string,
agent: TuiAgent,
timeoutMs: number
timeoutMs: number,
signal?: AbortSignal
): Promise<RuntimeTerminalWait> {
const abort = signal ? { signal } : {}
if (getLaunchedAgentReadinessLane(agent) === 'composer-marker') {
return runtime.waitForFreshWorkerComposer(handle, agent, timeoutMs)
return signal
? runtime.waitForFreshWorkerComposer(handle, agent, timeoutMs, abort)
: runtime.waitForFreshWorkerComposer(handle, agent, timeoutMs)
}
if (!workerStartReadsComposerMarker(agent)) {
return runtime.waitForTerminal(handle, {
condition: 'tui-idle',
timeoutMs,
launchReadiness: true
launchReadiness: true,
...abort
})
}
const stop = new AbortController()
const lanes = signal ? AbortSignal.any([signal, stop.signal]) : stop.signal
try {
return await firstAnswer(
runtime.waitForTerminal(handle, {
condition: 'tui-idle',
timeoutMs,
launchReadiness: true,
signal: stop.signal
signal: lanes
}),
runtime.waitForFreshWorkerComposer(handle, agent, timeoutMs, {
requireComposerMarker: true,
signal: stop.signal
signal: lanes
})
)
} finally {
@@ -160,9 +202,12 @@ export async function waitForWorkerAgentReady(
creation?: TerminalReadinessCreation
}
): Promise<RuntimeTerminalWait> {
const { agent } = args
try {
const result = await (args.agent && !args.reusesTerminal
? waitForWorkerStartComposer(runtime, handle, args.agent, args.timeoutMs)
const result = await (agent && !args.reusesTerminal
? waitWatchingForAgentExit(runtime, handle, agent, args.creation, undefined, (signal) =>
waitForWorkerStartComposer(runtime, handle, agent, args.timeoutMs, signal)
)
: runtime.waitForTerminal(handle, { condition: 'tui-idle', timeoutMs: args.timeoutMs }))
return args.reusesTerminal
? result
@@ -233,13 +278,15 @@ async function waitForLaunchedAgentComposerImpl(
export async function waitForLaunchedAgentComposer(
...args: Parameters<typeof waitForLaunchedAgentComposerImpl>
): Promise<RuntimeTerminalWait> {
const [runtime, handle, , , , creation] = args
const [runtime, handle, agent, timeoutMs, signal, creation] = args
try {
return requireFreshReadiness(
runtime,
handle,
creation,
await waitForLaunchedAgentComposerImpl(...args)
await waitWatchingForAgentExit(runtime, handle, agent, creation, signal, (watchSignal) =>
waitForLaunchedAgentComposerImpl(runtime, handle, agent, timeoutMs, watchSignal, creation)
)
)
} catch (error) {
throw readinessFailure(runtime, handle, creation, error)
@@ -0,0 +1,296 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { createTranscriptPane, TRANSCRIPT_PANE_PTY_ID } from './agent-transcript-pane-test-harness'
import { waitForWorkerAgentReady } from './launched-agent-composer-readiness'
import type { LaunchedAgentForeground } from './launched-agent-foreground'
import { failOnLaunchedAgentExit, type LaunchedAgentExitRuntime } from './launched-agent-exit-watch'
vi.mock('electron', () => ({
BrowserWindow: { fromId: vi.fn(() => null) },
webContents: { fromId: vi.fn(() => null) },
ipcMain: { on: vi.fn(), removeListener: vi.fn() },
app: { getPath: vi.fn(() => '/tmp') }
}))
// Raw bytes from zsh under Orca's wrapper in a PTY.
const ZSH_PROMPT = '\x1b]133;A\x07\r\x1b[0m\x1b[27m\x1b[24m\x1b[Jplain% \x1b[K\x1b[?2004h'
const ZSH_RUNS_LAUNCH_LINE =
"c\x08cursor-agent '--model' 'not-a-model'\x1b[?2004l\r\r\n\x1b]133;C\x07"
const REJECTION = 'Cannot use this model: not-a-model. Available models: auto, gpt-5.3\r\n'
const ZSH_PROMPT_AFTER_EXIT = [
`\x1b[1m\x1b[7m%\x1b[27m\x1b[1m\x1b[0m${' '.repeat(79)}\r \r`,
'\x1b]133;D;1\x07\x1b]133;A\x07\r\x1b[0m\x1b[27m\x1b[24m\x1b[Jplain% \x1b[K',
'\x1b]777;orca-shell-ready\x07\x1b[?2004h'
].join('')
const NESTED_SHELL_COMMAND = '\x1b]133;C\x07ls\n\x1b]133;D;0\x07'
/** Past every re-read the watch makes after a shell return. */
async function runRereads(): Promise<void> {
await vi.advanceTimersByTimeAsync(10_000)
}
function watchedLaunch(options: {
foreground?: () => LaunchedAgentForeground | Promise<LaunchedAgentForeground>
replay?: string
}) {
let listener: (data: string) => void = () => {}
let recent = options.replay ?? ''
const unsubscribe = vi.fn()
const runtime: LaunchedAgentExitRuntime = {
readLaunchedAgentForeground: vi.fn(async () => (options.foreground ?? (() => 'shell'))()),
subscribeToTerminalData: (_ptyId, next) => {
listener = next
return unsubscribe
},
readRecentTerminalOutput: () => recent
}
let finishWait: (value: string) => void = () => {}
let waitSignal: AbortSignal | undefined
const settled = vi.fn()
const result = failOnLaunchedAgentExit(runtime, 'pty-1', 'cursor', undefined, (signal) => {
waitSignal = signal
return new Promise<string>((resolve) => {
finishWait = resolve
})
})
void result.then(settled, settled)
return {
runtime,
result,
settled,
unsubscribe,
waitSignal: () => waitSignal,
finishWait: (value: string) => finishWait(value),
setRecent: (value: string) => {
recent = value
},
emit: (data: string) => {
recent += data
listener(data)
}
}
}
describe('a launched agent exiting during readiness', () => {
beforeEach(() => {
vi.useFakeTimers()
})
afterEach(() => {
vi.useRealTimers()
})
it('fails with what the launch command printed', async () => {
const launch = watchedLaunch({})
launch.emit(ZSH_PROMPT)
launch.emit(ZSH_RUNS_LAUNCH_LINE)
launch.emit(REJECTION)
launch.emit(ZSH_PROMPT_AFTER_EXIT)
await runRereads()
await expect(launch.result).rejects.toThrow(
`The agent's launch command exited before the agent was ready. Its output:\n${REJECTION.trim()}`
)
expect(launch.waitSignal()?.aborted).toBe(true)
expect(launch.unsubscribe).toHaveBeenCalledOnce()
})
it('finds an exit that happened before the wait began', async () => {
const launch = watchedLaunch({
replay: ZSH_PROMPT + ZSH_RUNS_LAUNCH_LINE + REJECTION + ZSH_PROMPT_AFTER_EXIT
})
await runRereads()
await expect(launch.result).rejects.toThrow(REJECTION.trim())
})
it('says so when the launch command printed nothing', async () => {
const launch = watchedLaunch({ replay: ZSH_RUNS_LAUNCH_LINE + ZSH_PROMPT_AFTER_EXIT })
await runRereads()
await expect(launch.result).rejects.toThrow('without printing anything')
})
it('keeps what a verbose launch printed last after its start left the buffer', async () => {
const launch = watchedLaunch({})
launch.emit(ZSH_RUNS_LAUNCH_LINE)
launch.setRecent(REJECTION)
launch.emit(ZSH_PROMPT_AFTER_EXIT)
await runRereads()
await expect(launch.result).rejects.toThrow(REJECTION.trim())
})
it('reads a redrawn line as its last frame, over CRLF', async () => {
const launch = watchedLaunch({
replay: `${ZSH_RUNS_LAUNCH_LINE}loading 10%\rloading 90%\r\nfatal: bad model\r\n${ZSH_PROMPT_AFTER_EXIT}`
})
await runRereads()
await expect(launch.result).rejects.toThrow(/Its output:\nloading 90%\nfatal: bad model$/)
})
it('keeps a last line with no newline, which zsh’s prompt redraw would erase', async () => {
const launch = watchedLaunch({
replay: `${ZSH_RUNS_LAUNCH_LINE}fatal: bad model${ZSH_PROMPT_AFTER_EXIT}`
})
await runRereads()
await expect(launch.result).rejects.toThrow(/Its output:\nfatal: bad model$/)
})
it('reads again while a prompt’s own command leads the group', async () => {
const reads: LaunchedAgentForeground[] = ['agent', 'unknown', 'shell']
const launch = watchedLaunch({ foreground: () => reads.shift() ?? 'agent' })
launch.emit(ZSH_RUNS_LAUNCH_LINE + REJECTION + ZSH_PROMPT_AFTER_EXIT)
await runRereads()
await expect(launch.result).rejects.toThrow(REJECTION.trim())
})
it('stops reading once the shell’s return never proves out', async () => {
const launch = watchedLaunch({ foreground: () => 'agent' })
launch.emit(ZSH_RUNS_LAUNCH_LINE + ZSH_PROMPT_AFTER_EXIT)
await runRereads()
const reads = vi.mocked(launch.runtime.readLaunchedAgentForeground).mock.calls.length
await runRereads()
expect(vi.mocked(launch.runtime.readLaunchedAgentForeground)).toHaveBeenCalledTimes(reads)
expect(launch.settled).not.toHaveBeenCalled()
})
it('leaves a slow start alone while the shell runs the launch line', async () => {
const launch = watchedLaunch({})
// An rc file toggling bracketed paste looks like a command ending without integration marks.
launch.emit('\x1b[?2004l\x1b[?2004h')
launch.emit(ZSH_PROMPT)
launch.emit(ZSH_RUNS_LAUNCH_LINE)
await runRereads()
expect(launch.runtime.readLaunchedAgentForeground).not.toHaveBeenCalled()
launch.finishWait('ready')
await expect(launch.result).resolves.toBe('ready')
})
it('ignores a finish no command came before, as prompt hooks can report', async () => {
const launch = watchedLaunch({})
launch.emit('\x1b]133;D;0\x07')
await runRereads()
expect(launch.runtime.readLaunchedAgentForeground).not.toHaveBeenCalled()
expect(launch.settled).not.toHaveBeenCalled()
})
it('outlasts a shell nested in the agent, then still sees the agent exit', async () => {
let foreground: LaunchedAgentForeground = 'agent'
const launch = watchedLaunch({ foreground: () => foreground })
launch.emit(ZSH_RUNS_LAUNCH_LINE)
launch.emit(NESTED_SHELL_COMMAND)
await runRereads()
expect(launch.settled).not.toHaveBeenCalled()
foreground = 'shell'
launch.emit(REJECTION + ZSH_PROMPT_AFTER_EXIT)
await runRereads()
await expect(launch.result).rejects.toThrow(REJECTION.trim())
})
it('keeps waiting when no foreground read can tell', async () => {
const launch = watchedLaunch({ foreground: () => 'unknown' })
launch.emit(ZSH_RUNS_LAUNCH_LINE + ZSH_PROMPT_AFTER_EXIT)
await runRereads()
expect(launch.settled).not.toHaveBeenCalled()
launch.finishWait('ready')
await expect(launch.result).resolves.toBe('ready')
})
it('discards a read that a later command outran', async () => {
let answer: (value: LaunchedAgentForeground) => void = () => {}
const launch = watchedLaunch({
foreground: () =>
new Promise((resolve) => {
answer = resolve
})
})
launch.emit(ZSH_RUNS_LAUNCH_LINE + ZSH_PROMPT_AFTER_EXIT)
await runRereads()
launch.emit(ZSH_RUNS_LAUNCH_LINE)
answer('shell')
await runRereads()
expect(launch.settled).not.toHaveBeenCalled()
})
it('keeps a ready answer when the exit verdict lands after it', async () => {
let answer: (value: LaunchedAgentForeground) => void = () => {}
const launch = watchedLaunch({
foreground: () =>
new Promise((resolve) => {
answer = resolve
})
})
launch.emit(ZSH_RUNS_LAUNCH_LINE + ZSH_PROMPT_AFTER_EXIT)
await runRereads()
launch.finishWait('ready')
await runRereads()
answer('shell')
await expect(launch.result).resolves.toBe('ready')
})
})
describe('worker readiness on a real runtime pane', () => {
afterEach(() => {
vi.useRealTimers()
vi.restoreAllMocks()
})
// cursor waits on tui-idle, opencode on its composer marker, claude on the first of both.
it.each(['cursor', 'opencode', 'claude'] as const)(
'%s: a rejected launch fails on the shell’s report, not at the deadline',
async (agent) => {
const { runtime, handle, terminal } = await createTranscriptPane({
paneTitle: 'shell',
foregroundProcess: 'zsh',
launchAgent: agent,
data: ''
})
vi.spyOn(runtime, 'readLaunchedAgentForeground').mockResolvedValue('shell')
vi.useFakeTimers()
const wait = waitForWorkerAgentReady(runtime, handle, {
agent,
reusesTerminal: false,
creation: terminal,
timeoutMs: 60_000
}).catch((error: unknown) => error)
runtime.onPtyData(TRANSCRIPT_PANE_PTY_ID, ZSH_PROMPT + ZSH_RUNS_LAUNCH_LINE, 1)
runtime.onPtyData(TRANSCRIPT_PANE_PTY_ID, REJECTION + ZSH_PROMPT_AFTER_EXIT, 2)
await vi.advanceTimersByTimeAsync(1_000)
expect(await wait).toMatchObject({ message: expect.stringContaining(REJECTION.trim()) })
}
)
it('does not watch a reused terminal, whose shell ran commands before this start', async () => {
const { runtime, handle, terminal } = await createTranscriptPane({
paneTitle: 'shell',
foregroundProcess: 'zsh',
launchAgent: 'cursor',
data: ''
})
const read = vi.spyOn(runtime, 'readLaunchedAgentForeground').mockResolvedValue('shell')
runtime.onPtyData(TRANSCRIPT_PANE_PTY_ID, ZSH_RUNS_LAUNCH_LINE + ZSH_PROMPT_AFTER_EXIT, 1)
vi.useFakeTimers()
const wait = waitForWorkerAgentReady(runtime, handle, {
agent: 'cursor',
reusesTerminal: true,
creation: terminal,
timeoutMs: 50
}).catch((error: unknown) => error)
await vi.advanceTimersByTimeAsync(100)
await wait
expect(read).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,108 @@
import { FOREGROUND_COMMAND_READS } from '../../shared/foreground-command-settle'
import { createOsc133CommandFinishedScanner } from '../../shared/terminal-osc133-command-finished'
import { terminalSafe } from '../../shared/terminal-safe-text'
import type { TuiAgent } from '../../shared/tui-agent'
import type { OrcaRuntimeService } from './orca-runtime'
import { applyTerminalLineControls } from './terminal-tail-line-controls'
const COMMAND_STARTED_MARK = '\x1b]133;C'
const COMMAND_FINISHED_MARK = '\x1b]133;D'
const LAUNCH_OUTPUT_MAX_CHARS = 2_000
// What zsh prints before each prompt so it starts on a clean line: its end-of-line mark, and a redraw that erases
// the last line for a terminal that would keep it. Its mark is reverse video and the redraw ends right at the finish.
// eslint-disable-next-line no-control-regex -- the mark's escapes and the redraw's returns are what identify them.
const ZSH_PROMPT_SP = /\x1b\[7m[%#]\x1b\[27m| *\r *\r$/g
// The finish fires before the prompt draws, whose commands (a git status, starship) lead the group until it has.
const SHELL_RETURN_READ_DELAYS_MS = [
FOREGROUND_COMMAND_READS.settleMs,
...FOREGROUND_COMMAND_READS.retryDelaysMs
]
export type LaunchedAgentExitRuntime = Pick<
OrcaRuntimeService,
'readLaunchedAgentForeground' | 'subscribeToTerminalData' | 'readRecentTerminalOutput'
>
/** What the launch command printed, from the shell's last finished command in the pane's recent bytes. */
function readLaunchCommandOutput(recentOutput: string): string {
const finished = recentOutput.lastIndexOf(COMMAND_FINISHED_MARK)
if (finished === -1) {
return ''
}
// A launch that printed past the buffer's limit has lost its start, not the end that names the error.
const started = Math.max(0, recentOutput.lastIndexOf(COMMAND_STARTED_MARK, finished))
return recentOutput
.slice(started, finished)
.replace(ZSH_PROMPT_SP, '')
.split(/\r*\n/)
.map((line) => terminalSafe(applyTerminalLineControls(line).text).trim())
.filter(Boolean)
.join('\n')
.slice(-LAUNCH_OUTPUT_MAX_CHARS)
}
function launchedAgentExitError(recentOutput: string | undefined): Error {
const output = readLaunchCommandOutput(recentOutput ?? '')
return new Error(
output
? `The agent's launch command exited before the agent was ready. Its output:\n${output}`
: "The agent's launch command exited before the agent was ready, without printing anything."
)
}
/** Fails a fresh launch's readiness wait once its launch command has finished and the shell is back in front.
* Neither proves it alone: a shell nested in the agent leaks OSC 133;D, and the shell also leads until the agent starts.
* Panes without shell integration, or that cannot prove a shell, wait as long as they did without this. */
export function failOnLaunchedAgentExit<T>(
runtime: LaunchedAgentExitRuntime,
ptyId: string,
agent: TuiAgent,
signal: AbortSignal | undefined,
wait: (signal: AbortSignal) => Promise<T>
): Promise<T> {
const stop = new AbortController()
let unsubscribe = (): void => {}
return new Promise<T>((resolve, reject) => {
let commandStarted = false
// Moves on every start and finish: only the latest shell return may decide.
let marks = 0
const checkForExit = async (): Promise<void> => {
const mark = marks
for (const delayMs of SHELL_RETURN_READ_DELAYS_MS) {
await new Promise((next) => setTimeout(next, delayMs))
if (stop.signal.aborted || mark !== marks) {
return
}
const foreground = await runtime.readLaunchedAgentForeground(ptyId, agent)
if (foreground === 'shell' && mark === marks) {
reject(launchedAgentExitError(runtime.readRecentTerminalOutput(ptyId)))
return
}
}
}
const scanner = createOsc133CommandFinishedScanner(
() => {
// Prompt hooks can report a finish before any command began.
if (commandStarted) {
marks += 1
// A failed read proves nothing; the wait goes on.
void checkForExit().catch(() => {})
}
},
() => {
commandStarted = true
marks += 1
}
)
unsubscribe = runtime.subscribeToTerminalData(ptyId, scanner.scan)
wait(signal ? AbortSignal.any([signal, stop.signal]) : stop.signal).then(resolve, reject)
// The launch can have ended before this wait began.
const replay = runtime.readRecentTerminalOutput(ptyId)
if (replay) {
scanner.scan(replay)
}
}).finally(() => {
unsubscribe()
stop.abort()
})
}
@@ -212,4 +212,8 @@ export abstract class OrcaRuntimeWithRecordAgentPromptLifecycleState extends Orc
): () => void {
return this.terminalStreamConsumers.subscribe(ptyId, listener)
}
readRecentTerminalOutput(ptyId: string): string | undefined {
return this.recentPtyOutputById.get(ptyId)?.read()
}
}