Fix Codex stream error status completion

This commit is contained in:
Neil
2026-07-02 22:59:09 -07:00
parent b51dd80252
commit b2aed8a8c4
7 changed files with 404 additions and 6 deletions
@@ -1,4 +1,5 @@
import type { ParsedAgentStatusPayload } from '../../../../shared/agent-status-types'
import type { RecognizedAgentProcess } from '../../../../shared/agent-process-recognition'
import type { GlobalSettings } from '../../../../shared/types'
import type { RuntimeTerminalProcessInspection } from '@/runtime/runtime-terminal-inspection'
@@ -30,6 +31,9 @@ export type AgentCompletionCoordinatorOptions = {
isLive: () => boolean
shouldPollProcessCadence?: () => boolean
shouldSuppressHookCompletion?: (payload: AgentCompletionStatusSnapshot) => boolean
onProcessExitCompletion?: (
agent: RecognizedAgentProcess
) => AgentCompletionStatusSnapshot | null | undefined
}
export type AgentCompletionCoordinator = {
@@ -203,6 +203,47 @@ describe('agent completion coordinator', () => {
expect(dispatchCompletion).toHaveBeenCalledWith('codex')
})
it('passes synthesized status detail when the process-exit backstop completes a row', async () => {
let foregroundProcess: string | null = 'codex'
const dispatchCompletion = vi.fn()
const onProcessExitCompletion = vi.fn(() => ({
state: 'done' as const,
prompt: 'Fix the spinner',
agentType: 'codex' as const,
lastAssistantMessage: 'Codex exited before sending Stop.'
}))
const coordinator = createAgentCompletionCoordinator({
paneKey: 'tab-1:leaf-1',
getPtyId: () => 'pty-1',
getSettings: () => null,
inspectProcess: vi.fn(async () => processResult(foregroundProcess)),
dispatchCompletion,
onProcessExitCompletion,
isLive: () => true,
shouldPollProcessCadence: () => false
})
coordinator.observeTitle('Codex working')
await vi.advanceTimersByTimeAsync(3_000)
foregroundProcess = null
await vi.advanceTimersByTimeAsync(6_000)
expect(onProcessExitCompletion).toHaveBeenCalledWith({
agent: 'codex',
processName: 'codex'
})
expect(dispatchCompletion).toHaveBeenCalledWith('codex', {
source: 'process-exit',
quietedHookDone: false,
agentStatus: {
state: 'done',
prompt: 'Fix the spinner',
agentType: 'codex',
lastAssistantMessage: 'Codex exited before sending Stop.'
}
})
})
it('clears process evidence after agent exit so later non-agent spinner titles do not notify', async () => {
let foregroundProcess: string | null = 'codex'
const dispatchCompletion = vi.fn()
@@ -254,6 +254,7 @@ export function createAgentCompletionCoordinator(
quietedHookDone?: boolean
agentStatus?: AgentCompletionStatusSnapshot
completionIdentity?: LastCompletionIdentity | null
processExitAgent?: RecognizedAgentProcess
} = {}
): void {
if (source !== 'hook' && pendingHookDoneTimer !== null) {
@@ -281,11 +282,16 @@ export function createAgentCompletionCoordinator(
if (optionsOverride.completionIdentity) {
lastCompletionIdentityByPaneKey.set(options.paneKey, optionsOverride.completionIdentity)
}
if (optionsOverride.quietedHookDone === true) {
const agentStatus =
optionsOverride.agentStatus ??
(source === 'process-exit' && optionsOverride.processExitAgent
? (options.onProcessExitCompletion?.(optionsOverride.processExitAgent) ?? undefined)
: undefined)
if (optionsOverride.quietedHookDone === true || agentStatus) {
options.dispatchCompletion(title, {
source,
quietedHookDone: true,
...(optionsOverride.agentStatus ? { agentStatus: optionsOverride.agentStatus } : {})
quietedHookDone: optionsOverride.quietedHookDone === true,
...(agentStatus ? { agentStatus } : {})
})
} else {
options.dispatchCompletion(title)
@@ -418,7 +424,8 @@ export function createAgentCompletionCoordinator(
source: 'process-exit',
identity: `${lastForegroundAgent.agent}:${lastForegroundAgent.processName}`,
agentIdentity: lastForegroundAgent.agent
}
},
processExitAgent: lastForegroundAgent
})
}
processSession += 1
@@ -466,7 +473,8 @@ export function createAgentCompletionCoordinator(
source: 'process-exit',
identity: `${exited.agent}:${exited.processName}`,
agentIdentity: exited.agent
}
},
processExitAgent: exited
})
lastForegroundAgent = null
clearAgentRunEvidence()
@@ -0,0 +1,47 @@
import { describe, expect, it, vi } from 'vitest'
import { createCodexErrorOutputStatusDetector } from './codex-error-output-status'
describe('Codex error output status detector', () => {
it('detects stream-disconnect errors from Codex output', () => {
const onStreamError = vi.fn()
const detector = createCodexErrorOutputStatusDetector({ onStreamError })
const observed = detector.observe(
'■ stream disconnected before completion: error sending request for url (http://openclaw:2455/backend-api/codex/responses)\r\n'
)
expect(observed).toBe(true)
expect(onStreamError).toHaveBeenCalledWith(
'stream disconnected before completion: error sending request for url (http://openclaw:2455/backend-api/codex/responses)'
)
})
it('detects a stream-disconnect error split across chunks', () => {
const onStreamError = vi.fn()
const detector = createCodexErrorOutputStatusDetector({ onStreamError })
expect(detector.observe('■ stream discon')).toBe(false)
expect(
detector.observe(
'nected before completion: error sending request for url (http://openclaw:2455/backend-api/codex/responses)\r\n'
)
).toBe(true)
expect(onStreamError).toHaveBeenCalledWith(
'stream disconnected before completion: error sending request for url (http://openclaw:2455/backend-api/codex/responses)'
)
})
it('strips terminal control sequences before reporting the error message', () => {
const onStreamError = vi.fn()
const detector = createCodexErrorOutputStatusDetector({ onStreamError })
detector.observe(
'\x1b[31mstream disconnected before completion: error sending request for url (http://openclaw:2455/backend-api/codex/responses)\x1b[0m\r\n'
)
expect(onStreamError).toHaveBeenCalledWith(
'stream disconnected before completion: error sending request for url (http://openclaw:2455/backend-api/codex/responses)'
)
})
})
@@ -0,0 +1,138 @@
type CodexErrorOutputStatusDetector = {
observe: (data: string) => boolean
}
const ESC = String.fromCharCode(0x1b)
const BEL = String.fromCharCode(0x07)
const ANSI_ESCAPE_RE = new RegExp(
`${ESC}(?:[@-Z\\\\-_]|\\[[0-?]*[ -/]*[@-~]|\\][^${BEL}]*(?:${BEL}|${ESC}\\\\))`,
'g'
)
const INCOMPLETE_ANSI_ESCAPE_RE = new RegExp(
`${ESC}(?:\\[[0-?]*[ -/]*|\\][^${BEL}${ESC}]*|\\S?)?$`,
'g'
)
const RECENT_RAW_TEXT_LIMIT = 300
const STATUS_SCAN_TEXT_LIMIT = 4096
const CODEX_STREAM_DISCONNECTED_MARKER = 'stream disconnected before completion:'
const CODEX_STREAM_DISCONNECTED_RE = /\bstream disconnected before completion:[^\r\n]*/
function terminalControlMayAffectText(data: string): boolean {
for (let index = 0; index < data.length; index += 1) {
const code = data.charCodeAt(index)
if (
code === 0x0d ||
code === 0x1b ||
(code <= 0x1f && code !== 0x0a) ||
(code >= 0x7f && code <= 0x9f)
) {
return true
}
}
return false
}
function stripTerminalControl(data: string): string {
if (!terminalControlMayAffectText(data)) {
return data
}
const withoutAnsi = data.replace(ANSI_ESCAPE_RE, '').replace(INCOMPLETE_ANSI_ESCAPE_RE, '')
let output = ''
for (let index = 0; index < withoutAnsi.length; index += 1) {
const code = withoutAnsi.charCodeAt(index)
if ((code <= 0x1f && code !== 0x0a && code !== 0x0d) || (code >= 0x7f && code <= 0x9f)) {
continue
}
output += withoutAnsi[index]
}
return output
}
function appendRecentRawText(previousRawText: string, data: string): string {
if (data.length >= RECENT_RAW_TEXT_LIMIT) {
return data.slice(-RECENT_RAW_TEXT_LIMIT)
}
return (previousRawText + data).slice(-RECENT_RAW_TEXT_LIMIT)
}
function buildStatusScanRawText(prefix: string, data: string): string {
const boundedPrefix =
prefix.length > RECENT_RAW_TEXT_LIMIT + 1 ? prefix.slice(-(RECENT_RAW_TEXT_LIMIT + 1)) : prefix
const dataBudget = STATUS_SCAN_TEXT_LIMIT - boundedPrefix.length
if (dataBudget <= 0) {
return boundedPrefix.slice(-STATUS_SCAN_TEXT_LIMIT)
}
if (data.length <= dataBudget) {
return boundedPrefix + data
}
const headBudget = Math.max(0, Math.floor((dataBudget - 1) / 2))
const tailBudget = Math.max(0, dataBudget - headBudget - 1)
const head = headBudget > 0 ? data.slice(0, headBudget) : ''
const tail = tailBudget > 0 ? data.slice(-tailBudget) : ''
// Why: PTY chunks can be large pastes or hidden-output restores; stream-error
// detection only needs chunk-boundary context plus the current edge windows.
return `${boundedPrefix}${head}\n${tail}`
}
function rawTextMayContainCodexStreamError(rawText: string): boolean {
// Why: Codex emits this error in lowercase; avoid allocating a lowercased
// copy for every ordinary PTY chunk on the no-match hot path.
return rawText.includes(CODEX_STREAM_DISCONNECTED_MARKER)
}
function findOverlappingCodexStreamError(
scanText: string,
previousTextLength: number
): RegExpMatchArray | null {
const re = new RegExp(CODEX_STREAM_DISCONNECTED_RE.source, 'g')
for (const match of scanText.matchAll(re)) {
const start = match.index ?? 0
if (start + match[0].length > previousTextLength) {
return match
}
}
return null
}
function extractCodexStreamErrorMessage(scanText: string, match: RegExpMatchArray): string {
const start = match.index ?? 0
const lineEndCandidates = [scanText.indexOf('\r', start), scanText.indexOf('\n', start)].filter(
(index) => index >= 0
)
const lineEnd = lineEndCandidates.length > 0 ? Math.min(...lineEndCandidates) : scanText.length
return scanText.slice(start, lineEnd).replace(/\s+/g, ' ').trim()
}
export function createCodexErrorOutputStatusDetector(args: {
onStreamError: (message: string) => void
}): CodexErrorOutputStatusDetector {
let recentRawText = ''
return {
observe(data: string): boolean {
const previousRawText = recentRawText
recentRawText = appendRecentRawText(previousRawText, data)
const scanRawText = buildStatusScanRawText(previousRawText, data)
if (!rawTextMayContainCodexStreamError(scanRawText)) {
return false
}
const scanText = stripTerminalControl(scanRawText)
const previousTextLength = previousRawText ? stripTerminalControl(previousRawText).length : 0
const match = findOverlappingCodexStreamError(scanText, previousTextLength)
if (!match) {
return false
}
const message = extractCodexStreamErrorMessage(scanText, match)
if (!message) {
return false
}
args.onStreamError(message)
return true
}
}
}
@@ -673,6 +673,7 @@ describe('connectPanePty', () => {
markAgentCompletionPaneUnread: vi.fn()
} as StoreState
;(globalThis as unknown as { window: unknown }).window = {
dispatchEvent: vi.fn(),
api: {
ssh: {
connect: vi.fn().mockResolvedValue({ status: 'connected' }),
@@ -1448,6 +1449,94 @@ describe('connectPanePty', () => {
)
})
it('marks a Codex stream-disconnect output as done instead of leaving the row working', async () => {
const { connectPanePty } = await import('./pty-connection')
const capturedDataCallback: { current: ((data: string) => void) | null } = { current: null }
const transport = createMockTransport()
transport.connect.mockImplementation(async ({ callbacks }: { callbacks: ConnectCallbacks }) => {
capturedDataCallback.current = callbacks.onData ?? null
return 'pty-codex'
})
transportFactoryQueue.push(transport)
const pane = createPane(1)
const manager = createManager(1)
const deps = createDeps()
const paneKey = makePaneKey('tab-1', LEAF_1)
mockStoreState.agentStatusByPaneKey[paneKey] = {
paneKey,
state: 'working',
prompt: 'Fix the spinner',
agentType: 'codex',
updatedAt: Date.now(),
stateStartedAt: Date.now(),
stateHistory: []
}
connectPanePty(pane as never, manager as never, deps as never)
await flushAsyncTicks()
expect(capturedDataCallback.current).not.toBeNull()
capturedDataCallback.current?.(
'■ stream disconnected before completion: error sending request for url (http://openclaw:2455/backend-api/codex/responses)\r\n'
)
expect(mockStoreState.agentStatusByPaneKey[paneKey]).toMatchObject({
state: 'done',
prompt: 'Fix the spinner',
agentType: 'codex',
lastAssistantMessage: expect.stringContaining('stream disconnected before completion')
})
expect(mockStoreState.setAgentStatus).toHaveBeenLastCalledWith(
paneKey,
expect.objectContaining({
state: 'done',
prompt: 'Fix the spinner',
agentType: 'codex',
lastAssistantMessage: expect.stringContaining('stream disconnected before completion')
}),
'Codex ready'
)
})
it('does not complete a non-Codex row from Codex stream-disconnect text', async () => {
const { connectPanePty } = await import('./pty-connection')
const capturedDataCallback: { current: ((data: string) => void) | null } = { current: null }
const transport = createMockTransport()
transport.connect.mockImplementation(async ({ callbacks }: { callbacks: ConnectCallbacks }) => {
capturedDataCallback.current = callbacks.onData ?? null
return 'pty-claude'
})
transportFactoryQueue.push(transport)
const pane = createPane(1)
const paneKey = makePaneKey('tab-1', LEAF_1)
mockStoreState.agentStatusByPaneKey[paneKey] = {
paneKey,
state: 'working',
prompt: 'Keep working',
agentType: 'claude',
updatedAt: Date.now(),
stateStartedAt: Date.now(),
stateHistory: []
}
connectPanePty(pane as never, createManager(1) as never, createDeps() as never)
await flushAsyncTicks()
expect(capturedDataCallback.current).not.toBeNull()
capturedDataCallback.current?.(
'stream disconnected before completion: error sending request for url (http://openclaw:2455/backend-api/codex/responses)\r\n'
)
expect(mockStoreState.setAgentStatus).not.toHaveBeenCalled()
expect(mockStoreState.agentStatusByPaneKey[paneKey]).toMatchObject({
state: 'working',
prompt: 'Keep working',
agentType: 'claude'
})
})
it('delivers terminal-paste startup commands through xterm before submitting', async () => {
const pendingTimeouts: (() => void)[] = []
const originalSetTimeout = globalThis.setTimeout
@@ -118,6 +118,7 @@ import { getTerminalPasteSshRemotePlatform } from './terminal-paste-ssh-platform
import { resolveTerminalPasteRuntime } from './terminal-paste-runtime'
import { isKnownTuiAgentTerminalStartupCommand } from './terminal-startup-command-classifier'
import { createCommandCodeOutputStatusDetector } from './command-code-output-status'
import { createCodexErrorOutputStatusDetector } from './codex-error-output-status'
import type { PtyDataMeta } from './pty-dispatcher'
import { getEagerPtyBufferHandle } from './pty-dispatcher'
import { createTerminalGitHubPRLinkDetector } from '@/lib/terminal-github-pr-link-detector'
@@ -163,7 +164,10 @@ import {
beginAgentStartupDeliveryAttempt,
releaseAgentStartupDeliveryAttempt
} from '@/lib/agent-startup-delayed-delivery'
import { isExpectedAgentProcess } from '../../../../shared/agent-process-recognition'
import {
isExpectedAgentProcess,
type RecognizedAgentProcess
} from '../../../../shared/agent-process-recognition'
const pendingSpawnByPaneKey = new Map<string, Promise<string | null>>()
const SSH_SESSION_EXPIRED_ERROR = 'SSH_SESSION_EXPIRED'
@@ -1473,6 +1477,43 @@ export function connectPanePty(
delete pane.container.dataset.ptyId
}
const completeWorkingStatusFromProcessExit = (
agent: RecognizedAgentProcess
): AgentCompletionStatusSnapshot | null => {
const currentState = useAppStore.getState()
const currentEntry = currentState.agentStatusByPaneKey[cacheKey]
if (!currentEntry || currentEntry.state !== 'working') {
return null
}
const ownerAgent = getAuthoritativePaneAgent() ?? agent.agent
const agentType = resolveCompatibleAgentTypeForOwner(
currentEntry.agentType ?? agent.agent,
ownerAgent
)
const processAgentType = resolveCompatibleAgentTypeForOwner(agent.agent, ownerAgent)
if (!agentType || agentType !== processAgentType) {
return null
}
const statusPayload = {
state: 'done' as const,
prompt: currentEntry.prompt,
agentType,
...(currentEntry.lastAssistantMessage
? { lastAssistantMessage: currentEntry.lastAssistantMessage }
: {})
}
const currentTitle =
currentState.runtimePaneTitlesByTabId?.[deps.tabId]?.[pane.id] ?? currentEntry.terminalTitle
const statusTitle = resolveAgentStatusTerminalTitle(statusPayload, currentTitle)
// Why: the foreground-process backstop can prove the TUI disappeared even
// when the agent missed its final hook; keep the explicit row in sync.
currentState.setAgentStatus(cacheKey, statusPayload, statusTitle)
const storedStatus = useAppStore.getState().agentStatusByPaneKey[cacheKey]
return typeof storedStatus?.stateStartedAt === 'number'
? { ...statusPayload, stateStartedAt: storedStatus.stateStartedAt }
: statusPayload
}
const agentCompletionCoordinator = createAgentCompletionCoordinator({
paneKey: cacheKey,
getPtyId: () => transport.getPtyId(),
@@ -1489,6 +1530,7 @@ export function connectPanePty(
}),
shouldPollProcessCadence: () =>
isAgentTaskCompleteTrackingEnabled() && deps.isVisibleRef.current,
onProcessExitCompletion: completeWorkingStatusFromProcessExit,
isLive: () => {
if (disposed) {
return false
@@ -1772,6 +1814,34 @@ export function connectPanePty(
onWorking: seedCommandCodeOutputWorkingStatus,
onDone: scheduleCommandCodeOutputDoneStatus
})
const completeCodexOutputStreamErrorStatus = (message: string): void => {
const currentState = useAppStore.getState()
const currentEntry = currentState.agentStatusByPaneKey[cacheKey]
if (!currentEntry || currentEntry.state !== 'working') {
return
}
const agentType = resolveCompatibleAgentTypeForOwner(
currentEntry.agentType ?? 'codex',
getAuthoritativePaneAgent()
)
if (agentType !== 'codex') {
return
}
const statusPayload = {
state: 'done' as const,
prompt: currentEntry.prompt,
agentType,
lastAssistantMessage: message
}
const currentTitle = currentState.runtimePaneTitlesByTabId?.[deps.tabId]?.[pane.id]
const statusTitle = resolveAgentStatusTerminalTitle(statusPayload, currentTitle)
// Why: Codex failed turns can finalize the TUI without emitting the managed
// Stop hook Orca normally uses to clear an explicit working row.
currentState.setAgentStatus(cacheKey, statusPayload, statusTitle)
}
const codexErrorOutputStatusDetector = createCodexErrorOutputStatusDetector({
onStreamError: completeCodexOutputStreamErrorStatus
})
const observeTerminalGitHubPRLink = createTerminalGitHubPRLinkDetector()
const reportPanePtyVisibility = (ptyId: string | null | undefined, visible: boolean): void => {
if (!ptyId || isRemoteRuntimePtyId(ptyId)) {
@@ -4406,6 +4476,7 @@ export function connectPanePty(
for (const link of observeTerminalGitHubPRLink(data)) {
useAppStore.getState().observeTerminalGitHubPullRequestLink(deps.worktreeId, link)
}
codexErrorOutputStatusDetector.observe(data)
commandCodeOutputStatusDetector.observe(data)
commandLifecycle.handlePtyData(data)
// Why: split-pane layouts have multiple visible-but-inactive panes whose