Show Codex model capacity failures as red agent status (#28231)

* Show Codex model capacity failures in agent status

* Reduce Codex capacity detection work on terminal output

* Preserve Codex capacity notices across partial terminal frames
This commit is contained in:
Neil
2026-10-13 01:58:00 -07:00
committed by GitHub
parent 4559c250b3
commit 9e52645e6c
15 changed files with 805 additions and 10 deletions
@@ -0,0 +1,137 @@
// Run: ORCA_BACKGROUND_LAUNCH=1 node --import tsx config/scripts/codex-capacity-status-benchmark.ts
import { readFileSync } from 'node:fs'
import { performance } from 'node:perf_hooks'
import { CodexCapacityNoticeTracker } from '../../src/shared/codex-capacity-notice'
import { HeadlessEmulator } from '../../src/main/daemon/headless-emulator'
import { observeCodexCapacityTerminalStatus } from '../../src/main/runtime/codex-capacity-terminal-status'
const fixture = readFileSync(
new URL(
'../../src/main/runtime/__fixtures__/codex-model-capacity-fullscreen.txt',
import.meta.url
),
'utf8'
)
const notice = fixture.indexOf('■ Selected model')
if (notice === -1) {
throw new Error('Capacity notice missing from benchmark fixture')
}
const redraw = fixture.slice(0, fixture.lastIndexOf('\x1b[?2026h', notice))
const scenarios = [
{ name: 'plain-4K', data: 'x'.repeat(4096), iterations: 30_000 },
{ name: 'captured-TUI', data: redraw, iterations: 30_000 },
{
name: 'warning-footer',
data: '\x1b[?2026h\x1b[21;1H⚠ 1 warning · f2 to view\x1b[?2026l',
iterations: 30_000
},
{ name: 'tiny-typing', data: 'a', iterations: 30_000 },
{ name: 'marker-flood', data: '■'.repeat(4096), iterations: 1000 },
{
name: 'large-after-warning',
data: `⚠ unrelated\n${'x'.repeat(1024 * 1024)}`,
iterations: 128
}
]
function measure(iterations: number, observe: () => void): number {
const start = performance.now()
for (let index = 0; index < iterations; index++) {
observe()
}
return performance.now() - start
}
for (const scenario of scenarios) {
const run = () => {
const tracker = new CodexCapacityNoticeTracker()
let screenReads = 0
const readScreen = () => {
screenReads++
return []
}
const elapsedMs = measure(scenario.iterations, () => tracker.observe(scenario.data, readScreen))
if (screenReads !== 0) {
throw new Error(`${scenario.name} read the screen ${screenReads} times`)
}
return elapsedMs
}
run()
const samples = Array.from({ length: 5 }, run).sort((left, right) => left - right)
const elapsedMs = samples[2]
textResult(scenario.name, elapsedMs, scenario.iterations, scenario.data.length)
}
for (const launchAgent of [undefined, 'codex'] as const) {
const emulator = new HeadlessEmulator({ cols: 80, rows: 24 })
const data = 'x'.repeat(4096)
const iterations = 30_000
let statusReads = 0
const readStatus = () => {
statusReads++
return undefined
}
try {
const observe = () =>
observeCodexCapacityTerminalStatus(
emulator,
data,
'node /some/wrapper.cjs',
launchAgent,
readStatus,
1
)
measure(iterations, observe)
const samples = Array.from({ length: 5 }, () => measure(iterations, observe)).sort(
(left, right) => left - right
)
if (statusReads !== 0) {
throw new Error(`Ordinary output read status ${statusReads} times`)
}
textResult(launchAgent ?? 'unrelated-terminal', samples[2], iterations, data.length)
} finally {
emulator.dispose()
}
}
async function measureScreenRead(): Promise<void> {
const emulator = new HeadlessEmulator({ cols: 40, rows: 24 })
try {
await emulator.write(fixture)
for (const name of ['screen-only', 'capacity-parse-and-screen']) {
const observe = () => {
if (name === 'screen-only') {
emulator.getVisibleLines()
} else if (
!new CodexCapacityNoticeTracker().observe(fixture, () => emulator.getVisibleLines())
) {
throw new Error('Captured capacity notice was not detected')
}
}
measure(1000, observe)
const samples = Array.from({ length: 5 }, () => measure(1000, observe)).sort(
(left, right) => left - right
)
textResult(name, samples[2], 1000, fixture.length)
}
} finally {
emulator.dispose()
}
}
void measureScreenRead().catch((error: unknown) => {
console.error(error)
process.exitCode = 1
})
function textResult(name: string, elapsedMs: number, iterations: number, chars: number): void {
console.log(
JSON.stringify({
name,
chars,
iterations,
microsecondsPerChunk: Number(((elapsedMs * 1000) / iterations).toFixed(3)),
millisecondsPerMiB: Number((elapsedMs / ((chars * iterations) / 1048576)).toFixed(3))
})
)
}
@@ -0,0 +1,93 @@
import { afterEach, describe, expect, it } from 'vitest'
import { AgentHookServer, _internals } from './server'
import { buildBody, PANE, postHookEvent } from './server.test-fixtures'
import { CODEX_CAPACITY_MESSAGE } from '../../shared/codex-capacity-notice'
afterEach(() => _internals.resetCachesForTests())
describe('Codex capacity verdict in the shared status store', () => {
it('keeps a local failure through Stop and child activity, then clears it on retry', async () => {
const server = new AgentHookServer()
await server.start({ env: 'production' })
const post = (payload: Record<string, unknown>) =>
postHookEvent(server, buildBody({ session_id: 'session-1', ...payload }), '/hook/codex')
try {
expect(
(await post({ hook_event_name: 'UserPromptSubmit', prompt: 'Say hi', model: 'gpt-5' })).ok
).toBe(true)
const previous = server.getStatusSnapshotForPane(PANE)[0]
server.ingestTerminalStatus({
paneKey: PANE,
tabId: 'tab-1',
worktreeId: 'wt-1',
payload: {
state: 'done',
agentType: 'codex',
prompt: previous?.prompt ?? '',
model: previous?.model,
lastAssistantMessage: CODEX_CAPACITY_MESSAGE,
mainAgent: { state: 'done', outcome: 'failure', stateStartedAt: Date.now() }
}
})
expect(server.getStatusSnapshotForPane(PANE)[0]).toMatchObject({
prompt: 'Say hi',
model: 'gpt-5',
lastAssistantMessage: CODEX_CAPACITY_MESSAGE,
providerSession: { id: 'session-1' },
mainAgent: { state: 'done', outcome: 'failure' }
})
await post({ hook_event_name: 'Stop' })
await post({ hook_event_name: 'SubagentStart', agent_id: 'child-1' })
expect(server.getStatusSnapshotForPane(PANE)[0]).toMatchObject({
state: 'working',
mainAgent: { state: 'done', outcome: 'failure' }
})
await post({ hook_event_name: 'UserPromptSubmit', prompt: 'Say hi' })
expect(server.getStatusSnapshotForPane(PANE)[0]?.mainAgent).toMatchObject({
state: 'working'
})
expect(server.getStatusSnapshotForPane(PANE)[0]?.mainAgent?.outcome).toBeUndefined()
} finally {
server.stop()
}
})
it('accepts host capacity evidence after an earlier done and preserves it across relay restatements', () => {
const server = new AgentHookServer()
server.ingestRemote(
{
paneKey: PANE,
payload: {
state: 'done',
agentType: 'codex',
prompt: 'Say hi',
mainAgent: { state: 'done', stateStartedAt: 1 }
}
},
'ssh-host'
)
server.ingestTerminalStatus({
paneKey: PANE,
connectionId: 'ssh-host',
payload: {
state: 'done',
agentType: 'codex',
prompt: 'Say hi',
mainAgent: { state: 'done', outcome: 'failure', stateStartedAt: 2 }
}
})
server.ingestRemote(
{
paneKey: PANE,
hookEventName: 'Stop',
payload: {
state: 'done',
agentType: 'codex',
prompt: 'Say hi'
}
},
'ssh-host'
)
expect(server.getStatusSnapshotForPane(PANE)[0]?.mainAgent?.outcome).toBe('failure')
})
})
@@ -159,11 +159,12 @@ export abstract class AgentHookServerIngestTerminal extends AgentHookServerInges
// Why: OSC carries no main agent fact. While it repaints the state the hook row already holds, the
// main agent behind that state is unchanged too; a different state is a turn edge OSC cannot date.
const preservedMainAgent =
previous?.payload.mainAgent &&
event.payload.mainAgent ??
(previous?.payload.mainAgent &&
previous.payload.state === event.payload.state &&
(claimedAgentType === undefined || claimedAgentType === previous.payload.agentType)
? previous.payload.mainAgent
: undefined
: undefined)
// Why: OSC status is a runtime observation, not a prompt boundary; keep prompt-sent telemetry tied to native hooks.
this.applyNormalizedStatus(
{
@@ -121,7 +121,7 @@ export abstract class AgentHookServerStatusUpdate extends AgentHookServerStatusA
return enriched
}
const stateReconciledPayload =
terminalOwnedPayload.connectionId &&
(terminalOwnedPayload.connectionId || !terminalOwnedPayload.hookEventName) &&
terminalOwnedPayload.payload.agentType === 'codex' &&
(terminalOwnedPayload.hookEventName || terminalOwnedPayload.payload.mainAgent)
? {
+4
View File
@@ -308,6 +308,10 @@ export class HeadlessEmulator {
return lines
}
get isSynchronizedOutput(): boolean {
return this.terminal.modes.synchronizedOutputMode
}
getVisibleBufferRange(): { start: number; endExclusive: number; totalLength: number } {
const buffer = this.terminal.buffer.active
const start = buffer.viewportY
@@ -0,0 +1,30 @@
{
"capturedAt": "2026-10-13T07:26:06.388Z",
"platform": "darwin",
"command": [
"codex",
"--dangerously-bypass-approvals-and-sandbox",
"-c",
"model=\"gpt-5\"",
"-c",
"model_provider=\"capacity_fixture\"",
"-c",
"model_providers.capacity_fixture.name=\"Capacity fixture\"",
"-c",
"model_providers.capacity_fixture.base_url=\"http://127.0.0.1:18791/v1\"",
"-c",
"model_providers.capacity_fixture.wire_api=\"responses\"",
"-c",
"model_providers.capacity_fixture.request_max_retries=0",
"-c",
"model_providers.capacity_fixture.stream_max_retries=0",
"-c",
"model_providers.capacity_fixture.supports_websockets=false",
"-c",
"check_for_update_on_startup=false"
],
"cols": 40,
"rows": 24,
"note": "Real codex-cli 0.162.1, isolated home, local HTTP 503 server_is_overloaded, 40-column fullscreen terminal to capture wrapping.",
"exitCode": 0
}
@@ -0,0 +1 @@
[?2004h[>4;0m[>7u[?1004h]10;?\]11;?\[?u[?2026h[?25l[?1049h[>4;0m[>7u[?1007l[?1000h[?1002h[?1003h[?1006h[?25l[?2026h[?25l>_OpenAI Codex (v0.162.1)loading›Ask Codex to do anything?forshortcuts[0 q [?25h[?2026l[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25l~/…/orca/detect-at-capcity-codex[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?1006l[?1015l[?1003l[?1002l[?1000l[?1007h[?2026h[?25l Folder access/Usersnwprkr/orc/workspaces/orca/detect-at-capcity-codex Note: You’re in a subdirectory of aGit project. Trusting will apply tothe repository root:/Users/xxxxxxxx/projects/orca Trustthisfolder?Codexcanread,edit,andrunfileshere,subjecttoyourpermissionsettings.Foldersettingscanruncodeautomatically,evenwithoutamodelrequest.Continueonlyifyoutrustthesefiles.Yourtrustdecisionwillbesaved.› 1. Trust and continue  2. Quitenter continue · esc quit[?2026l[?1007l[?1000h[?1002h[?1003h[?1006h[?2026h>_OpenAI Codex (v0.162.1)~/…/orca/detect-at-capcity-codex›Ask Codex to do anything?forshortcuts[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l]0;detect-at-capcity-codex[?2026h[?25l permissions: YOLO modegpt-5default·~/orca/workspaces/orc…⚠1·f2[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l]0;⠋ detect-at-capcity-codex]0;detect-at-capcity-codex[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25l   [?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25lSay hi[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25l› SayhiAsk Codex to do anything?forshortcuts[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25l2[?25h[?2026l]0;⠴ detect-at-capcity-codex[?2026h[?25l•Working(0s • esc to interrupt)[?25h[?2026l]0;⠴ ⠴ | detect-at-capcity-codex[?2026h[?25h[?2026l]0;⠴ | detect-at-capcity-codex[?2026h[?25l■ Selected model is at capacity. Pleasetry a different model. [?25h[?2026l]0;detect-at-capcity-codex[?2026h[?25h[?2026l
@@ -0,0 +1,31 @@
{
"capturedAt": "2026-10-13T07:19:49.742Z",
"platform": "darwin",
"command": [
"codex",
"--no-alt-screen",
"--dangerously-bypass-approvals-and-sandbox",
"-c",
"model=\"gpt-5\"",
"-c",
"model_provider=\"capacity_fixture\"",
"-c",
"model_providers.capacity_fixture.name=\"Capacity fixture\"",
"-c",
"model_providers.capacity_fixture.base_url=\"http://127.0.0.1:18791/v1\"",
"-c",
"model_providers.capacity_fixture.wire_api=\"responses\"",
"-c",
"model_providers.capacity_fixture.request_max_retries=0",
"-c",
"model_providers.capacity_fixture.stream_max_retries=0",
"-c",
"model_providers.capacity_fixture.supports_websockets=false",
"-c",
"check_for_update_on_startup=false"
],
"cols": 80,
"rows": 24,
"note": "Real codex-cli 0.162.1 with a temporary CODEX_HOME, local HTTP 503 server_is_overloaded; no production request.",
"exitCode": 0
}
@@ -0,0 +1,14 @@
[?2004h[>4;0m[>7u[?1004h]10;?\]11;?\[?u[?2026h[?2026h[?25l  >_ OpenAI Codex (v0.162.1) loading  › Ask Codex to do anything  [0 q [?25h[?2026l[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25l~/orca/workspaces/orca/detect-at-capcity-codex[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25l Folder access/Users/xxxxxxxx/orca/workspaces/orca/detect-at-capcity-codex Note: You’re in a subdirectory of a Git project. Trusting will apply to therepository root:/Users/xxxxxxxx/projects/orca Trustthisfolder?Codexcanread,edit,andrunfileshere,subjecttoyourpermissionsettings.Foldersettingscanruncodeautomatically,evenwithoutamodelrequest.Continueonlyifyoutrustthesefiles.Yourtrustdecisionwillbesaved.› 1. Trust and continue 2.Quitenter continue · esc quit[?2026l[?2026h[?25l  >_ OpenAI Codex (v0.162.1) ~/orca/workspaces/orca/detect-at-capcity-codex  › Ask Codex to do anything  [0 q [?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l]0;detect-at-capcity-codex[?2026h[?25h[?2026l[?2026h[?25l  >_ OpenAI Codex (v0.162.1) ~/orca/workspaces/orca/detect-at-capcity-codex permissions: YOLO mode  › Ask Codex to do anything  gpt-5 default · ~/orca/workspaces/orca/detect-at-capcity-codex[0 q [?25h[?2026l[?2026h[?25l… ⚠ 1 warning·f2toview[?25h[?2026l[?2026h[?25h[?2026l]0;⠙ detect-at-capcity-codex]0;detect-at-capcity-codex[?2026hMMMMMMM

 >_ OpenAI Codex (v0.162.1)
 ~/orca/workspaces/orca/detect-at-capcity-codex
 permissions: YOLO mode

 Tip: Maximize usage with GPT-6.1 Sol. Try it on complex work for near-Astra
 performance at a lower cost.[?25l  › Ask Codex to do anything  gpt-5 default · ~/orca/workspaces/orca/detect-at-c… ⚠ 1 warning · f2 to view[0 q [?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25lSay hi[?25h[?2026l[?2026hMMMM


› Say hi
[?25lAsk Codex to do anything[0 q [?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25l… ⚠ 2 warnings[?25h[?2026l]0;⠼ detect-at-capcity-codex[?2026h[?25l • Working (0s • esc to interrupt)  › Ask Codex to do anything  gpt-5 default · ~/orca/workspaces/orca/detect-at-… ⚠ 2 warnings · f2 to view[0 q [?25h[?2026l]0;⠴ ⠴ | detect-at-capcity-codex[?2026h[?25h[?2026l]0;⠴ | detect-at-capcity-codex[?2026hMM

■ Selected model is at capacity. Please try a different model.[?25l  › Ask Codex to do anything  gpt-5 default · ~/orca/workspaces/orca/detect-at-… ⚠ 2 warnings · f2 to view[0 q [?25h[?2026l[?2026h[?25h[?2026l]0;detect-at-capcity-codex[?2026h[?25h[?2026l
@@ -0,0 +1,254 @@
import { readFileSync } from 'node:fs'
import { describe, expect, it, vi } from 'vitest'
import { HeadlessEmulator } from '../daemon/headless-emulator'
import {
CODEX_CAPACITY_MESSAGE,
CodexCapacityNoticeTracker
} from '../../shared/codex-capacity-notice'
import { observeCodexCapacityTerminalStatus } from './codex-capacity-terminal-status'
import * as agentProcessRecognition from '../../shared/agent-process-recognition'
import { createTranscriptPane, TRANSCRIPT_PANE_PTY_ID } from './agent-transcript-pane-test-harness'
const transcript = readFileSync(
new URL('./__fixtures__/codex-model-capacity.txt', import.meta.url),
'utf8'
)
const fullscreenTranscript = readFileSync(
new URL('./__fixtures__/codex-model-capacity-fullscreen.txt', import.meta.url),
'utf8'
)
describe('Codex model capacity terminal status', () => {
it.each([
[transcript, 80, 1],
[transcript, 80, 7],
[transcript, 80, 64],
[transcript, 80, 8192],
[fullscreenTranscript, 40, 1],
[fullscreenTranscript, 40, 7],
[fullscreenTranscript, 40, 64],
[fullscreenTranscript, 40, 8192]
] as const)('reads a captured notice %#', async (recording, cols, size) => {
const emulator = new HeadlessEmulator({ cols, rows: 24 })
const tracker = new CodexCapacityNoticeTracker()
let detected = false
try {
for (let index = 0; index < recording.length; index += size) {
const chunk = recording.slice(index, index + size)
await emulator.write(chunk)
detected ||= tracker.observe(chunk, () => emulator.getVisibleLines())
}
expect(detected).toBe(true)
} finally {
emulator.dispose()
}
})
it('does not read the screen for ordinary output or OSC string payloads', () => {
const tracker = new CodexCapacityNoticeTracker()
const readScreen = vi.fn(() => [`■ ${CODEX_CAPACITY_MESSAGE}`, '›'])
tracker.observe('x'.repeat(1_000_000), readScreen)
tracker.observe(`\x1b]0;■ ${CODEX_CAPACITY_MESSAGE}\x07`, readScreen)
expect(readScreen).not.toHaveBeenCalled()
})
it('ignores prose, old history before a new prompt, and a busy turn', () => {
for (const lines of [
[`• The error says: ■ ${CODEX_CAPACITY_MESSAGE}`, '›'],
[`■ ${CODEX_CAPACITY_MESSAGE}`, '› continue', '• Working (1s • esc to interrupt)', '›'],
[`■ ${CODEX_CAPACITY_MESSAGE}`, '• Working (1s • esc to interrupt)', '›']
]) {
const tracker = new CodexCapacityNoticeTracker()
expect(tracker.observe(`■ ${CODEX_CAPACITY_MESSAGE}`, () => lines)).toBe(false)
}
})
it('handles a wrapped native warning and waits for the synchronized frame to finish', () => {
const tracker = new CodexCapacityNoticeTracker()
const screen = vi.fn(() => [
'⚠ Selected model is at capacity. Please',
' try a different model.',
'',
'›'
])
expect(
tracker.observe(
'\x1b[?2026h⚠ Selected model is at\ncapacity. Please\ntry a different model.',
screen
)
).toBe(false)
expect(screen).not.toHaveBeenCalled()
expect(tracker.observe('\x1b[?2026l', screen)).toBe(true)
})
it('publishes a failed root through the execution-host terminal status path', async () => {
const onTerminalAgentStatus = vi.fn()
const { runtime } = await createTranscriptPane(
{
paneTitle: 'Codex',
foregroundProcess: 'codex',
launchAgent: 'codex',
data: ''
},
{ onTerminalAgentStatus }
)
try {
runtime.onPtyData(TRANSCRIPT_PANE_PTY_ID, transcript, 1)
await vi.waitFor(() =>
expect(onTerminalAgentStatus).toHaveBeenCalledWith(
expect.objectContaining({
payload: expect.objectContaining({
state: 'done',
agentType: 'codex',
lastAssistantMessage: CODEX_CAPACITY_MESSAGE,
mainAgent: expect.objectContaining({ state: 'done', outcome: 'failure' })
})
})
)
)
} finally {
await runtime.onPtyExit(TRANSCRIPT_PANE_PTY_ID, 0)
}
})
it('does not produce client-side status for an SSH-owned PTY', async () => {
const onTerminalAgentStatus = vi.fn()
const { runtime, handle } = await createTranscriptPane(
{
paneTitle: 'Codex',
foregroundProcess: 'codex',
launchAgent: 'codex',
connectionId: 'ssh-host',
data: ''
},
{ onTerminalAgentStatus }
)
try {
runtime.onPtyData(TRANSCRIPT_PANE_PTY_ID, transcript, 1)
await runtime.readTerminal(handle)
expect(onTerminalAgentStatus).not.toHaveBeenCalled()
} finally {
await runtime.onPtyExit(TRANSCRIPT_PANE_PTY_ID, 0)
}
})
it('ignores unrelated agents', async () => {
const emulator = new HeadlessEmulator({ cols: 80, rows: 24 })
try {
await emulator.write(transcript)
expect(
observeCodexCapacityTerminalStatus(
emulator,
transcript,
'claude',
'claude',
() => undefined
)
).toBeNull()
} finally {
emulator.dispose()
}
})
it('detects Codex launched inside a shell from its hook identity and skips normal output', async () => {
const emulator = new HeadlessEmulator({ cols: 80, rows: 24 })
const readStatus = vi.fn(() => ({ agentType: 'codex', prompt: 'Say hi', model: 'gpt-5' }))
try {
expect(
observeCodexCapacityTerminalStatus(emulator, 'normal output', 'zsh', undefined, readStatus)
).toBeNull()
expect(readStatus).not.toHaveBeenCalled()
await emulator.write(transcript)
expect(
observeCodexCapacityTerminalStatus(emulator, transcript, 'zsh', undefined, readStatus)
).toMatchObject({
prompt: 'Say hi',
model: 'gpt-5',
mainAgent: { state: 'done', outcome: 'failure' }
})
} finally {
emulator.dispose()
}
})
it.each([100, 101])(
'does not close a hook turn starting at or after output time (%i)',
async (turnStartedAt) => {
const emulator = new HeadlessEmulator({ cols: 80, rows: 24 })
try {
await emulator.write(transcript)
expect(
observeCodexCapacityTerminalStatus(
emulator,
transcript,
undefined,
'codex',
() => ({
agentType: 'codex',
prompt: 'Retry',
turnStartedAt
}),
100
)
).toBeNull()
} finally {
emulator.dispose()
}
}
)
it.each([1, 7, 64, 8192])(
'detects a first shell-launched notice in %i-character chunks',
async (size) => {
const start = fullscreenTranscript.lastIndexOf(
'\x1b[?2026h',
fullscreenTranscript.indexOf('■ Selected model')
)
const end = fullscreenTranscript.indexOf('\x1b[?2026l', start) + '\x1b[?2026l'.length
const emulator = new HeadlessEmulator({ cols: 40, rows: 24 })
let detections = 0
try {
await emulator.write(fullscreenTranscript.slice(0, start))
for (let index = start; index < end; index += size) {
const chunk = fullscreenTranscript.slice(index, Math.min(index + size, end))
await emulator.write(chunk)
if (
observeCodexCapacityTerminalStatus(emulator, chunk, 'zsh', undefined, () => ({
agentType: 'codex',
prompt: 'Say hi'
}))
) {
detections++
}
}
expect(detections).toBe(1)
} finally {
emulator.dispose()
}
}
)
it('recognizes an unrelated launch once and invalidates when its command changes', () => {
const emulator = new HeadlessEmulator({ cols: 80, rows: 24 })
const recognize = vi.spyOn(agentProcessRecognition, 'recognizeAgentProcessFromCommandLine')
const readStatus = vi.fn(() => undefined)
try {
for (let index = 0; index < 1000; index++) {
observeCodexCapacityTerminalStatus(
emulator,
'ordinary output',
'zsh',
undefined,
readStatus
)
}
expect(recognize).toHaveBeenCalledTimes(1)
observeCodexCapacityTerminalStatus(emulator, 'ordinary output', 'bash', undefined, readStatus)
expect(recognize).toHaveBeenCalledTimes(2)
expect(readStatus).not.toHaveBeenCalled()
} finally {
recognize.mockRestore()
emulator.dispose()
}
})
})
@@ -0,0 +1,79 @@
import {
CodexCapacityNoticeTracker,
CODEX_CAPACITY_MESSAGE
} from '../../shared/codex-capacity-notice'
import type {
AgentStatusIpcPayload,
ParsedAgentStatusPayload
} from '../../shared/agent-status-types'
import { recognizeAgentProcessFromCommandLine } from '../../shared/agent-process-recognition'
import type { HeadlessEmulator } from '../daemon/headless-emulator'
const trackers = new WeakMap<HeadlessEmulator, CodexCapacityNoticeTracker>()
const launchIdentities = new WeakMap<HeadlessEmulator, { command?: string; codex: boolean }>()
type CodexCapacityStatus = Pick<
AgentStatusIpcPayload,
'agentType' | 'prompt' | 'model' | 'subagents' | 'mainAgent' | 'turnStartedAt'
>
export function observeCodexCapacityTerminalStatus(
emulator: HeadlessEmulator,
data: string,
startupCommand: string | undefined,
launchAgent: string | null | undefined,
readStatus: () => CodexCapacityStatus | undefined,
outputObservedAt = Date.now()
): ParsedAgentStatusPayload | null {
let tracker = trackers.get(emulator)
if (!tracker && launchAgent !== 'codex') {
let identity = launchIdentities.get(emulator)
if (!identity || identity.command !== startupCommand) {
identity = {
command: startupCommand,
codex: recognizeAgentProcessFromCommandLine(startupCommand)?.agent === 'codex'
}
launchIdentities.set(emulator, identity)
}
// A Codex started from an existing shell is identified by its host-owned hook row.
if (
!identity.codex &&
((!data.includes('■') && !data.includes('⚠')) || readStatus()?.agentType !== 'codex')
) {
return null
}
}
if (!tracker) {
tracker = new CodexCapacityNoticeTracker()
trackers.set(emulator, tracker)
}
if (
!tracker.observe(
data,
() => emulator.getVisibleLines(),
() => emulator.isSynchronizedOutput
)
) {
return null
}
const previous = readStatus()
if (previous && previous.agentType !== 'codex') {
return null
}
// Same-tick ordering is ambiguous; queued output must yield to a newer hook turn.
if (previous?.turnStartedAt !== undefined && previous.turnStartedAt >= outputObservedAt) {
return null
}
if (previous?.mainAgent?.outcome === 'failure') {
return null
}
return {
state: 'done',
agentType: 'codex',
prompt: previous?.prompt ?? '',
model: previous?.model,
subagents: previous?.subagents,
lastAssistantMessage: CODEX_CAPACITY_MESSAGE,
lastAssistantMessageIsToolOutput: true,
mainAgent: { state: 'done', outcome: 'failure', stateStartedAt: Date.now() }
}
}
@@ -2,6 +2,7 @@
import { splitFreebuffScreenUpdates } from '../../shared/freebuff-screen-status'
import { OrcaRuntimeWithSerializeMainTerminalBuffer } from './orca-runtime-serialize-main-terminal-buffer'
import { observeFreebuffTerminalStatus } from './freebuff-terminal-status'
import { observeCodexCapacityTerminalStatus } from './codex-capacity-terminal-status'
import { MOBILE_SUBSCRIBE_SCROLLBACK_ROWS } from './scrollback-limits'
import { detectAgentStatusFromTitle, normalizeTerminalTitle } from '../../shared/agent-detection'
import { shouldModelAnswerHiddenPtyQueries } from './terminal-model-query-authority'
@@ -173,6 +174,7 @@ export class OrcaRuntimeWithMaybeHydrateHeadlessFromRenderer extends OrcaRuntime
forwardQueryReplies = false
): void {
const state = this.getOrCreateHeadlessTerminal(ptyId)
const outputObservedAt = Date.now()
const completion = state.writeChain.then(async () => {
// Why: the ingestion-time ownership decision is closed over this
// chain link; async scheduling cannot retroactively change it.
@@ -183,12 +185,21 @@ export class OrcaRuntimeWithMaybeHydrateHeadlessFromRenderer extends OrcaRuntime
await state.emulator.write(chunk, { forwardQueryReplies })
const pty = this.ptysById.get(ptyId)
if (pty && !pty.connectionId && this.headlessTerminals.get(ptyId) === state) {
const payload = observeFreebuffTerminalStatus(
state.emulator,
chunk,
this.terminalSpawnCommandsByPtyId.get(ptyId),
pty.launchAgent
)
const command = this.terminalSpawnCommandsByPtyId.get(ptyId)
const payload =
observeCodexCapacityTerminalStatus(
state.emulator,
chunk,
command,
pty.launchAgent,
() => {
const paneKey =
this.getLeavesForPty(ptyId).map((leaf) => this.makeRuntimePaneKey(leaf))[0] ??
pty.paneKey
return paneKey ? this.getAgentStatusSnapshotForPaneFn?.(paneKey)[0] : undefined
},
outputObservedAt
) ?? observeFreebuffTerminalStatus(state.emulator, chunk, command, pty.launchAgent)
if (payload) {
pty.lastExplicitAgentStatus = {
state: payload.state,
@@ -1,4 +1,4 @@
import type { ParsedAgentStatusPayload } from './agent-status-types'
import { mainAgentStatusEqual, type ParsedAgentStatusPayload } from './agent-status-types'
/** Whether a terminal status is the same hook status with only hook-owned metadata omitted.
*
@@ -13,6 +13,8 @@ export function terminalStatusPayloadMatchesHook(
// OSC never carries model/children; compare only fields its protocol owns.
return (
cachedHookPayload.state === terminalPayload.state &&
(terminalPayload.mainAgent === undefined ||
mainAgentStatusEqual(cachedHookPayload.mainAgent, terminalPayload.mainAgent)) &&
// OSC never carries workingMode either; only a stated mismatch breaks equivalence.
(terminalPayload.workingMode === undefined ||
cachedHookPayload.workingMode === terminalPayload.workingMode) &&
+51
View File
@@ -0,0 +1,51 @@
import { describe, expect, it, vi } from 'vitest'
import { CODEX_CAPACITY_MESSAGE, CodexCapacityNoticeTracker } from './codex-capacity-notice'
describe('Codex capacity candidate bounds', () => {
it('reads neither the screen nor its mode during ordinary output', () => {
const tracker = new CodexCapacityNoticeTracker()
const readScreen = vi.fn(() => [])
const readSynchronized = vi.fn(() => false)
tracker.observe('ordinary output'.repeat(1000), readScreen, readSynchronized)
expect(readScreen).not.toHaveBeenCalled()
expect(readSynchronized).not.toHaveBeenCalled()
})
it('does not inspect a partial synchronized frame with combined ANSI modes', () => {
const tracker = new CodexCapacityNoticeTracker()
const readScreen = vi.fn(() => [`■ ${CODEX_CAPACITY_MESSAGE}`, '›'])
expect(tracker.observe(`\x1b[?25;2026h■ ${CODEX_CAPACITY_MESSAGE}`, readScreen)).toBe(false)
expect(readScreen).not.toHaveBeenCalled()
expect(tracker.observe('\x1b[?2026;25l', readScreen)).toBe(true)
expect(readScreen).toHaveBeenCalledTimes(1)
})
it('ignores repeated unrelated markers and still accepts a later native notice', () => {
const tracker = new CodexCapacityNoticeTracker()
const readScreen = vi.fn(() => [`■ ${CODEX_CAPACITY_MESSAGE}`, '›'])
expect(tracker.observe('■'.repeat(100_000), readScreen)).toBe(false)
expect(readScreen).not.toHaveBeenCalled()
expect(tracker.observe(`■ ${CODEX_CAPACITY_MESSAGE}`, readScreen)).toBe(true)
expect(readScreen).toHaveBeenCalledTimes(1)
})
it('finds the notice inside a large text chunk after unrelated warning markers', () => {
const tracker = new CodexCapacityNoticeTracker()
const readScreen = vi.fn(() => [`■ ${CODEX_CAPACITY_MESSAGE}`, '›'])
expect(
tracker.observe(
`⚠ unrelated\n${'x'.repeat(1024 * 1024)}\n■ ${CODEX_CAPACITY_MESSAGE}`,
readScreen
)
).toBe(true)
expect(readScreen).toHaveBeenCalledTimes(1)
})
it('recognizes a split notice immediately after abandoning an unrelated warning', () => {
const tracker = new CodexCapacityNoticeTracker()
const readScreen = vi.fn(() => [`⚠ ${CODEX_CAPACITY_MESSAGE}`, '›'])
expect(tracker.observe('⚠ 1 warning · f2 to view', readScreen)).toBe(false)
expect(readScreen).not.toHaveBeenCalled()
expect(tracker.observe('⚠ Selected model is ', readScreen)).toBe(false)
expect(tracker.observe('at capacity. Please try a different model.', readScreen)).toBe(true)
expect(tracker.observe('ordinary output'.repeat(100_000), readScreen)).toBe(false)
expect(readScreen).toHaveBeenCalledTimes(1)
})
})
+87
View File
@@ -0,0 +1,87 @@
import { createTerminalEscapeScanner } from './terminal-escape-scanner'
export const CODEX_CAPACITY_MESSAGE = 'Selected model is at capacity. Please try a different model.'
const NOTICE = /^[■⚠]\s+Selected model is at capacity\. Please try a different model\.$/
const MAX_NOTICE_LENGTH = 160
/** Only a newly painted native notice at the end of the conversation can close a turn. */
export class CodexCapacityNoticeTracker {
private tail = ''
private pending = false
private synchronized = false
private readonly scanner = createTerminalEscapeScanner({
onCsi: (marker, params, final) => {
if (
marker === '?' &&
params.split(';').includes('2026') &&
(final === 'h' || final === 'l')
) {
this.synchronized = final === 'h'
} else if ((final === 'H' || final === 'f') && this.tail && !this.tail.endsWith(' ')) {
this.tail += ' '
}
},
onText: (text) => this.observeText(text)
})
private observeText(text: string): void {
if (this.tail) {
this.updateCandidate(this.tail + text.slice(0, MAX_NOTICE_LENGTH))
}
let error = text.indexOf('■')
let warning = text.indexOf('⚠')
while (error !== -1 || warning !== -1) {
const start = error === -1 ? warning : warning === -1 ? error : Math.min(error, warning)
this.updateCandidate(text.slice(start, start + MAX_NOTICE_LENGTH))
if (start === error) {
error = text.indexOf('■', start + 1)
} else {
warning = text.indexOf('⚠', start + 1)
}
}
}
private updateCandidate(text: string): void {
const content = text.slice(1).trimStart()
if (content && content[0] !== CODEX_CAPACITY_MESSAGE[0]) {
this.tail = ''
return
}
const candidate = text.replace(/\s+/g, ' ')
const message = candidate.slice(1).trimStart()
if (message.startsWith(CODEX_CAPACITY_MESSAGE)) {
this.pending = true
this.tail = ''
} else {
this.tail = CODEX_CAPACITY_MESSAGE.startsWith(message) ? candidate : ''
}
}
observe(
data: string,
readScreen: () => readonly string[],
readSynchronized?: () => boolean
): boolean {
this.scanner.observe(data)
if (!this.pending || (readSynchronized?.() ?? this.synchronized)) {
return false
}
this.pending = false
this.tail = ''
const lines = readScreen()
const composer = lines.findLastIndex((line) => /^\s*›/.test(line))
if (composer === -1) {
return false
}
const above = lines.slice(0, composer).filter((line) => line.trim() !== '')
let notice = ''
for (let index = above.length - 1; index >= 0 && notice.length < MAX_NOTICE_LENGTH; index--) {
notice = `${above[index].trim()} ${notice}`.trim()
if (NOTICE.test(notice)) {
return true
}
}
return false
}
}