From 9e52645e6cd8d6d110f37cd883b4b8987e99fbfd Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:58:00 -0700 Subject: [PATCH] Show Codex model capacity failures as red agent status (#28231) * Show Codex model capacity failures in agent status * Reduce Codex capacity detection work on terminal output * Preserve Codex capacity notices across partial terminal frames --- .../codex-capacity-status-benchmark.ts | 137 ++++++++++ .../agent-hooks/server-codex-capacity.test.ts | 93 +++++++ .../server/server-ingest-terminal.ts | 5 +- .../server/server-status-update.ts | 2 +- src/main/daemon/headless-emulator.ts | 4 + .../codex-model-capacity-fullscreen.meta.json | 30 +++ .../codex-model-capacity-fullscreen.txt | 1 + .../codex-model-capacity.meta.json | 31 +++ .../__fixtures__/codex-model-capacity.txt | 14 + .../codex-capacity-terminal-status.test.ts | 254 ++++++++++++++++++ .../runtime/codex-capacity-terminal-status.ts | 79 ++++++ ...me-maybe-hydrate-headless-from-renderer.ts | 23 +- .../agent-terminal-status-equivalence.ts | 4 +- src/shared/codex-capacity-notice.test.ts | 51 ++++ src/shared/codex-capacity-notice.ts | 87 ++++++ 15 files changed, 805 insertions(+), 10 deletions(-) create mode 100644 config/scripts/codex-capacity-status-benchmark.ts create mode 100644 src/main/agent-hooks/server-codex-capacity.test.ts create mode 100644 src/main/runtime/__fixtures__/codex-model-capacity-fullscreen.meta.json create mode 100644 src/main/runtime/__fixtures__/codex-model-capacity-fullscreen.txt create mode 100644 src/main/runtime/__fixtures__/codex-model-capacity.meta.json create mode 100644 src/main/runtime/__fixtures__/codex-model-capacity.txt create mode 100644 src/main/runtime/codex-capacity-terminal-status.test.ts create mode 100644 src/main/runtime/codex-capacity-terminal-status.ts create mode 100644 src/shared/codex-capacity-notice.test.ts create mode 100644 src/shared/codex-capacity-notice.ts diff --git a/config/scripts/codex-capacity-status-benchmark.ts b/config/scripts/codex-capacity-status-benchmark.ts new file mode 100644 index 00000000000..acf786f17ce --- /dev/null +++ b/config/scripts/codex-capacity-status-benchmark.ts @@ -0,0 +1,137 @@ +// Run: ORCA_BACKGROUND_LAUNCH=1 node --import tsx config/scripts/codex-capacity-status-benchmark.ts +import { readFileSync } from 'node:fs' +import { performance } from 'node:perf_hooks' +import { CodexCapacityNoticeTracker } from '../../src/shared/codex-capacity-notice' +import { HeadlessEmulator } from '../../src/main/daemon/headless-emulator' +import { observeCodexCapacityTerminalStatus } from '../../src/main/runtime/codex-capacity-terminal-status' + +const fixture = readFileSync( + new URL( + '../../src/main/runtime/__fixtures__/codex-model-capacity-fullscreen.txt', + import.meta.url + ), + 'utf8' +) +const notice = fixture.indexOf('■ Selected model') +if (notice === -1) { + throw new Error('Capacity notice missing from benchmark fixture') +} +const redraw = fixture.slice(0, fixture.lastIndexOf('\x1b[?2026h', notice)) +const scenarios = [ + { name: 'plain-4K', data: 'x'.repeat(4096), iterations: 30_000 }, + { name: 'captured-TUI', data: redraw, iterations: 30_000 }, + { + name: 'warning-footer', + data: '\x1b[?2026h\x1b[21;1H⚠ 1 warning · f2 to view\x1b[?2026l', + iterations: 30_000 + }, + { name: 'tiny-typing', data: 'a', iterations: 30_000 }, + { name: 'marker-flood', data: '■'.repeat(4096), iterations: 1000 }, + { + name: 'large-after-warning', + data: `⚠ unrelated\n${'x'.repeat(1024 * 1024)}`, + iterations: 128 + } +] + +function measure(iterations: number, observe: () => void): number { + const start = performance.now() + for (let index = 0; index < iterations; index++) { + observe() + } + return performance.now() - start +} + +for (const scenario of scenarios) { + const run = () => { + const tracker = new CodexCapacityNoticeTracker() + let screenReads = 0 + const readScreen = () => { + screenReads++ + return [] + } + const elapsedMs = measure(scenario.iterations, () => tracker.observe(scenario.data, readScreen)) + if (screenReads !== 0) { + throw new Error(`${scenario.name} read the screen ${screenReads} times`) + } + return elapsedMs + } + run() + const samples = Array.from({ length: 5 }, run).sort((left, right) => left - right) + const elapsedMs = samples[2] + textResult(scenario.name, elapsedMs, scenario.iterations, scenario.data.length) +} + +for (const launchAgent of [undefined, 'codex'] as const) { + const emulator = new HeadlessEmulator({ cols: 80, rows: 24 }) + const data = 'x'.repeat(4096) + const iterations = 30_000 + let statusReads = 0 + const readStatus = () => { + statusReads++ + return undefined + } + try { + const observe = () => + observeCodexCapacityTerminalStatus( + emulator, + data, + 'node /some/wrapper.cjs', + launchAgent, + readStatus, + 1 + ) + measure(iterations, observe) + const samples = Array.from({ length: 5 }, () => measure(iterations, observe)).sort( + (left, right) => left - right + ) + if (statusReads !== 0) { + throw new Error(`Ordinary output read status ${statusReads} times`) + } + textResult(launchAgent ?? 'unrelated-terminal', samples[2], iterations, data.length) + } finally { + emulator.dispose() + } +} + +async function measureScreenRead(): Promise { + const emulator = new HeadlessEmulator({ cols: 40, rows: 24 }) + try { + await emulator.write(fixture) + for (const name of ['screen-only', 'capacity-parse-and-screen']) { + const observe = () => { + if (name === 'screen-only') { + emulator.getVisibleLines() + } else if ( + !new CodexCapacityNoticeTracker().observe(fixture, () => emulator.getVisibleLines()) + ) { + throw new Error('Captured capacity notice was not detected') + } + } + measure(1000, observe) + const samples = Array.from({ length: 5 }, () => measure(1000, observe)).sort( + (left, right) => left - right + ) + textResult(name, samples[2], 1000, fixture.length) + } + } finally { + emulator.dispose() + } +} + +void measureScreenRead().catch((error: unknown) => { + console.error(error) + process.exitCode = 1 +}) + +function textResult(name: string, elapsedMs: number, iterations: number, chars: number): void { + console.log( + JSON.stringify({ + name, + chars, + iterations, + microsecondsPerChunk: Number(((elapsedMs * 1000) / iterations).toFixed(3)), + millisecondsPerMiB: Number((elapsedMs / ((chars * iterations) / 1048576)).toFixed(3)) + }) + ) +} diff --git a/src/main/agent-hooks/server-codex-capacity.test.ts b/src/main/agent-hooks/server-codex-capacity.test.ts new file mode 100644 index 00000000000..3677462dd8d --- /dev/null +++ b/src/main/agent-hooks/server-codex-capacity.test.ts @@ -0,0 +1,93 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { AgentHookServer, _internals } from './server' +import { buildBody, PANE, postHookEvent } from './server.test-fixtures' +import { CODEX_CAPACITY_MESSAGE } from '../../shared/codex-capacity-notice' + +afterEach(() => _internals.resetCachesForTests()) + +describe('Codex capacity verdict in the shared status store', () => { + it('keeps a local failure through Stop and child activity, then clears it on retry', async () => { + const server = new AgentHookServer() + await server.start({ env: 'production' }) + const post = (payload: Record) => + postHookEvent(server, buildBody({ session_id: 'session-1', ...payload }), '/hook/codex') + try { + expect( + (await post({ hook_event_name: 'UserPromptSubmit', prompt: 'Say hi', model: 'gpt-5' })).ok + ).toBe(true) + const previous = server.getStatusSnapshotForPane(PANE)[0] + server.ingestTerminalStatus({ + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + payload: { + state: 'done', + agentType: 'codex', + prompt: previous?.prompt ?? '', + model: previous?.model, + lastAssistantMessage: CODEX_CAPACITY_MESSAGE, + mainAgent: { state: 'done', outcome: 'failure', stateStartedAt: Date.now() } + } + }) + expect(server.getStatusSnapshotForPane(PANE)[0]).toMatchObject({ + prompt: 'Say hi', + model: 'gpt-5', + lastAssistantMessage: CODEX_CAPACITY_MESSAGE, + providerSession: { id: 'session-1' }, + mainAgent: { state: 'done', outcome: 'failure' } + }) + await post({ hook_event_name: 'Stop' }) + await post({ hook_event_name: 'SubagentStart', agent_id: 'child-1' }) + expect(server.getStatusSnapshotForPane(PANE)[0]).toMatchObject({ + state: 'working', + mainAgent: { state: 'done', outcome: 'failure' } + }) + await post({ hook_event_name: 'UserPromptSubmit', prompt: 'Say hi' }) + expect(server.getStatusSnapshotForPane(PANE)[0]?.mainAgent).toMatchObject({ + state: 'working' + }) + expect(server.getStatusSnapshotForPane(PANE)[0]?.mainAgent?.outcome).toBeUndefined() + } finally { + server.stop() + } + }) + + it('accepts host capacity evidence after an earlier done and preserves it across relay restatements', () => { + const server = new AgentHookServer() + server.ingestRemote( + { + paneKey: PANE, + payload: { + state: 'done', + agentType: 'codex', + prompt: 'Say hi', + mainAgent: { state: 'done', stateStartedAt: 1 } + } + }, + 'ssh-host' + ) + server.ingestTerminalStatus({ + paneKey: PANE, + connectionId: 'ssh-host', + payload: { + state: 'done', + agentType: 'codex', + prompt: 'Say hi', + mainAgent: { state: 'done', outcome: 'failure', stateStartedAt: 2 } + } + }) + server.ingestRemote( + { + paneKey: PANE, + hookEventName: 'Stop', + payload: { + state: 'done', + agentType: 'codex', + prompt: 'Say hi' + } + }, + 'ssh-host' + ) + expect(server.getStatusSnapshotForPane(PANE)[0]?.mainAgent?.outcome).toBe('failure') + }) +}) diff --git a/src/main/agent-hooks/server/server-ingest-terminal.ts b/src/main/agent-hooks/server/server-ingest-terminal.ts index 004a6462d27..f334f57f528 100644 --- a/src/main/agent-hooks/server/server-ingest-terminal.ts +++ b/src/main/agent-hooks/server/server-ingest-terminal.ts @@ -159,11 +159,12 @@ export abstract class AgentHookServerIngestTerminal extends AgentHookServerInges // Why: OSC carries no main agent fact. While it repaints the state the hook row already holds, the // main agent behind that state is unchanged too; a different state is a turn edge OSC cannot date. const preservedMainAgent = - previous?.payload.mainAgent && + event.payload.mainAgent ?? + (previous?.payload.mainAgent && previous.payload.state === event.payload.state && (claimedAgentType === undefined || claimedAgentType === previous.payload.agentType) ? previous.payload.mainAgent - : undefined + : undefined) // Why: OSC status is a runtime observation, not a prompt boundary; keep prompt-sent telemetry tied to native hooks. this.applyNormalizedStatus( { diff --git a/src/main/agent-hooks/server/server-status-update.ts b/src/main/agent-hooks/server/server-status-update.ts index 691462d9a86..e9d98098c05 100644 --- a/src/main/agent-hooks/server/server-status-update.ts +++ b/src/main/agent-hooks/server/server-status-update.ts @@ -121,7 +121,7 @@ export abstract class AgentHookServerStatusUpdate extends AgentHookServerStatusA return enriched } const stateReconciledPayload = - terminalOwnedPayload.connectionId && + (terminalOwnedPayload.connectionId || !terminalOwnedPayload.hookEventName) && terminalOwnedPayload.payload.agentType === 'codex' && (terminalOwnedPayload.hookEventName || terminalOwnedPayload.payload.mainAgent) ? { diff --git a/src/main/daemon/headless-emulator.ts b/src/main/daemon/headless-emulator.ts index fa1cb054ee3..b78dd2857da 100644 --- a/src/main/daemon/headless-emulator.ts +++ b/src/main/daemon/headless-emulator.ts @@ -308,6 +308,10 @@ export class HeadlessEmulator { return lines } + get isSynchronizedOutput(): boolean { + return this.terminal.modes.synchronizedOutputMode + } + getVisibleBufferRange(): { start: number; endExclusive: number; totalLength: number } { const buffer = this.terminal.buffer.active const start = buffer.viewportY diff --git a/src/main/runtime/__fixtures__/codex-model-capacity-fullscreen.meta.json b/src/main/runtime/__fixtures__/codex-model-capacity-fullscreen.meta.json new file mode 100644 index 00000000000..d1467510ae0 --- /dev/null +++ b/src/main/runtime/__fixtures__/codex-model-capacity-fullscreen.meta.json @@ -0,0 +1,30 @@ +{ + "capturedAt": "2026-10-13T07:26:06.388Z", + "platform": "darwin", + "command": [ + "codex", + "--dangerously-bypass-approvals-and-sandbox", + "-c", + "model=\"gpt-5\"", + "-c", + "model_provider=\"capacity_fixture\"", + "-c", + "model_providers.capacity_fixture.name=\"Capacity fixture\"", + "-c", + "model_providers.capacity_fixture.base_url=\"http://127.0.0.1:18791/v1\"", + "-c", + "model_providers.capacity_fixture.wire_api=\"responses\"", + "-c", + "model_providers.capacity_fixture.request_max_retries=0", + "-c", + "model_providers.capacity_fixture.stream_max_retries=0", + "-c", + "model_providers.capacity_fixture.supports_websockets=false", + "-c", + "check_for_update_on_startup=false" + ], + "cols": 40, + "rows": 24, + "note": "Real codex-cli 0.162.1, isolated home, local HTTP 503 server_is_overloaded, 40-column fullscreen terminal to capture wrapping.", + "exitCode": 0 +} diff --git a/src/main/runtime/__fixtures__/codex-model-capacity-fullscreen.txt b/src/main/runtime/__fixtures__/codex-model-capacity-fullscreen.txt new file mode 100644 index 00000000000..f0cddc06e68 --- /dev/null +++ b/src/main/runtime/__fixtures__/codex-model-capacity-fullscreen.txt @@ -0,0 +1 @@ +[?2004h[>4;0m[>7u[?1004h]10;?\]11;?\[?u[?2026h[?25l[?1049h[>4;0m[>7u[?1007l[?1000h[?1002h[?1003h[?1006h[?25l[?2026h[?25l>_OpenAI Codex (v0.162.1)loading›Ask Codex to do anything?forshortcuts[0 q [?25h[?2026l[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25l~/…/orca/detect-at-capcity-codex[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?1006l[?1015l[?1003l[?1002l[?1000l[?1007h[?2026h[?25l Folder access/Usersnwprkr/orc/workspaces/orca/detect-at-capcity-codex Note: You’re in a subdirectory of aGit project. Trusting will apply tothe repository root:/Users/xxxxxxxx/projects/orca Trustthisfolder?Codexcanread,edit,andrunfileshere,subjecttoyourpermissionsettings.Foldersettingscanruncodeautomatically,evenwithoutamodelrequest.Continueonlyifyoutrustthesefiles.Yourtrustdecisionwillbesaved.› 1. Trust and continue  2. Quitenter continue · esc quit[?2026l[?1007l[?1000h[?1002h[?1003h[?1006h[?2026h>_OpenAI Codex (v0.162.1)~/…/orca/detect-at-capcity-codex›Ask Codex to do anything?forshortcuts[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l]0;detect-at-capcity-codex[?2026h[?25l permissions: YOLO modegpt-5default·~/orca/workspaces/orc…⚠1·f2[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l]0;⠋ detect-at-capcity-codex]0;detect-at-capcity-codex[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25l   [?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25lSay hi[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25l› SayhiAsk Codex to do anything?forshortcuts[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25l2[?25h[?2026l]0;⠴ detect-at-capcity-codex[?2026h[?25l•Working(0s • esc to interrupt)[?25h[?2026l]0;⠴ ⠴ | detect-at-capcity-codex[?2026h[?25h[?2026l]0;⠴ | detect-at-capcity-codex[?2026h[?25l■ Selected model is at capacity. Pleasetry a different model. [?25h[?2026l]0;detect-at-capcity-codex[?2026h[?25h[?2026l \ No newline at end of file diff --git a/src/main/runtime/__fixtures__/codex-model-capacity.meta.json b/src/main/runtime/__fixtures__/codex-model-capacity.meta.json new file mode 100644 index 00000000000..9e6f7d8c578 --- /dev/null +++ b/src/main/runtime/__fixtures__/codex-model-capacity.meta.json @@ -0,0 +1,31 @@ +{ + "capturedAt": "2026-10-13T07:19:49.742Z", + "platform": "darwin", + "command": [ + "codex", + "--no-alt-screen", + "--dangerously-bypass-approvals-and-sandbox", + "-c", + "model=\"gpt-5\"", + "-c", + "model_provider=\"capacity_fixture\"", + "-c", + "model_providers.capacity_fixture.name=\"Capacity fixture\"", + "-c", + "model_providers.capacity_fixture.base_url=\"http://127.0.0.1:18791/v1\"", + "-c", + "model_providers.capacity_fixture.wire_api=\"responses\"", + "-c", + "model_providers.capacity_fixture.request_max_retries=0", + "-c", + "model_providers.capacity_fixture.stream_max_retries=0", + "-c", + "model_providers.capacity_fixture.supports_websockets=false", + "-c", + "check_for_update_on_startup=false" + ], + "cols": 80, + "rows": 24, + "note": "Real codex-cli 0.162.1 with a temporary CODEX_HOME, local HTTP 503 server_is_overloaded; no production request.", + "exitCode": 0 +} diff --git a/src/main/runtime/__fixtures__/codex-model-capacity.txt b/src/main/runtime/__fixtures__/codex-model-capacity.txt new file mode 100644 index 00000000000..924a2dcd9be --- /dev/null +++ b/src/main/runtime/__fixtures__/codex-model-capacity.txt @@ -0,0 +1,14 @@ +[?2004h[>4;0m[>7u[?1004h]10;?\]11;?\[?u[?2026h[?2026h[?25l  >_ OpenAI Codex (v0.162.1) loading  › Ask Codex to do anything  [0 q [?25h[?2026l[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25l~/orca/workspaces/orca/detect-at-capcity-codex[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25l Folder access/Users/xxxxxxxx/orca/workspaces/orca/detect-at-capcity-codex Note: You’re in a subdirectory of a Git project. Trusting will apply to therepository root:/Users/xxxxxxxx/projects/orca Trustthisfolder?Codexcanread,edit,andrunfileshere,subjecttoyourpermissionsettings.Foldersettingscanruncodeautomatically,evenwithoutamodelrequest.Continueonlyifyoutrustthesefiles.Yourtrustdecisionwillbesaved.› 1. Trust and continue 2.Quitenter continue · esc quit[?2026l[?2026h[?25l  >_ OpenAI Codex (v0.162.1) ~/orca/workspaces/orca/detect-at-capcity-codex  › Ask Codex to do anything  [0 q [?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25h[?2026l]0;detect-at-capcity-codex[?2026h[?25h[?2026l[?2026h[?25l  >_ OpenAI Codex (v0.162.1) ~/orca/workspaces/orca/detect-at-capcity-codex permissions: YOLO mode  › Ask Codex to do anything  gpt-5 default · ~/orca/workspaces/orca/detect-at-capcity-codex[0 q [?25h[?2026l[?2026h[?25l… ⚠ 1 warning·f2toview[?25h[?2026l[?2026h[?25h[?2026l]0;⠙ detect-at-capcity-codex]0;detect-at-capcity-codex[?2026hMMMMMMM + + >_ OpenAI Codex (v0.162.1) + ~/orca/workspaces/orca/detect-at-capcity-codex + permissions: YOLO mode + + Tip: Maximize usage with GPT-6.1 Sol. Try it on complex work for near-Astra + performance at a lower cost.[?25l  › Ask Codex to do anything  gpt-5 default · ~/orca/workspaces/orca/detect-at-c… ⚠ 1 warning · f2 to view[0 q [?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25lSay hi[?25h[?2026l[?2026hMMMM + + +› Say hi +[?25lAsk Codex to do anything[0 q [?25h[?2026l[?2026h[?25h[?2026l[?2026h[?25l… ⚠ 2 warnings[?25h[?2026l]0;⠼ detect-at-capcity-codex[?2026h[?25l • Working (0s • esc to interrupt)  › Ask Codex to do anything  gpt-5 default · ~/orca/workspaces/orca/detect-at-… ⚠ 2 warnings · f2 to view[0 q [?25h[?2026l]0;⠴ ⠴ | detect-at-capcity-codex[?2026h[?25h[?2026l]0;⠴ | detect-at-capcity-codex[?2026hMM + +■ Selected model is at capacity. Please try a different model.[?25l  › Ask Codex to do anything  gpt-5 default · ~/orca/workspaces/orca/detect-at-… ⚠ 2 warnings · f2 to view[0 q [?25h[?2026l[?2026h[?25h[?2026l]0;detect-at-capcity-codex[?2026h[?25h[?2026l \ No newline at end of file diff --git a/src/main/runtime/codex-capacity-terminal-status.test.ts b/src/main/runtime/codex-capacity-terminal-status.test.ts new file mode 100644 index 00000000000..c0b8d51ac3a --- /dev/null +++ b/src/main/runtime/codex-capacity-terminal-status.test.ts @@ -0,0 +1,254 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, it, vi } from 'vitest' +import { HeadlessEmulator } from '../daemon/headless-emulator' +import { + CODEX_CAPACITY_MESSAGE, + CodexCapacityNoticeTracker +} from '../../shared/codex-capacity-notice' +import { observeCodexCapacityTerminalStatus } from './codex-capacity-terminal-status' +import * as agentProcessRecognition from '../../shared/agent-process-recognition' +import { createTranscriptPane, TRANSCRIPT_PANE_PTY_ID } from './agent-transcript-pane-test-harness' + +const transcript = readFileSync( + new URL('./__fixtures__/codex-model-capacity.txt', import.meta.url), + 'utf8' +) +const fullscreenTranscript = readFileSync( + new URL('./__fixtures__/codex-model-capacity-fullscreen.txt', import.meta.url), + 'utf8' +) + +describe('Codex model capacity terminal status', () => { + it.each([ + [transcript, 80, 1], + [transcript, 80, 7], + [transcript, 80, 64], + [transcript, 80, 8192], + [fullscreenTranscript, 40, 1], + [fullscreenTranscript, 40, 7], + [fullscreenTranscript, 40, 64], + [fullscreenTranscript, 40, 8192] + ] as const)('reads a captured notice %#', async (recording, cols, size) => { + const emulator = new HeadlessEmulator({ cols, rows: 24 }) + const tracker = new CodexCapacityNoticeTracker() + let detected = false + try { + for (let index = 0; index < recording.length; index += size) { + const chunk = recording.slice(index, index + size) + await emulator.write(chunk) + detected ||= tracker.observe(chunk, () => emulator.getVisibleLines()) + } + expect(detected).toBe(true) + } finally { + emulator.dispose() + } + }) + + it('does not read the screen for ordinary output or OSC string payloads', () => { + const tracker = new CodexCapacityNoticeTracker() + const readScreen = vi.fn(() => [`■ ${CODEX_CAPACITY_MESSAGE}`, '›']) + tracker.observe('x'.repeat(1_000_000), readScreen) + tracker.observe(`\x1b]0;■ ${CODEX_CAPACITY_MESSAGE}\x07`, readScreen) + expect(readScreen).not.toHaveBeenCalled() + }) + + it('ignores prose, old history before a new prompt, and a busy turn', () => { + for (const lines of [ + [`• The error says: ■ ${CODEX_CAPACITY_MESSAGE}`, '›'], + [`■ ${CODEX_CAPACITY_MESSAGE}`, '› continue', '• Working (1s • esc to interrupt)', '›'], + [`■ ${CODEX_CAPACITY_MESSAGE}`, '• Working (1s • esc to interrupt)', '›'] + ]) { + const tracker = new CodexCapacityNoticeTracker() + expect(tracker.observe(`■ ${CODEX_CAPACITY_MESSAGE}`, () => lines)).toBe(false) + } + }) + + it('handles a wrapped native warning and waits for the synchronized frame to finish', () => { + const tracker = new CodexCapacityNoticeTracker() + const screen = vi.fn(() => [ + '⚠ Selected model is at capacity. Please', + ' try a different model.', + '', + '›' + ]) + expect( + tracker.observe( + '\x1b[?2026h⚠ Selected model is at\ncapacity. Please\ntry a different model.', + screen + ) + ).toBe(false) + expect(screen).not.toHaveBeenCalled() + expect(tracker.observe('\x1b[?2026l', screen)).toBe(true) + }) + + it('publishes a failed root through the execution-host terminal status path', async () => { + const onTerminalAgentStatus = vi.fn() + const { runtime } = await createTranscriptPane( + { + paneTitle: 'Codex', + foregroundProcess: 'codex', + launchAgent: 'codex', + data: '' + }, + { onTerminalAgentStatus } + ) + try { + runtime.onPtyData(TRANSCRIPT_PANE_PTY_ID, transcript, 1) + await vi.waitFor(() => + expect(onTerminalAgentStatus).toHaveBeenCalledWith( + expect.objectContaining({ + payload: expect.objectContaining({ + state: 'done', + agentType: 'codex', + lastAssistantMessage: CODEX_CAPACITY_MESSAGE, + mainAgent: expect.objectContaining({ state: 'done', outcome: 'failure' }) + }) + }) + ) + ) + } finally { + await runtime.onPtyExit(TRANSCRIPT_PANE_PTY_ID, 0) + } + }) + + it('does not produce client-side status for an SSH-owned PTY', async () => { + const onTerminalAgentStatus = vi.fn() + const { runtime, handle } = await createTranscriptPane( + { + paneTitle: 'Codex', + foregroundProcess: 'codex', + launchAgent: 'codex', + connectionId: 'ssh-host', + data: '' + }, + { onTerminalAgentStatus } + ) + try { + runtime.onPtyData(TRANSCRIPT_PANE_PTY_ID, transcript, 1) + await runtime.readTerminal(handle) + expect(onTerminalAgentStatus).not.toHaveBeenCalled() + } finally { + await runtime.onPtyExit(TRANSCRIPT_PANE_PTY_ID, 0) + } + }) + + it('ignores unrelated agents', async () => { + const emulator = new HeadlessEmulator({ cols: 80, rows: 24 }) + try { + await emulator.write(transcript) + expect( + observeCodexCapacityTerminalStatus( + emulator, + transcript, + 'claude', + 'claude', + () => undefined + ) + ).toBeNull() + } finally { + emulator.dispose() + } + }) + + it('detects Codex launched inside a shell from its hook identity and skips normal output', async () => { + const emulator = new HeadlessEmulator({ cols: 80, rows: 24 }) + const readStatus = vi.fn(() => ({ agentType: 'codex', prompt: 'Say hi', model: 'gpt-5' })) + try { + expect( + observeCodexCapacityTerminalStatus(emulator, 'normal output', 'zsh', undefined, readStatus) + ).toBeNull() + expect(readStatus).not.toHaveBeenCalled() + await emulator.write(transcript) + expect( + observeCodexCapacityTerminalStatus(emulator, transcript, 'zsh', undefined, readStatus) + ).toMatchObject({ + prompt: 'Say hi', + model: 'gpt-5', + mainAgent: { state: 'done', outcome: 'failure' } + }) + } finally { + emulator.dispose() + } + }) + + it.each([100, 101])( + 'does not close a hook turn starting at or after output time (%i)', + async (turnStartedAt) => { + const emulator = new HeadlessEmulator({ cols: 80, rows: 24 }) + try { + await emulator.write(transcript) + expect( + observeCodexCapacityTerminalStatus( + emulator, + transcript, + undefined, + 'codex', + () => ({ + agentType: 'codex', + prompt: 'Retry', + turnStartedAt + }), + 100 + ) + ).toBeNull() + } finally { + emulator.dispose() + } + } + ) + + it.each([1, 7, 64, 8192])( + 'detects a first shell-launched notice in %i-character chunks', + async (size) => { + const start = fullscreenTranscript.lastIndexOf( + '\x1b[?2026h', + fullscreenTranscript.indexOf('■ Selected model') + ) + const end = fullscreenTranscript.indexOf('\x1b[?2026l', start) + '\x1b[?2026l'.length + const emulator = new HeadlessEmulator({ cols: 40, rows: 24 }) + let detections = 0 + try { + await emulator.write(fullscreenTranscript.slice(0, start)) + for (let index = start; index < end; index += size) { + const chunk = fullscreenTranscript.slice(index, Math.min(index + size, end)) + await emulator.write(chunk) + if ( + observeCodexCapacityTerminalStatus(emulator, chunk, 'zsh', undefined, () => ({ + agentType: 'codex', + prompt: 'Say hi' + })) + ) { + detections++ + } + } + expect(detections).toBe(1) + } finally { + emulator.dispose() + } + } + ) + + it('recognizes an unrelated launch once and invalidates when its command changes', () => { + const emulator = new HeadlessEmulator({ cols: 80, rows: 24 }) + const recognize = vi.spyOn(agentProcessRecognition, 'recognizeAgentProcessFromCommandLine') + const readStatus = vi.fn(() => undefined) + try { + for (let index = 0; index < 1000; index++) { + observeCodexCapacityTerminalStatus( + emulator, + 'ordinary output', + 'zsh', + undefined, + readStatus + ) + } + expect(recognize).toHaveBeenCalledTimes(1) + observeCodexCapacityTerminalStatus(emulator, 'ordinary output', 'bash', undefined, readStatus) + expect(recognize).toHaveBeenCalledTimes(2) + expect(readStatus).not.toHaveBeenCalled() + } finally { + recognize.mockRestore() + emulator.dispose() + } + }) +}) diff --git a/src/main/runtime/codex-capacity-terminal-status.ts b/src/main/runtime/codex-capacity-terminal-status.ts new file mode 100644 index 00000000000..acb984fc337 --- /dev/null +++ b/src/main/runtime/codex-capacity-terminal-status.ts @@ -0,0 +1,79 @@ +import { + CodexCapacityNoticeTracker, + CODEX_CAPACITY_MESSAGE +} from '../../shared/codex-capacity-notice' +import type { + AgentStatusIpcPayload, + ParsedAgentStatusPayload +} from '../../shared/agent-status-types' +import { recognizeAgentProcessFromCommandLine } from '../../shared/agent-process-recognition' +import type { HeadlessEmulator } from '../daemon/headless-emulator' + +const trackers = new WeakMap() +const launchIdentities = new WeakMap() +type CodexCapacityStatus = Pick< + AgentStatusIpcPayload, + 'agentType' | 'prompt' | 'model' | 'subagents' | 'mainAgent' | 'turnStartedAt' +> + +export function observeCodexCapacityTerminalStatus( + emulator: HeadlessEmulator, + data: string, + startupCommand: string | undefined, + launchAgent: string | null | undefined, + readStatus: () => CodexCapacityStatus | undefined, + outputObservedAt = Date.now() +): ParsedAgentStatusPayload | null { + let tracker = trackers.get(emulator) + if (!tracker && launchAgent !== 'codex') { + let identity = launchIdentities.get(emulator) + if (!identity || identity.command !== startupCommand) { + identity = { + command: startupCommand, + codex: recognizeAgentProcessFromCommandLine(startupCommand)?.agent === 'codex' + } + launchIdentities.set(emulator, identity) + } + // A Codex started from an existing shell is identified by its host-owned hook row. + if ( + !identity.codex && + ((!data.includes('■') && !data.includes('⚠')) || readStatus()?.agentType !== 'codex') + ) { + return null + } + } + if (!tracker) { + tracker = new CodexCapacityNoticeTracker() + trackers.set(emulator, tracker) + } + if ( + !tracker.observe( + data, + () => emulator.getVisibleLines(), + () => emulator.isSynchronizedOutput + ) + ) { + return null + } + const previous = readStatus() + if (previous && previous.agentType !== 'codex') { + return null + } + // Same-tick ordering is ambiguous; queued output must yield to a newer hook turn. + if (previous?.turnStartedAt !== undefined && previous.turnStartedAt >= outputObservedAt) { + return null + } + if (previous?.mainAgent?.outcome === 'failure') { + return null + } + return { + state: 'done', + agentType: 'codex', + prompt: previous?.prompt ?? '', + model: previous?.model, + subagents: previous?.subagents, + lastAssistantMessage: CODEX_CAPACITY_MESSAGE, + lastAssistantMessageIsToolOutput: true, + mainAgent: { state: 'done', outcome: 'failure', stateStartedAt: Date.now() } + } +} diff --git a/src/main/runtime/orca-runtime-maybe-hydrate-headless-from-renderer.ts b/src/main/runtime/orca-runtime-maybe-hydrate-headless-from-renderer.ts index 1cab3801ffe..18cdfb7371c 100644 --- a/src/main/runtime/orca-runtime-maybe-hydrate-headless-from-renderer.ts +++ b/src/main/runtime/orca-runtime-maybe-hydrate-headless-from-renderer.ts @@ -2,6 +2,7 @@ import { splitFreebuffScreenUpdates } from '../../shared/freebuff-screen-status' import { OrcaRuntimeWithSerializeMainTerminalBuffer } from './orca-runtime-serialize-main-terminal-buffer' import { observeFreebuffTerminalStatus } from './freebuff-terminal-status' +import { observeCodexCapacityTerminalStatus } from './codex-capacity-terminal-status' import { MOBILE_SUBSCRIBE_SCROLLBACK_ROWS } from './scrollback-limits' import { detectAgentStatusFromTitle, normalizeTerminalTitle } from '../../shared/agent-detection' import { shouldModelAnswerHiddenPtyQueries } from './terminal-model-query-authority' @@ -173,6 +174,7 @@ export class OrcaRuntimeWithMaybeHydrateHeadlessFromRenderer extends OrcaRuntime forwardQueryReplies = false ): void { const state = this.getOrCreateHeadlessTerminal(ptyId) + const outputObservedAt = Date.now() const completion = state.writeChain.then(async () => { // Why: the ingestion-time ownership decision is closed over this // chain link; async scheduling cannot retroactively change it. @@ -183,12 +185,21 @@ export class OrcaRuntimeWithMaybeHydrateHeadlessFromRenderer extends OrcaRuntime await state.emulator.write(chunk, { forwardQueryReplies }) const pty = this.ptysById.get(ptyId) if (pty && !pty.connectionId && this.headlessTerminals.get(ptyId) === state) { - const payload = observeFreebuffTerminalStatus( - state.emulator, - chunk, - this.terminalSpawnCommandsByPtyId.get(ptyId), - pty.launchAgent - ) + const command = this.terminalSpawnCommandsByPtyId.get(ptyId) + const payload = + observeCodexCapacityTerminalStatus( + state.emulator, + chunk, + command, + pty.launchAgent, + () => { + const paneKey = + this.getLeavesForPty(ptyId).map((leaf) => this.makeRuntimePaneKey(leaf))[0] ?? + pty.paneKey + return paneKey ? this.getAgentStatusSnapshotForPaneFn?.(paneKey)[0] : undefined + }, + outputObservedAt + ) ?? observeFreebuffTerminalStatus(state.emulator, chunk, command, pty.launchAgent) if (payload) { pty.lastExplicitAgentStatus = { state: payload.state, diff --git a/src/shared/agent-terminal-status-equivalence.ts b/src/shared/agent-terminal-status-equivalence.ts index d80eab80f34..f30eec0d032 100644 --- a/src/shared/agent-terminal-status-equivalence.ts +++ b/src/shared/agent-terminal-status-equivalence.ts @@ -1,4 +1,4 @@ -import type { ParsedAgentStatusPayload } from './agent-status-types' +import { mainAgentStatusEqual, type ParsedAgentStatusPayload } from './agent-status-types' /** Whether a terminal status is the same hook status with only hook-owned metadata omitted. * @@ -13,6 +13,8 @@ export function terminalStatusPayloadMatchesHook( // OSC never carries model/children; compare only fields its protocol owns. return ( cachedHookPayload.state === terminalPayload.state && + (terminalPayload.mainAgent === undefined || + mainAgentStatusEqual(cachedHookPayload.mainAgent, terminalPayload.mainAgent)) && // OSC never carries workingMode either; only a stated mismatch breaks equivalence. (terminalPayload.workingMode === undefined || cachedHookPayload.workingMode === terminalPayload.workingMode) && diff --git a/src/shared/codex-capacity-notice.test.ts b/src/shared/codex-capacity-notice.test.ts new file mode 100644 index 00000000000..011c0780991 --- /dev/null +++ b/src/shared/codex-capacity-notice.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it, vi } from 'vitest' +import { CODEX_CAPACITY_MESSAGE, CodexCapacityNoticeTracker } from './codex-capacity-notice' + +describe('Codex capacity candidate bounds', () => { + it('reads neither the screen nor its mode during ordinary output', () => { + const tracker = new CodexCapacityNoticeTracker() + const readScreen = vi.fn(() => []) + const readSynchronized = vi.fn(() => false) + tracker.observe('ordinary output'.repeat(1000), readScreen, readSynchronized) + expect(readScreen).not.toHaveBeenCalled() + expect(readSynchronized).not.toHaveBeenCalled() + }) + it('does not inspect a partial synchronized frame with combined ANSI modes', () => { + const tracker = new CodexCapacityNoticeTracker() + const readScreen = vi.fn(() => [`■ ${CODEX_CAPACITY_MESSAGE}`, '›']) + expect(tracker.observe(`\x1b[?25;2026h■ ${CODEX_CAPACITY_MESSAGE}`, readScreen)).toBe(false) + expect(readScreen).not.toHaveBeenCalled() + expect(tracker.observe('\x1b[?2026;25l', readScreen)).toBe(true) + expect(readScreen).toHaveBeenCalledTimes(1) + }) + it('ignores repeated unrelated markers and still accepts a later native notice', () => { + const tracker = new CodexCapacityNoticeTracker() + const readScreen = vi.fn(() => [`■ ${CODEX_CAPACITY_MESSAGE}`, '›']) + expect(tracker.observe('■'.repeat(100_000), readScreen)).toBe(false) + expect(readScreen).not.toHaveBeenCalled() + expect(tracker.observe(`■ ${CODEX_CAPACITY_MESSAGE}`, readScreen)).toBe(true) + expect(readScreen).toHaveBeenCalledTimes(1) + }) + it('finds the notice inside a large text chunk after unrelated warning markers', () => { + const tracker = new CodexCapacityNoticeTracker() + const readScreen = vi.fn(() => [`■ ${CODEX_CAPACITY_MESSAGE}`, '›']) + expect( + tracker.observe( + `⚠ unrelated\n${'x'.repeat(1024 * 1024)}\n■ ${CODEX_CAPACITY_MESSAGE}`, + readScreen + ) + ).toBe(true) + expect(readScreen).toHaveBeenCalledTimes(1) + }) + + it('recognizes a split notice immediately after abandoning an unrelated warning', () => { + const tracker = new CodexCapacityNoticeTracker() + const readScreen = vi.fn(() => [`⚠ ${CODEX_CAPACITY_MESSAGE}`, '›']) + expect(tracker.observe('⚠ 1 warning · f2 to view', readScreen)).toBe(false) + expect(readScreen).not.toHaveBeenCalled() + expect(tracker.observe('⚠ Selected model is ', readScreen)).toBe(false) + expect(tracker.observe('at capacity. Please try a different model.', readScreen)).toBe(true) + expect(tracker.observe('ordinary output'.repeat(100_000), readScreen)).toBe(false) + expect(readScreen).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/shared/codex-capacity-notice.ts b/src/shared/codex-capacity-notice.ts new file mode 100644 index 00000000000..c0aa3ec6787 --- /dev/null +++ b/src/shared/codex-capacity-notice.ts @@ -0,0 +1,87 @@ +import { createTerminalEscapeScanner } from './terminal-escape-scanner' + +export const CODEX_CAPACITY_MESSAGE = 'Selected model is at capacity. Please try a different model.' + +const NOTICE = /^[■⚠]\s+Selected model is at capacity\. Please try a different model\.$/ +const MAX_NOTICE_LENGTH = 160 + +/** Only a newly painted native notice at the end of the conversation can close a turn. */ +export class CodexCapacityNoticeTracker { + private tail = '' + private pending = false + private synchronized = false + private readonly scanner = createTerminalEscapeScanner({ + onCsi: (marker, params, final) => { + if ( + marker === '?' && + params.split(';').includes('2026') && + (final === 'h' || final === 'l') + ) { + this.synchronized = final === 'h' + } else if ((final === 'H' || final === 'f') && this.tail && !this.tail.endsWith(' ')) { + this.tail += ' ' + } + }, + onText: (text) => this.observeText(text) + }) + + private observeText(text: string): void { + if (this.tail) { + this.updateCandidate(this.tail + text.slice(0, MAX_NOTICE_LENGTH)) + } + let error = text.indexOf('■') + let warning = text.indexOf('⚠') + while (error !== -1 || warning !== -1) { + const start = error === -1 ? warning : warning === -1 ? error : Math.min(error, warning) + this.updateCandidate(text.slice(start, start + MAX_NOTICE_LENGTH)) + if (start === error) { + error = text.indexOf('■', start + 1) + } else { + warning = text.indexOf('⚠', start + 1) + } + } + } + + private updateCandidate(text: string): void { + const content = text.slice(1).trimStart() + if (content && content[0] !== CODEX_CAPACITY_MESSAGE[0]) { + this.tail = '' + return + } + const candidate = text.replace(/\s+/g, ' ') + const message = candidate.slice(1).trimStart() + if (message.startsWith(CODEX_CAPACITY_MESSAGE)) { + this.pending = true + this.tail = '' + } else { + this.tail = CODEX_CAPACITY_MESSAGE.startsWith(message) ? candidate : '' + } + } + + observe( + data: string, + readScreen: () => readonly string[], + readSynchronized?: () => boolean + ): boolean { + this.scanner.observe(data) + if (!this.pending || (readSynchronized?.() ?? this.synchronized)) { + return false + } + this.pending = false + this.tail = '' + const lines = readScreen() + const composer = lines.findLastIndex((line) => /^\s*›/.test(line)) + if (composer === -1) { + return false + } + const above = lines.slice(0, composer).filter((line) => line.trim() !== '') + let notice = '' + for (let index = above.length - 1; index >= 0 && notice.length < MAX_NOTICE_LENGTH; index--) { + notice = `${above[index].trim()} ${notice}`.trim() + if (NOTICE.test(notice)) { + return true + } + } + return false + } +}