mirror of
https://github.com/stablyai/orca.git
synced 2026-10-23 08:03:39 +00:00
fix structured notification classification
This commit is contained in:
@@ -0,0 +1,78 @@
|
||||
// Stable ServerNotification method discriminators generated by codex-cli 0.147.0.
|
||||
export const CODEX_APP_SERVER_NOTIFICATION_METHODS = [
|
||||
'error',
|
||||
'thread/started',
|
||||
'thread/status/changed',
|
||||
'thread/archived',
|
||||
'thread/deleted',
|
||||
'thread/unarchived',
|
||||
'thread/closed',
|
||||
'skills/changed',
|
||||
'thread/name/updated',
|
||||
'thread/goal/updated',
|
||||
'thread/goal/cleared',
|
||||
'thread/environment/connected',
|
||||
'thread/environment/disconnected',
|
||||
'thread/settings/updated',
|
||||
'thread/tokenUsage/updated',
|
||||
'turn/started',
|
||||
'hook/started',
|
||||
'turn/completed',
|
||||
'hook/completed',
|
||||
'turn/diff/updated',
|
||||
'turn/plan/updated',
|
||||
'item/started',
|
||||
'item/autoApprovalReview/started',
|
||||
'item/autoApprovalReview/completed',
|
||||
'item/completed',
|
||||
'rawResponseItem/completed',
|
||||
'rawResponse/completed',
|
||||
'item/agentMessage/delta',
|
||||
'item/plan/delta',
|
||||
'command/exec/outputDelta',
|
||||
'process/outputDelta',
|
||||
'process/exited',
|
||||
'item/commandExecution/outputDelta',
|
||||
'item/commandExecution/terminalInteraction',
|
||||
'item/fileChange/outputDelta',
|
||||
'item/fileChange/patchUpdated',
|
||||
'serverRequest/resolved',
|
||||
'item/mcpToolCall/progress',
|
||||
'mcpServer/oauthLogin/completed',
|
||||
'mcpServer/startupStatus/updated',
|
||||
'account/updated',
|
||||
'account/rateLimits/updated',
|
||||
'app/list/updated',
|
||||
'remoteControl/status/changed',
|
||||
'externalAgentConfig/import/progress',
|
||||
'externalAgentConfig/import/completed',
|
||||
'fs/changed',
|
||||
'item/reasoning/summaryTextDelta',
|
||||
'item/reasoning/summaryPartAdded',
|
||||
'item/reasoning/textDelta',
|
||||
'thread/compacted',
|
||||
'model/rerouted',
|
||||
'model/verification',
|
||||
'turn/moderationMetadata',
|
||||
'model/safetyBuffering/updated',
|
||||
'warning',
|
||||
'guardianWarning',
|
||||
'deprecationNotice',
|
||||
'configWarning',
|
||||
'fuzzyFileSearch/sessionUpdated',
|
||||
'fuzzyFileSearch/sessionCompleted',
|
||||
'thread/realtime/started',
|
||||
'thread/realtime/itemAdded',
|
||||
'thread/realtime/transcript/delta',
|
||||
'thread/realtime/transcript/done',
|
||||
'thread/realtime/outputAudio/delta',
|
||||
'thread/realtime/sdp',
|
||||
'thread/realtime/error',
|
||||
'thread/realtime/closed',
|
||||
'windows/worldWritableWarning',
|
||||
'windowsSandbox/setupCompleted',
|
||||
'account/login/completed'
|
||||
] as const
|
||||
|
||||
export type CodexAppServerNotificationMethod =
|
||||
(typeof CODEX_APP_SERVER_NOTIFICATION_METHODS)[number]
|
||||
@@ -469,6 +469,43 @@ describe('codex journal translation', () => {
|
||||
expect(timeline).toEqual([])
|
||||
})
|
||||
|
||||
it('projects only user and assistant content for a complete turn with hooks', () => {
|
||||
const { translator, tap } = translatorWith()
|
||||
|
||||
translator.handle(notification('thread/started', { thread: { id: THREAD_ID } }))
|
||||
translator.handle(notification('hook/started', { run: { id: 'hook-1', status: 'running' } }))
|
||||
translator.handle(notification('account/rateLimits/updated', { rateLimits: { primary: null } }))
|
||||
translator.handle(TURN_STARTED)
|
||||
translator.handle(
|
||||
notification('item/completed', {
|
||||
item: { type: 'userMessage', id: 'item-0', text: 'hi' }
|
||||
})
|
||||
)
|
||||
translator.handle(
|
||||
notification('hook/completed', { run: { id: 'hook-1', status: 'completed' } })
|
||||
)
|
||||
translator.handle(
|
||||
notification('item/completed', {
|
||||
item: { type: 'agentMessage', id: 'item-1', text: 'hello' }
|
||||
})
|
||||
)
|
||||
translator.handle(notification('turn/completed', { turn: { id: TURN_ID } }))
|
||||
|
||||
const timeline = projectStructuredItemsToNativeChat(
|
||||
tap.rows.map((row, index) => ({
|
||||
itemId: row.key,
|
||||
revision: 1,
|
||||
sequence: index + 1,
|
||||
observedAt: index + 1,
|
||||
body: row.body
|
||||
}))
|
||||
)
|
||||
expect(timeline.map(({ role, blocks }) => ({ role, blocks }))).toEqual([
|
||||
{ role: 'user', blocks: [{ type: 'text', text: 'hi' }] },
|
||||
{ role: 'assistant', blocks: [{ type: 'text', text: 'hello' }] }
|
||||
])
|
||||
})
|
||||
|
||||
it('renders a system error carried by a suppressed status kind', () => {
|
||||
const { translator, tap } = translatorWith()
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
// SDKMessage discriminators from Claude Agent SDK 0.3.231 / Claude Code 2.1.231.
|
||||
export const CLAUDE_STREAM_JSON_FRAME_KINDS = [
|
||||
'message:assistant',
|
||||
'message:user',
|
||||
'message:result',
|
||||
'message:system:init',
|
||||
'message:stream_event:message_start',
|
||||
'message:stream_event:message_delta',
|
||||
'message:stream_event:message_stop',
|
||||
'message:stream_event:content_block_start',
|
||||
'message:stream_event:content_block_delta',
|
||||
'message:stream_event:content_block_stop',
|
||||
'message:system:compact_boundary',
|
||||
'message:system:status',
|
||||
'message:system:api_retry',
|
||||
'message:system:control_request_progress',
|
||||
'message:system:model_refusal_fallback',
|
||||
'message:system:model_refusal_no_fallback',
|
||||
'message:system:local_command_output',
|
||||
'message:system:hook_started',
|
||||
'message:system:hook_progress',
|
||||
'message:system:hook_response',
|
||||
'message:system:plugin_install',
|
||||
'message:tool_progress',
|
||||
'message:auth_status',
|
||||
'message:system:task_notification',
|
||||
'message:system:task_started',
|
||||
'message:system:task_updated',
|
||||
'message:system:task_progress',
|
||||
'message:system:background_tasks_changed',
|
||||
'message:system:thinking_tokens',
|
||||
'message:system:session_state_changed',
|
||||
'message:system:worker_shutting_down',
|
||||
'message:system:commands_changed',
|
||||
'message:system:notification',
|
||||
'message:system:files_persisted',
|
||||
'message:tool_use_summary',
|
||||
'message:system:memory_recall',
|
||||
'message:rate_limit_event',
|
||||
'message:system:elicitation_complete',
|
||||
'message:system:permission_denied',
|
||||
'message:prompt_suggestion',
|
||||
'message:system:mirror_error',
|
||||
'message:system:informational',
|
||||
'message:conversation_reset'
|
||||
] as const
|
||||
|
||||
export type ClaudeStreamJsonFrameKind = (typeof CLAUDE_STREAM_JSON_FRAME_KINDS)[number]
|
||||
@@ -0,0 +1,54 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { CODEX_APP_SERVER_NOTIFICATION_METHODS } from '../../codex/codex-app-server-notification-schema'
|
||||
import { CLAUDE_STREAM_JSON_FRAME_KINDS } from './claude-stream-json-frame-schema'
|
||||
import { classifyProviderFrame, PROVIDER_FRAME_CLASSIFICATIONS } from './provider-frame-disposition'
|
||||
|
||||
describe('provider frame classification catalog', () => {
|
||||
it('classifies every pinned Codex app-server notification method', () => {
|
||||
expect(Object.keys(PROVIDER_FRAME_CLASSIFICATIONS.codex)).toEqual([
|
||||
...CODEX_APP_SERVER_NOTIFICATION_METHODS
|
||||
])
|
||||
})
|
||||
|
||||
it('classifies every pinned Claude stream-json frame kind', () => {
|
||||
expect(Object.keys(PROVIDER_FRAME_CLASSIFICATIONS.claude)).toEqual([
|
||||
...CLAUDE_STREAM_JSON_FRAME_KINDS
|
||||
])
|
||||
})
|
||||
|
||||
it('suppresses benign hook lifecycle and Codex rate-limit frames', () => {
|
||||
expect(classifyProviderFrame('codex', 'notification:hook/started', {})).toBe(
|
||||
'suppressed-benign'
|
||||
)
|
||||
expect(classifyProviderFrame('codex', 'notification:hook/completed', {})).toBe(
|
||||
'suppressed-benign'
|
||||
)
|
||||
expect(classifyProviderFrame('codex', 'notification:account/rateLimits/updated', {})).toBe(
|
||||
'suppressed-benign'
|
||||
)
|
||||
expect(classifyProviderFrame('claude', 'message:system:hook_started', {})).toBe(
|
||||
'suppressed-benign'
|
||||
)
|
||||
})
|
||||
|
||||
it('promotes payload failures over a benign catalog classification', () => {
|
||||
expect(
|
||||
classifyProviderFrame('codex', 'notification:hook/completed', {
|
||||
run: { status: 'failed' }
|
||||
})
|
||||
).toBe('error-surface')
|
||||
expect(
|
||||
classifyProviderFrame('claude', 'message:system:hook_response', {
|
||||
outcome: 'error',
|
||||
stderr: 'hook failed'
|
||||
})
|
||||
).toBe('error-surface')
|
||||
})
|
||||
|
||||
it('keeps unknown future frames on the substantive bounded fallback path', () => {
|
||||
expect(classifyProviderFrame('codex', 'notification:future/event', {})).toBe(
|
||||
'timeline-substantive'
|
||||
)
|
||||
expect(classifyProviderFrame('claude', 'message:future_event', {})).toBe('timeline-substantive')
|
||||
})
|
||||
})
|
||||
@@ -1,4 +1,138 @@
|
||||
export type ProviderFrameDisposition = 'debug-only' | 'timeline'
|
||||
import type { CodexAppServerNotificationMethod } from '../../codex/codex-app-server-notification-schema'
|
||||
import type { ClaudeStreamJsonFrameKind } from './claude-stream-json-frame-schema'
|
||||
|
||||
export type ProviderFrameClassification =
|
||||
| 'timeline-substantive'
|
||||
| 'status-chrome'
|
||||
| 'suppressed-benign'
|
||||
| 'error-surface'
|
||||
|
||||
type ProviderFrameClassificationTable = {
|
||||
codex: Record<CodexAppServerNotificationMethod, ProviderFrameClassification>
|
||||
claude: Record<ClaudeStreamJsonFrameKind, ProviderFrameClassification>
|
||||
}
|
||||
|
||||
export const PROVIDER_FRAME_CLASSIFICATIONS = {
|
||||
codex: {
|
||||
error: 'error-surface',
|
||||
'thread/started': 'status-chrome',
|
||||
'thread/status/changed': 'status-chrome',
|
||||
'thread/archived': 'status-chrome',
|
||||
'thread/deleted': 'status-chrome',
|
||||
'thread/unarchived': 'status-chrome',
|
||||
'thread/closed': 'status-chrome',
|
||||
'skills/changed': 'status-chrome',
|
||||
'thread/name/updated': 'status-chrome',
|
||||
'thread/goal/updated': 'status-chrome',
|
||||
'thread/goal/cleared': 'status-chrome',
|
||||
'thread/environment/connected': 'status-chrome',
|
||||
'thread/environment/disconnected': 'status-chrome',
|
||||
'thread/settings/updated': 'status-chrome',
|
||||
'thread/tokenUsage/updated': 'status-chrome',
|
||||
'turn/started': 'status-chrome',
|
||||
'hook/started': 'suppressed-benign',
|
||||
'turn/completed': 'status-chrome',
|
||||
'hook/completed': 'suppressed-benign',
|
||||
'turn/diff/updated': 'timeline-substantive',
|
||||
'turn/plan/updated': 'timeline-substantive',
|
||||
'item/started': 'timeline-substantive',
|
||||
'item/autoApprovalReview/started': 'status-chrome',
|
||||
'item/autoApprovalReview/completed': 'status-chrome',
|
||||
'item/completed': 'timeline-substantive',
|
||||
'rawResponseItem/completed': 'suppressed-benign',
|
||||
'rawResponse/completed': 'suppressed-benign',
|
||||
'item/agentMessage/delta': 'timeline-substantive',
|
||||
'item/plan/delta': 'timeline-substantive',
|
||||
'command/exec/outputDelta': 'timeline-substantive',
|
||||
'process/outputDelta': 'timeline-substantive',
|
||||
'process/exited': 'timeline-substantive',
|
||||
'item/commandExecution/outputDelta': 'timeline-substantive',
|
||||
'item/commandExecution/terminalInteraction': 'timeline-substantive',
|
||||
'item/fileChange/outputDelta': 'timeline-substantive',
|
||||
'item/fileChange/patchUpdated': 'timeline-substantive',
|
||||
'serverRequest/resolved': 'suppressed-benign',
|
||||
'item/mcpToolCall/progress': 'status-chrome',
|
||||
'mcpServer/oauthLogin/completed': 'status-chrome',
|
||||
'mcpServer/startupStatus/updated': 'status-chrome',
|
||||
'account/updated': 'status-chrome',
|
||||
'account/rateLimits/updated': 'suppressed-benign',
|
||||
'app/list/updated': 'status-chrome',
|
||||
'remoteControl/status/changed': 'status-chrome',
|
||||
'externalAgentConfig/import/progress': 'status-chrome',
|
||||
'externalAgentConfig/import/completed': 'status-chrome',
|
||||
'fs/changed': 'suppressed-benign',
|
||||
'item/reasoning/summaryTextDelta': 'timeline-substantive',
|
||||
'item/reasoning/summaryPartAdded': 'timeline-substantive',
|
||||
'item/reasoning/textDelta': 'timeline-substantive',
|
||||
'thread/compacted': 'status-chrome',
|
||||
'model/rerouted': 'status-chrome',
|
||||
'model/verification': 'status-chrome',
|
||||
'turn/moderationMetadata': 'suppressed-benign',
|
||||
'model/safetyBuffering/updated': 'status-chrome',
|
||||
warning: 'error-surface',
|
||||
guardianWarning: 'error-surface',
|
||||
deprecationNotice: 'error-surface',
|
||||
configWarning: 'error-surface',
|
||||
'fuzzyFileSearch/sessionUpdated': 'suppressed-benign',
|
||||
'fuzzyFileSearch/sessionCompleted': 'suppressed-benign',
|
||||
'thread/realtime/started': 'status-chrome',
|
||||
'thread/realtime/itemAdded': 'timeline-substantive',
|
||||
'thread/realtime/transcript/delta': 'timeline-substantive',
|
||||
'thread/realtime/transcript/done': 'timeline-substantive',
|
||||
'thread/realtime/outputAudio/delta': 'timeline-substantive',
|
||||
'thread/realtime/sdp': 'suppressed-benign',
|
||||
'thread/realtime/error': 'error-surface',
|
||||
'thread/realtime/closed': 'status-chrome',
|
||||
'windows/worldWritableWarning': 'error-surface',
|
||||
'windowsSandbox/setupCompleted': 'status-chrome',
|
||||
'account/login/completed': 'status-chrome'
|
||||
},
|
||||
claude: {
|
||||
'message:assistant': 'timeline-substantive',
|
||||
'message:user': 'timeline-substantive',
|
||||
'message:result': 'status-chrome',
|
||||
'message:system:init': 'status-chrome',
|
||||
'message:stream_event:message_start': 'status-chrome',
|
||||
'message:stream_event:message_delta': 'status-chrome',
|
||||
'message:stream_event:message_stop': 'status-chrome',
|
||||
'message:stream_event:content_block_start': 'status-chrome',
|
||||
'message:stream_event:content_block_delta': 'timeline-substantive',
|
||||
'message:stream_event:content_block_stop': 'status-chrome',
|
||||
'message:system:compact_boundary': 'status-chrome',
|
||||
'message:system:status': 'status-chrome',
|
||||
'message:system:api_retry': 'status-chrome',
|
||||
'message:system:control_request_progress': 'status-chrome',
|
||||
'message:system:model_refusal_fallback': 'status-chrome',
|
||||
'message:system:model_refusal_no_fallback': 'error-surface',
|
||||
'message:system:local_command_output': 'timeline-substantive',
|
||||
'message:system:hook_started': 'suppressed-benign',
|
||||
'message:system:hook_progress': 'suppressed-benign',
|
||||
'message:system:hook_response': 'suppressed-benign',
|
||||
'message:system:plugin_install': 'status-chrome',
|
||||
'message:tool_progress': 'status-chrome',
|
||||
'message:auth_status': 'status-chrome',
|
||||
'message:system:task_notification': 'status-chrome',
|
||||
'message:system:task_started': 'status-chrome',
|
||||
'message:system:task_updated': 'status-chrome',
|
||||
'message:system:task_progress': 'status-chrome',
|
||||
'message:system:background_tasks_changed': 'status-chrome',
|
||||
'message:system:thinking_tokens': 'status-chrome',
|
||||
'message:system:session_state_changed': 'status-chrome',
|
||||
'message:system:worker_shutting_down': 'status-chrome',
|
||||
'message:system:commands_changed': 'status-chrome',
|
||||
'message:system:notification': 'status-chrome',
|
||||
'message:system:files_persisted': 'status-chrome',
|
||||
'message:tool_use_summary': 'timeline-substantive',
|
||||
'message:system:memory_recall': 'timeline-substantive',
|
||||
'message:rate_limit_event': 'status-chrome',
|
||||
'message:system:elicitation_complete': 'status-chrome',
|
||||
'message:system:permission_denied': 'error-surface',
|
||||
'message:prompt_suggestion': 'status-chrome',
|
||||
'message:system:mirror_error': 'error-surface',
|
||||
'message:system:informational': 'timeline-substantive',
|
||||
'message:conversation_reset': 'status-chrome'
|
||||
}
|
||||
} as const satisfies ProviderFrameClassificationTable
|
||||
|
||||
const ERROR_VARIANT_KEYS = new Set(['type', 'status', 'state', 'subtype', 'outcome'])
|
||||
const ERROR_VALUE_KEYS = new Set(['error', 'failureReason', 'failure_reason'])
|
||||
@@ -49,49 +183,35 @@ function notificationKind(kind: string): string {
|
||||
return kind.startsWith('notification:') ? kind.slice('notification:'.length) : kind
|
||||
}
|
||||
|
||||
function isCodexStatusFrame(kind: string): boolean {
|
||||
const method = notificationKind(kind)
|
||||
return (
|
||||
/^(?:thread\/(?:started|closed|archived|unarchived)|thread\/(?:name|status)\/(?:changed|updated))$/.test(
|
||||
method
|
||||
) ||
|
||||
/^thread\/(?:tokenUsage\/updated|goal\/(?:updated|cleared))$/.test(method) ||
|
||||
method === 'mcpServer/startupStatus/updated' ||
|
||||
method === 'remoteControl/status/changed'
|
||||
)
|
||||
}
|
||||
|
||||
function isClaudeStatusFrame(kind: string): boolean {
|
||||
return (
|
||||
kind === 'message:system:init' ||
|
||||
kind === 'message:system:status' ||
|
||||
kind === 'message:result' ||
|
||||
/^message:stream_event:(?:message_start|message_stop|content_block_start|content_block_stop)$/.test(
|
||||
kind
|
||||
)
|
||||
)
|
||||
function catalogClassification(
|
||||
provider: string,
|
||||
kind: string
|
||||
): ProviderFrameClassification | undefined {
|
||||
if (provider === 'codex') {
|
||||
return PROVIDER_FRAME_CLASSIFICATIONS.codex[
|
||||
notificationKind(kind) as CodexAppServerNotificationMethod
|
||||
]
|
||||
}
|
||||
if (provider === 'claude') {
|
||||
return PROVIDER_FRAME_CLASSIFICATIONS.claude[kind as ClaudeStreamJsonFrameKind]
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
export function classifyProviderFrame(
|
||||
provider: string,
|
||||
kind: string,
|
||||
payload: unknown
|
||||
): ProviderFrameDisposition {
|
||||
): ProviderFrameClassification {
|
||||
if (hasProviderError(payload)) {
|
||||
return 'timeline'
|
||||
return 'error-surface'
|
||||
}
|
||||
if (provider === 'codex' && isCodexStatusFrame(kind)) {
|
||||
return 'debug-only'
|
||||
if (provider === 'claude' && kind === 'message:result') {
|
||||
const subtype =
|
||||
typeof payload === 'object' && payload !== null
|
||||
? (payload as Record<string, unknown>).subtype
|
||||
: undefined
|
||||
return subtype === 'success' ? 'status-chrome' : 'error-surface'
|
||||
}
|
||||
if (provider === 'claude' && isClaudeStatusFrame(kind)) {
|
||||
if (kind === 'message:result') {
|
||||
const record =
|
||||
typeof payload === 'object' && payload !== null
|
||||
? (payload as Record<string, unknown>)
|
||||
: null
|
||||
return record?.subtype === 'success' ? 'debug-only' : 'timeline'
|
||||
}
|
||||
return 'debug-only'
|
||||
}
|
||||
return 'timeline'
|
||||
return catalogClassification(provider, kind) ?? 'timeline-substantive'
|
||||
}
|
||||
|
||||
@@ -132,6 +132,21 @@ describe('unhandled provider frame journal fallback', () => {
|
||||
).not.toBeNull()
|
||||
})
|
||||
|
||||
it('surfaces a failed hook completion while suppressing successful hook lifecycle', () => {
|
||||
const kind = 'notification:hook/completed'
|
||||
|
||||
expect(
|
||||
unhandledProviderFrameJournalItem('codex', kind, {
|
||||
run: { id: 'hook-1', status: 'completed' }
|
||||
})
|
||||
).toBeNull()
|
||||
expect(
|
||||
unhandledProviderFrameJournalItem('codex', kind, {
|
||||
run: { id: 'hook-1', status: 'failed' }
|
||||
})
|
||||
).not.toBeNull()
|
||||
})
|
||||
|
||||
it('keeps unknown substantive frames visible for both providers', () => {
|
||||
expect(
|
||||
unhandledProviderFrameJournalItem('codex', 'notification:future/event', {})
|
||||
|
||||
@@ -27,7 +27,8 @@ export function unhandledProviderFrameJournalItem(
|
||||
payload: unknown,
|
||||
limits: JournalPayloadLimits = DEFAULT_JOURNAL_PAYLOAD_LIMITS
|
||||
): UnhandledProviderFrameJournalItem | null {
|
||||
if (classifyProviderFrame(provider, kind, payload) === 'debug-only') {
|
||||
const classification = classifyProviderFrame(provider, kind, payload)
|
||||
if (classification === 'status-chrome' || classification === 'suppressed-benign') {
|
||||
return null
|
||||
}
|
||||
const serialized = serializeProviderPayload(payload)
|
||||
|
||||
Reference in New Issue
Block a user