Files
archived-orca/src/shared/agent-session-failure-words.test.ts
T

378 lines
15 KiB
TypeScript

import { describe, expect, it } from 'vitest'
import {
AGENT_SESSION_ATTACHMENT_PROBLEM_REASONS,
AGENT_SESSION_FAILURE_KINDS,
isSubmissionRejectionFact,
readAgentSessionFailureFact,
type AgentSessionFailureFact,
type AgentSessionFailureKind,
type SubmissionRejectionKind
} from './agent-session-failure'
import {
agentSessionFailureSentence,
agentSessionFailureWords,
type AgentSessionFailureSurface
} from './agent-session-failure-words'
import {
classifyDispatchRejection,
DISPATCH_REJECTED_CANCELLED,
DISPATCH_REJECTED_CODEX_QUEUE_FULL,
DISPATCH_REJECTED_QUEUE_FULL,
DISPATCH_REJECTED_WRITE_FAILED
} from './structured-agent-session-dispatch-rejection'
/** What Orca's own text looks like: a code or marker, a uuid, a path, an exception. */
const ORCA_INTERNAL = /\b[a-z]+_[a-z_]+\b|[0-9a-f]{8}-[0-9a-f]{4}-|[/\\][\w.-]+[/\\]|Error:|ENOENT/
const SURFACES: readonly AgentSessionFailureSurface[] = ['row', 'rejection', 'answer']
const LEGACY_MARKER_KINDS: ReadonlySet<SubmissionRejectionKind> = new Set([
'cancelled',
'writeFailed',
'queueFull'
])
/** Cover every fact shape, with person/log/refusal details and every attachment reason with/without a limit. */
function factsFor(kind: AgentSessionFailureKind): AgentSessionFailureFact[] {
const facts: AgentSessionFailureFact[] = [
{ kind },
{ kind, detail: { text: 'Context window exceeded.', audience: 'person' } },
{
kind,
// A log detail and a refusal ride with the fact; neither may reach the sentence.
detail: { text: 'Error: ENOENT /Users/me/.claude/x agent_session_conflict', audience: 'log' },
refusal: { code: 'agent_session_identity_required', details: { reason: 'recordMissing' } }
}
]
if (kind === 'providerRetrying') {
facts.push({ kind, retry: { error: 'rate_limit', status: 429 } })
facts.push({ kind, retry: { error: 'overloaded', status: 529 } })
facts.push({ kind, retry: { status: 500 } })
facts.push({ kind, retry: { error: 'server_error', status: 502, attempt: 3, maxRetries: 10 } })
facts.push({ kind, retry: { attempt: 1 } })
}
if (kind === 'attachmentInvalid') {
for (const reason of AGENT_SESSION_ATTACHMENT_PROBLEM_REASONS) {
facts.push({ kind, attachment: { reason } })
facts.push({ kind, attachment: { reason, limit: 5 * 1024 * 1024 } })
}
}
return facts
}
describe('the words written beside a failure fact', () => {
describe.each(SURFACES)('on the %s surface', (surface) => {
it.each(AGENT_SESSION_FAILURE_KINDS)('for %s are one sentence a person can read', (kind) => {
for (const fact of factsFor(kind)) {
for (const [agentName, command] of [
['Claude', undefined],
[undefined, undefined],
['Codex', 'clear']
] as const) {
const context = { agentName, command, provider: 'claude' } as const
const sentence = agentSessionFailureSentence(fact, surface, context)
expect(sentence).toMatch(/[^.]\.$/)
expect(sentence).not.toMatch(/\.\./)
expect(sentence).not.toMatch(ORCA_INTERNAL)
if (surface === 'row') {
expect(agentSessionFailureWords(fact, { ...context, surface })).toEqual({
text: sentence,
failure: fact
})
} else if (
surface === 'rejection' &&
isSubmissionRejectionFact(fact) &&
!LEGACY_MARKER_KINDS.has(fact.kind)
) {
expect(agentSessionFailureWords(fact, { ...context, surface })).toEqual({
reason: sentence,
rejection: fact
})
}
}
}
})
})
it('writes the legacy markers released clients hide, byte for byte, on a rejection', () => {
const reason = (kind: SubmissionRejectionKind, provider?: 'claude' | 'codex') =>
agentSessionFailureWords({ kind }, { surface: 'rejection', provider }).reason
expect(reason('cancelled')).toBe(DISPATCH_REJECTED_CANCELLED)
expect(DISPATCH_REJECTED_CANCELLED).toBe('provider_cancelled_before_start')
expect(reason('queueFull', 'claude')).toBe(DISPATCH_REJECTED_QUEUE_FULL)
expect(DISPATCH_REJECTED_QUEUE_FULL).toBe('claude structured dispatch queue is full')
expect(reason('queueFull', 'codex')).toBe(DISPATCH_REJECTED_CODEX_QUEUE_FULL)
expect(DISPATCH_REJECTED_CODEX_QUEUE_FULL).toBe('codex structured dispatch queue is full')
// Bare: the error that once followed it belongs in the log.
expect(reason('writeFailed')).toBe(DISPATCH_REJECTED_WRITE_FAILED)
expect(DISPATCH_REJECTED_WRITE_FAILED).toBe('provider_write_failed')
// With no provider to name its marker, a full queue gets a sentence.
expect(reason('queueFull')).toBe(
'Too many messages were waiting for the agent, so this one was not sent.'
)
for (const kind of LEGACY_MARKER_KINDS) {
const written = agentSessionFailureWords(
{ kind },
{ surface: 'rejection', provider: 'codex' }
)
expect(classifyDispatchRejection({ reason: written.reason })).toMatchObject({ kind })
}
})
it('names /compact as the next step for the start a /compact needed', () => {
const compact = (kind: AgentSessionFailureKind) =>
agentSessionFailureSentence({ kind }, 'rejection', {
agentName: 'Claude',
command: 'compact'
})
expect(compact('notSignedIn')).toBe(
"Claude isn't signed in. Run `claude auth login`, or choose an account in Claude Accounts settings. Run /compact again."
)
expect(
agentSessionFailureSentence({ kind: 'notSignedIn', account: 'managed' }, 'rejection', {
agentName: 'Codex',
command: 'compact'
})
).toBe(
"This Codex account isn't signed in. Sign in again in Codex Accounts settings. Run /compact again."
)
expect(compact('startFailed')).toBe("Claude couldn't start. Run /compact again.")
expect(compact('restartFailed')).toBe("Claude couldn't restart. Run /compact again.")
expect(compact('providerStartFailed')).toBe(
'Claude stopped before it finished starting. Run /compact again.'
)
})
it('names the agent that stopped starting, and a failed start or restart gives a next step', () => {
const sentence = (kind: AgentSessionFailureKind, agentName?: string) =>
agentSessionFailureSentence({ kind }, 'rejection', { agentName })
expect(sentence('providerStartFailed', 'Claude')).toBe(
'Claude stopped before it finished starting. Send your message to try again.'
)
expect(sentence('providerStartFailed')).toBe(
'The agent stopped before it finished starting. Send your message to try again.'
)
expect(sentence('startFailed', 'Codex')).toBe(
"Codex couldn't start. Send your message to try again."
)
expect(sentence('restartFailed')).toBe(
"The agent couldn't restart. Send your message to try again."
)
// Nothing to restart from: a new chat is the only step, so no retry is offered.
expect(
agentSessionFailureSentence(
{
kind: 'startFailed',
refusal: { code: 'agent_session_identity_required', details: { reason: 'recordMissing' } }
},
'row',
{ agentName: 'Codex' }
)
).toBe("Codex couldn't start. Start a new chat to continue.")
})
it("names a too-old agent's installed and minimum versions, and still reads Codex's legacy fact", () => {
const words = (details: Record<string, unknown>) =>
agentSessionFailureSentence(
readAgentSessionFailureFact({
kind: 'startFailed',
refusal: {
code: 'agent_session_operation_invalid',
details: { reason: 'attachFailed', ...details }
}
})!,
'row',
{}
)
expect(
words({
agentInstallation: {
agent: 'opencode',
installedVersion: '1.0.0',
minimumVersion: '1.18.31'
}
})
).toBe('OpenCode 1.0.0 is too old for chats. Update to 1.18.31 or newer.')
expect(
words({ codexInstallation: { installedVersion: '0.135.0', minimumVersion: '0.136.0' } })
).toBe('Codex 0.135.0 is too old for chats. Update to 0.136.0 or newer.')
})
it('says a start refused beside a process Orca could not stop in its own words', () => {
const refused = (context: { command?: 'compact'; retryControl?: boolean } = {}) =>
agentSessionFailureSentence(
{
kind: 'restartFailed',
refusal: {
code: 'agent_session_ownership_unknown',
details: { reason: 'previousExitUnverifiable' }
}
},
'rejection',
{ agentName: 'Claude', ...context }
)
expect(refused()).toBe(
"Couldn't stop Claude from before. Send your message again to try once more."
)
expect(refused({ command: 'compact' })).toBe(
"Couldn't stop Claude from before. Run /compact again."
)
expect(refused({ retryControl: true })).toBe("Couldn't stop Claude from before.")
})
it('names the exit a row reports differently from the message it left unsent', () => {
expect(
agentSessionFailureWords({ kind: 'providerExited' }, { surface: 'row', agentName: 'Claude' })
.text
).toBe(
'Claude stopped while this response was in progress. You can continue in this conversation.'
)
expect(
agentSessionFailureWords(
{ kind: 'providerExited' },
{ surface: 'rejection', agentName: 'Codex' }
).reason
).toBe('Codex stopped before this message was sent.')
expect(agentSessionFailureWords({ kind: 'providerExited' }, { surface: 'row' }).text).toBe(
'The agent stopped while this response was in progress. You can continue in this conversation.'
)
expect(
agentSessionFailureWords({ kind: 'providerExited' }, { surface: 'rejection' }).reason
).toBe('The agent stopped before this message was sent.')
})
it('says a refused Stop reached the agent, which declined it', () => {
const refused = (detail?: { text: string; audience: 'person' | 'log' }, agentName?: string) =>
agentSessionFailureWords(
{ kind: 'stopRefused', ...(detail ? { detail } : {}) },
{ surface: 'row', agentName }
).text
expect(refused(undefined, 'Codex')).toBe('Codex had no turn running to stop.')
expect(refused()).toBe('The agent had no turn running to stop.')
expect(refused({ text: 'no active turn to interrupt.', audience: 'person' }, 'Codex')).toBe(
"Codex didn't stop: no active turn to interrupt."
)
// Words Codex wrote for the log are never quoted to a person.
expect(refused({ text: 'rpc -32600', audience: 'log' }, 'Codex')).toBe(
'Codex had no turn running to stop.'
)
})
it('says what frees a chat a terminal agent still holds, without naming its process', () => {
expect(
agentSessionFailureSentence(
{
kind: 'restartFailed',
refusal: { code: 'agent_session_conflict', details: { reason: 'claimConflicted' } }
},
'row',
{ agentName: 'Codex' }
)
).toBe(
"Codex couldn't restart. This chat is still open in a terminal agent. Quit that agent to continue the chat here."
)
})
it("quotes a provider's own retry progress, and keeps a log detail out of it", () => {
const retrying = (fact: Omit<AgentSessionFailureFact, 'kind'>) =>
agentSessionFailureSentence({ kind: 'providerRetrying', ...fact }, 'row', {
agentName: 'Codex'
})
expect(retrying({ detail: { text: 'Reconnecting... 2/5', audience: 'person' } })).toBe(
'Codex is retrying: Reconnecting... 2/5.'
)
expect(
retrying({
detail: { text: '{"type":"system","subtype":"api_retry"}', audience: 'log' },
retry: { error: 'rate_limit', status: 429 }
})
).toBe('Codex is rate-limited and retrying.\nLast error: HTTP 429 rate limit.')
expect(retrying({})).toBe('Codex hit a temporary problem and is retrying.')
})
it("puts the provider's account of what failed on the retry row's second line", () => {
const retrying = (fact: Omit<AgentSessionFailureFact, 'kind'>) =>
agentSessionFailureSentence({ kind: 'providerRetrying', ...fact }, 'row', {
agentName: 'Codex'
})
expect(
retrying({
detail: { text: 'Reconnecting... 2/5', audience: 'person' },
retry: { cause: 'stream disconnected before completion' }
})
).toBe('Codex is retrying: Reconnecting... 2/5.\nstream disconnected before completion')
expect(retrying({ retry: { status: 429, cause: 'Too many requests' } })).toBe(
'Codex is rate-limited and retrying.\nToo many requests'
)
})
it("says which retry it is, and the provider's codes when it wrote no account", () => {
const retrying = (retry: AgentSessionFailureFact['retry']) =>
agentSessionFailureSentence({ kind: 'providerRetrying', retry }, 'row', {
agentName: 'Claude'
})
expect(retrying({ error: 'server_error', status: 502, attempt: 3, maxRetries: 10 })).toBe(
'Claude hit a temporary problem and is retrying. Retry 3 of 10.\nLast error: HTTP 502 server error.'
)
expect(retrying({ status: 502, attempt: 3 })).toBe(
'Claude hit a temporary problem and is retrying. Retry 3.\nLast error: HTTP 502.'
)
// The provider's own account outranks its codes.
expect(retrying({ status: 429, cause: 'Too many requests', attempt: 2, maxRetries: 5 })).toBe(
'Claude is rate-limited and retrying. Retry 2 of 5.\nToo many requests'
)
// A provider that words its own progress is quoted alone, never counted twice.
expect(
agentSessionFailureSentence(
{
kind: 'providerRetrying',
detail: { text: 'Reconnecting... 2/5', audience: 'person' },
retry: { status: 502, attempt: 2, maxRetries: 5 }
},
'row',
{ agentName: 'Codex' }
)
).toBe('Codex is retrying: Reconnecting... 2/5.')
})
it('says which limit an attachment broke, in megabytes', () => {
const sentence = (attachment: AgentSessionFailureFact['attachment']) =>
agentSessionFailureSentence({ kind: 'attachmentInvalid', attachment }, 'rejection', {
agentName: 'Claude'
})
expect(sentence({ reason: 'tooLarge', limit: 5 * 1024 * 1024 })).toBe(
'An image on this message is larger than 5 MB, so the message was not sent.'
)
expect(sentence({ reason: 'totalTooLarge', limit: 20 * 1024 * 1024 })).toBe(
'The images on this message add up to more than 20 MB, so the message was not sent.'
)
expect(sentence({ reason: 'tooMany', limit: 20 })).toBe(
'Claude accepts at most 20 images in one message, so this message was not sent.'
)
expect(sentence(undefined)).toBe("An attachment on this message can't be sent to the agent.")
})
})
describe('an attachment problem as a reader meets it', () => {
it('keeps what it can place and drops what it cannot', () => {
expect(
readAgentSessionFailureFact({
kind: 'attachmentInvalid',
attachment: { reason: 'tooMany', limit: 20 }
})
).toEqual({ kind: 'attachmentInvalid', attachment: { reason: 'tooMany', limit: 20 } })
expect(
readAgentSessionFailureFact({
kind: 'attachmentInvalid',
attachment: { reason: 'tooMany', limit: 'twenty' }
})
).toEqual({ kind: 'attachmentInvalid', attachment: { reason: 'tooMany' } })
// A reason a newer host added reads as the generic attachment failure.
expect(
readAgentSessionFailureFact({
kind: 'attachmentInvalid',
attachment: { reason: 'futureReason' }
})
).toEqual({ kind: 'attachmentInvalid' })
})
})