diff --git a/src/main/claude/claude-api-retry-row.test.ts b/src/main/claude/claude-api-retry-row.test.ts index f2933b8a702..26ad6d3a26a 100644 --- a/src/main/claude/claude-api-retry-row.test.ts +++ b/src/main/claude/claude-api-retry-row.test.ts @@ -77,11 +77,11 @@ describe('a Claude api_retry frame', () => { expect(rows).toHaveLength(1) const [row] = rows expect(row).toMatchObject({ - text: 'Claude is rate-limited and retrying.', + text: 'Claude is rate-limited and retrying. Retry 3 of 10.\nLast error: HTTP 429 rate limit.', tone: 'warning', failure: { kind: 'providerRetrying', - retry: { error: 'rate_limit', status: 429 }, + retry: { error: 'rate_limit', status: 429, attempt: 3, maxRetries: 10 }, detail: { audience: 'log' } } }) @@ -90,11 +90,21 @@ describe('a Claude api_retry frame', () => { expect(row.failure?.detail?.text).toContain('"attempt":3') }) + it('says how far a gateway failure has got, and the code it failed with', async () => { + const [row] = await statusRowsFor([ + apiRetry(1, { error: 'server_error', error_status: 502 }), + apiRetry(2, { error: 'server_error', error_status: 502 }) + ]) + expect(row?.text).toBe( + 'Claude hit a temporary problem and is retrying. Retry 2 of 10.\nLast error: HTTP 502 server error.' + ) + }) + it('starts a new row when a later run starts over', async () => { const rows = await statusRowsFor([apiRetry(1), apiRetry(2), apiRetry(1)]) - expect(rows.map((row) => row.text)).toEqual([ - 'Claude is rate-limited and retrying.', - 'Claude is rate-limited and retrying.' + expect(rows.map((row) => row.text.split('\n')[0])).toEqual([ + 'Claude is rate-limited and retrying. Retry 2 of 10.', + 'Claude is rate-limited and retrying. Retry 1 of 10.' ]) }) @@ -106,10 +116,10 @@ describe('a Claude api_retry frame', () => { apiRetry(1, { error: 'something_new', error_status: 429 }) ]) expect(rows.map((row) => row.text)).toEqual([ - 'Claude hit a temporary problem and is retrying.', - 'Claude hit a temporary problem and is retrying.', - 'Claude hit a temporary problem and is retrying.', - 'Claude is rate-limited and retrying.' + 'Claude hit a temporary problem and is retrying. Retry 1 of 10.\nLast error: HTTP 529 overloaded.', + 'Claude hit a temporary problem and is retrying. Retry 1 of 10.\nLast error: HTTP 500 server error.', + 'Claude hit a temporary problem and is retrying. Retry 1 of 10.', + 'Claude is rate-limited and retrying. Retry 1 of 10.\nLast error: HTTP 429 something new.' ]) expect(rows[0]?.failure).toMatchObject({ retry: { error: 'overloaded', status: 529 } }) }) diff --git a/src/main/claude/claude-api-retry-row.ts b/src/main/claude/claude-api-retry-row.ts index 195e30461a7..8790adc5bd4 100644 --- a/src/main/claude/claude-api-retry-row.ts +++ b/src/main/claude/claude-api-retry-row.ts @@ -13,7 +13,12 @@ import { TUI_AGENT_DISPLAY_NAMES } from '../../shared/tui-agent-display-names' export const CLAUDE_API_RETRY_FRAME_KIND = 'message:system:api_retry' export function claudeApiRetryRowBody(message: Record): AgentJournalStatusItem { - const retry = readProviderRetry({ error: message.error, status: message.error_status }) + const retry = readProviderRetry({ + error: message.error, + status: message.error_status, + attempt: message.attempt, + maxRetries: message.max_retries + }) const words = agentSessionFailureWords( agentSessionFailureFact('providerRetrying', { detail: providerDiagnostic(JSON.stringify(message), 'log'), diff --git a/src/renderer/src/components/native-chat/agent-session-failure-words-text.ts b/src/renderer/src/components/native-chat/agent-session-failure-words-text.ts index 121f167712f..2c455769a78 100644 --- a/src/renderer/src/components/native-chat/agent-session-failure-words-text.ts +++ b/src/renderer/src/components/native-chat/agent-session-failure-words-text.ts @@ -263,6 +263,24 @@ const PIECES: Record + translate( + 'components.native-chat.failureWords.providerRetryNumber', + COPY.providerRetryNumber, + values + ), + providerRetryNumberOf: (values) => + translate( + 'components.native-chat.failureWords.providerRetryNumberOf', + COPY.providerRetryNumberOf, + values + ), + providerRetryLastError: (values) => + translate( + 'components.native-chat.failureWords.providerRetryLastError', + COPY.providerRetryLastError, + values + ), previousExitUnverifiable: (values) => translate( 'components.native-chat.failureWords.previousExitUnverifiable', diff --git a/src/renderer/src/i18n/en-runtime-required.json b/src/renderer/src/i18n/en-runtime-required.json index 565d59047d8..7a0ed36def4 100644 --- a/src/renderer/src/i18n/en-runtime-required.json +++ b/src/renderer/src/i18n/en-runtime-required.json @@ -2808,6 +2808,9 @@ "providerRejectedQuoted": "The provider did not accept this message: {{detail}}.", "providerRetrying": "{{agent}} hit a temporary problem and is retrying.", "providerRetryingQuoted": "{{agent}} is retrying: {{detail}}.", + "providerRetryNumber": "Retry {{attempt}}.", + "providerRetryNumberOf": "Retry {{attempt}} of {{maxRetries}}.", + "providerRetryLastError": "Last error: {{detail}}.", "providerStartFailed": "{{agent}} stopped before it finished starting.", "queueFull": "Too many messages were waiting for the agent, so this one was not sent.", "runCommandAgain": "Run /{{command}} again.", diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index ab40a3656bd..ed1227bc15a 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -17833,6 +17833,9 @@ "providerRateLimited": "{{agent}} is rate-limited and retrying.", "providerRetrying": "{{agent}} hit a temporary problem and is retrying.", "providerRetryingQuoted": "{{agent}} is retrying: {{detail}}.", + "providerRetryNumber": "Retry {{attempt}}.", + "providerRetryNumberOf": "Retry {{attempt}} of {{maxRetries}}.", + "providerRetryLastError": "Last error: {{detail}}.", "previousExitUnverifiable": "Couldn't stop {{agent}} from before.", "argumentsUnsupportedOption": "Saved Arguments contain an unsupported option ({{option}}).", "argumentsMissingValue": "Saved Arguments need a value for {{option}}.", diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index 5d9fe87229c..40eacd2614c 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -17612,6 +17612,9 @@ "providerRateLimited": "{{agent}} alcanzó un límite de solicitudes y está reintentando.", "providerRetrying": "{{agent}} tuvo un problema temporal y está reintentando.", "providerRetryingQuoted": "{{agent}} está reintentando: {{detail}}.", + "providerRetryNumber": "Reintento {{attempt}}.", + "providerRetryNumberOf": "Reintento {{attempt}} de {{maxRetries}}.", + "providerRetryLastError": "Último error: {{detail}}.", "previousExitUnverifiable": "No se pudo detener la ejecución anterior de {{agent}}." }, "writeNotice": { diff --git a/src/renderer/src/i18n/locales/fr.json b/src/renderer/src/i18n/locales/fr.json index f5c657e6d25..dbaaa188e92 100644 --- a/src/renderer/src/i18n/locales/fr.json +++ b/src/renderer/src/i18n/locales/fr.json @@ -17713,6 +17713,9 @@ "providerRateLimited": "{{agent}} est limité en débit et réessaie.", "providerRetrying": "{{agent}} a rencontré un problème temporaire et réessaie.", "providerRetryingQuoted": "{{agent}} réessaie : {{detail}}.", + "providerRetryNumber": "Nouvelle tentative {{attempt}}.", + "providerRetryNumberOf": "Nouvelle tentative {{attempt}} sur {{maxRetries}}.", + "providerRetryLastError": "Dernière erreur : {{detail}}.", "previousExitUnverifiable": "Impossible d'arrêter l'exécution précédente de {{agent}}." }, "writeNotice": { diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index e9f9c9e8c17..99eaf64aa7a 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -17647,6 +17647,9 @@ "providerRateLimited": "{{agent}} はレート制限を受けているため、再試行しています。", "providerRetrying": "{{agent}} で一時的な問題が発生したため、再試行しています。", "providerRetryingQuoted": "{{agent}} は再試行しています: {{detail}}。", + "providerRetryNumber": "{{attempt}} 回目の再試行です。", + "providerRetryNumberOf": "{{maxRetries}} 回中 {{attempt}} 回目の再試行です。", + "providerRetryLastError": "直近のエラー: {{detail}}。", "previousExitUnverifiable": "以前の {{agent}} を停止できませんでした。" }, "writeNotice": { diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index a44ed6c6fd2..c5cc00dcf38 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -17647,6 +17647,9 @@ "providerRateLimited": "{{agent}}이(가) 속도 제한에 걸려 다시 시도하고 있습니다.", "providerRetrying": "{{agent}}에 일시적인 문제가 발생하여 다시 시도하고 있습니다.", "providerRetryingQuoted": "{{agent}}이(가) 다시 시도하고 있습니다: {{detail}}.", + "providerRetryNumber": "{{attempt}}번째 재시도입니다.", + "providerRetryNumberOf": "{{maxRetries}}회 중 {{attempt}}번째 재시도입니다.", + "providerRetryLastError": "마지막 오류: {{detail}}.", "previousExitUnverifiable": "이전에 실행된 {{agent}}을(를) 중지하지 못했습니다." }, "writeNotice": { diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index 5ca74326af4..4b2aa106a26 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -17612,6 +17612,9 @@ "providerRateLimited": "{{agent}} 已被限流,正在重试。", "providerRetrying": "{{agent}} 遇到临时问题,正在重试。", "providerRetryingQuoted": "{{agent}} 正在重试:{{detail}}。", + "providerRetryNumber": "第 {{attempt}} 次重试。", + "providerRetryNumberOf": "第 {{attempt}} 次重试,共 {{maxRetries}} 次。", + "providerRetryLastError": "上次错误:{{detail}}。", "previousExitUnverifiable": "无法停止之前的 {{agent}}。" }, "writeNotice": { diff --git a/src/shared/agent-session-failure-copy.ts b/src/shared/agent-session-failure-copy.ts index 83cb8dfcdea..a55978aea29 100644 --- a/src/shared/agent-session-failure-copy.ts +++ b/src/shared/agent-session-failure-copy.ts @@ -89,6 +89,9 @@ export const AGENT_SESSION_FAILURE_COPY = { providerRateLimited: '{{agent}} is rate-limited and retrying.', providerRetrying: '{{agent}} hit a temporary problem and is retrying.', providerRetryingQuoted: '{{agent}} is retrying: {{detail}}.', + providerRetryNumber: 'Retry {{attempt}}.', + providerRetryNumberOf: 'Retry {{attempt}} of {{maxRetries}}.', + providerRetryLastError: 'Last error: {{detail}}.', previousExitUnverifiable: "Couldn't stop {{agent}} from before." } as const @@ -102,6 +105,8 @@ export type AgentSessionFailureCopyValues = { option?: string limit?: string size?: string + attempt?: string + maxRetries?: string } /** One piece in the reader's language, placeholders filled. */ diff --git a/src/shared/agent-session-failure-words.test.ts b/src/shared/agent-session-failure-words.test.ts index 4e6b81b265a..a909a1e2b90 100644 --- a/src/shared/agent-session-failure-words.test.ts +++ b/src/shared/agent-session-failure-words.test.ts @@ -48,6 +48,8 @@ function factsFor(kind: AgentSessionFailureKind): AgentSessionFailureFact[] { facts.push({ kind, retry: { error: 'rate_limit', status: 429 } }) facts.push({ kind, retry: { error: 'overloaded', status: 529 } }) facts.push({ kind, retry: { status: 500 } }) + facts.push({ kind, retry: { error: 'server_error', status: 502, attempt: 3, maxRetries: 10 } }) + facts.push({ kind, retry: { attempt: 1 } }) } if (kind === 'attachmentInvalid') { for (const reason of AGENT_SESSION_ATTACHMENT_PROBLEM_REASONS) { @@ -239,7 +241,7 @@ describe('the words written beside a failure fact', () => { detail: { text: '{"type":"system","subtype":"api_retry"}', audience: 'log' }, retry: { error: 'rate_limit', status: 429 } }) - ).toBe('Codex is rate-limited and retrying.') + ).toBe('Codex is rate-limited and retrying.\nLast error: HTTP 429 rate limit.') expect(retrying({})).toBe('Codex hit a temporary problem and is retrying.') }) @@ -259,6 +261,35 @@ describe('the words written beside a failure fact', () => { ) }) + it("says which retry it is, and the provider's codes when it wrote no account", () => { + const retrying = (retry: AgentSessionFailureFact['retry']) => + agentSessionFailureSentence({ kind: 'providerRetrying', retry }, 'row', { + agentName: 'Claude' + }) + expect(retrying({ error: 'server_error', status: 502, attempt: 3, maxRetries: 10 })).toBe( + 'Claude hit a temporary problem and is retrying. Retry 3 of 10.\nLast error: HTTP 502 server error.' + ) + expect(retrying({ status: 502, attempt: 3 })).toBe( + 'Claude hit a temporary problem and is retrying. Retry 3.\nLast error: HTTP 502.' + ) + // The provider's own account outranks its codes. + expect(retrying({ status: 429, cause: 'Too many requests', attempt: 2, maxRetries: 5 })).toBe( + 'Claude is rate-limited and retrying. Retry 2 of 5.\nToo many requests' + ) + // A provider that words its own progress is quoted alone, never counted twice. + expect( + agentSessionFailureSentence( + { + kind: 'providerRetrying', + detail: { text: 'Reconnecting... 2/5', audience: 'person' }, + retry: { status: 502, attempt: 2, maxRetries: 5 } + }, + 'row', + { agentName: 'Codex' } + ) + ).toBe('Codex is retrying: Reconnecting... 2/5.') + }) + it('says which limit an attachment broke, in megabytes', () => { const sentence = (attachment: AgentSessionFailureFact['attachment']) => agentSessionFailureSentence({ kind: 'attachmentInvalid', attachment }, 'rejection', { diff --git a/src/shared/agent-session-failure-words.ts b/src/shared/agent-session-failure-words.ts index 2c68589b409..abb601ff1b9 100644 --- a/src/shared/agent-session-failure-words.ts +++ b/src/shared/agent-session-failure-words.ts @@ -25,6 +25,7 @@ import { type AgentSessionFailureSay } from './agent-session-failure-copy' import type { AgentSessionWireRefusalCode } from './agent-session-wire-refusals' +import { providerRetryWords, withRetryCause } from './agent-session-provider-retry-words' import { joinSentences } from './sentence-joining' import { DISPATCH_REJECTED_CANCELLED, @@ -122,11 +123,6 @@ function quotingPersonDetail( return quoted ? say(quotedLead, { ...values, detail: quoted }) : say(lead, values) } -/** The provider's account of what failed goes on the line under the sentence, as it wrote it. */ -function withRetryCause(sentence: string, cause: string | undefined): string { - return cause ? `${sentence}\n${cause}` : sentence -} - /** The next step after a start or restart that failed: the command, or the message, again. */ function startRetry( say: AgentSessionFailureSay, @@ -274,23 +270,18 @@ const FAILURE_SENTENCES = { hostStopped: (context, _fact, _surface, say) => say('hostStopped', agent(say, context)), // A provider that says how its retry is going, for a person, is quoted: that is the progress. providerRetrying: (context, { retry, detail }, _surface, say) => - withRetryCause( - detail?.audience === 'person' - ? quotingPersonDetail( + detail?.audience === 'person' + ? withRetryCause( + quotingPersonDetail( say, 'providerRetrying', 'providerRetryingQuoted', detail, agent(say, context) - ) - : say( - retry?.error === 'rate_limit' || retry?.status === 429 - ? 'providerRateLimited' - : 'providerRetrying', - agent(say, context) ), - retry?.cause - ), + retry?.cause + ) + : providerRetryWords(say, agent(say, context), retry), previousExitUnverifiable: (context, _fact, _surface, say) => say('previousExitUnverifiable', agent(say, context)) } satisfies Record diff --git a/src/shared/agent-session-failure.test.ts b/src/shared/agent-session-failure.test.ts index 048d33c995b..14cd75f7d6f 100644 --- a/src/shared/agent-session-failure.test.ts +++ b/src/shared/agent-session-failure.test.ts @@ -92,7 +92,7 @@ describe('reading a failure fact', () => { expect( readAgentSessionFailureFact({ kind: 'providerRetrying', - retry: { error: 'rate_limit', status: 429, attempt: 3 } + retry: { error: 'rate_limit', status: 429, retryDelayMs: 600 } }) ).toEqual({ kind: 'providerRetrying', retry: { error: 'rate_limit', status: 429 } }) expect( @@ -130,6 +130,17 @@ describe('reading a failure fact', () => { }) ).toEqual({ kind: 'restartFailed', refusal: { code: 'agent_session_conflict' } }) }) + + it('keeps which retry it is, and its maximum only where it bounds that retry', () => { + expect(readProviderRetry({ attempt: 3, maxRetries: 10 })).toEqual({ + attempt: 3, + maxRetries: 10 + }) + expect(readProviderRetry({ status: 502, maxRetries: 10 })).toEqual({ status: 502 }) + expect(readProviderRetry({ attempt: 12, maxRetries: 10 })).toEqual({ attempt: 12 }) + expect(readProviderRetry({ attempt: 0, maxRetries: 10 })).toBeUndefined() + expect(readProviderRetry({ attempt: 2.5, status: 502 })).toEqual({ status: 502 }) + }) }) describe('reading all of a failure fact', () => { @@ -146,6 +157,14 @@ describe('reading all of a failure fact', () => { }), agentSessionFailureFact('providerRetrying', { retry: readProviderRetry({ error: 'rate_limit', status: 429 }) + }), + agentSessionFailureFact('providerRetrying', { + retry: readProviderRetry({ + error: 'server_error', + status: 502, + attempt: 3, + maxRetries: 10 + }) }) ]) { expect(readWholeAgentSessionFailureFact(JSON.parse(JSON.stringify(fact)))).toEqual(fact) diff --git a/src/shared/agent-session-failure.ts b/src/shared/agent-session-failure.ts index 7f2a0463935..fe3180be6c8 100644 --- a/src/shared/agent-session-failure.ts +++ b/src/shared/agent-session-failure.ts @@ -128,6 +128,10 @@ export type AgentSessionProviderRetry = { status?: number /** The provider's own account of what failed, written for a person: the row's second line. */ cause?: string + /** Which retry this is, counted from 1. */ + attempt?: number + /** The most retries the provider makes before it gives up. */ + maxRetries?: number } export type AgentSessionFailureFact = { @@ -224,17 +228,28 @@ export function readProviderRetry(value: unknown): AgentSessionProviderRetry | u } const error = typeof value.error === 'string' && value.error.trim() ? value.error.trim() : undefined - const status = - typeof value.status === 'number' && Number.isInteger(value.status) && value.status > 0 - ? value.status - : undefined + const status = positiveInteger(value.status) const cause = typeof value.cause === 'string' ? providerDiagnostic(value.cause, 'person')?.text : undefined - return error || status || cause - ? { ...(error ? { error } : {}), ...(status ? { status } : {}), ...(cause ? { cause } : {}) } + const attempt = positiveInteger(value.attempt) + // A maximum bounds a retry; alone, or below the retry it bounds, it says nothing. + const max = positiveInteger(value.maxRetries) + const maxRetries = attempt && max && max >= attempt ? max : undefined + return error || status || cause || attempt + ? { + ...(error ? { error } : {}), + ...(status ? { status } : {}), + ...(cause ? { cause } : {}), + ...(attempt ? { attempt } : {}), + ...(maxRetries ? { maxRetries } : {}) + } : undefined } +function positiveInteger(value: unknown): number | undefined { + return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : undefined +} + /** A fact as a reader meets it. Undefined for anything this build cannot place, including a kind a * newer host added, so the reader falls back to what it does for a row with no fact. */ export function readAgentSessionFailureFact(value: unknown): AgentSessionFailureFact | undefined { diff --git a/src/shared/agent-session-provider-retry-words.ts b/src/shared/agent-session-provider-retry-words.ts new file mode 100644 index 00000000000..ece476ca096 --- /dev/null +++ b/src/shared/agent-session-provider-retry-words.ts @@ -0,0 +1,60 @@ +// The words for a provider retry: which retry it is and what last failed, from the provider's +// own fields, for a provider that wrote none of its own for a person. + +import type { AgentSessionProviderRetry } from './agent-session-failure' +import type { + AgentSessionFailureCopyValues, + AgentSessionFailureSay +} from './agent-session-failure-copy' +import { joinSentences } from './sentence-joining' + +/** The provider's account of what failed goes on the line under the sentence, as it wrote it. */ +export function withRetryCause(sentence: string, cause: string | undefined): string { + return cause ? `${sentence}\n${cause}` : sentence +} + +export function providerRetryWords( + say: AgentSessionFailureSay, + agent: AgentSessionFailureCopyValues, + retry: AgentSessionProviderRetry | undefined +): string { + return withRetryCause( + joinSentences([ + say( + retry?.error === 'rate_limit' || retry?.status === 429 + ? 'providerRateLimited' + : 'providerRetrying', + agent + ), + ...retryNumber(say, retry) + ]), + retry?.cause ?? retryLastError(say, retry) + ) +} + +function retryNumber( + say: AgentSessionFailureSay, + retry: AgentSessionProviderRetry | undefined +): string[] { + if (!retry?.attempt) { + return [] + } + const attempt = String(retry.attempt) + return [ + retry.maxRetries + ? say('providerRetryNumberOf', { attempt, maxRetries: String(retry.maxRetries) }) + : say('providerRetryNumber', { attempt }) + ] +} + +/** The provider's own codes for what failed, when it wrote no account of it for a person. */ +function retryLastError( + say: AgentSessionFailureSay, + retry: AgentSessionProviderRetry | undefined +): string | undefined { + // `server_error` reads as words once its separators are spaces. + const codes = [retry?.status ? `HTTP ${retry.status}` : '', retry?.error?.replace(/_+/g, ' ')] + .filter(Boolean) + .join(' ') + return codes ? say('providerRetryLastError', { detail: codes }) : undefined +}