From cbd75eb0084aaa16feb04a079140f77631be6892 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 16 Nov 2026 02:05:16 -0800 Subject: [PATCH] fix(acp): keep saved chats retryable after temporary restore failures (#42981) * test(acp): cover temporary restore recovery and cancellation Co-authored-by: outworkAnton * fix(acp): preserve saved conversations through temporary restore failures Share temporary provider-answer classification and bounded abort-aware retries. Isolate each load attempt so partial failed replay cannot enter successful history catch-up. Co-authored-by: outworkAnton * fix(acp): end automatic startup after the restore retry budget * test(acp): use the advertised resume preference type * fix(acp): retry temporary moved-session lookup failures within the same flight --------- Co-authored-by: outworkAnton --- .../discovery/acp-listing-session-models.ts | 22 ++-- src/main/acp/protocol/acp-request-retry.ts | 27 ++++ .../acp-structured-restore-retry.test.ts | 114 ++++++++++++++++ ...-structured-session-adapter-resume.test.ts | 122 +++++++++++++++++- .../start/acp-session-reopen-failure.ts | 21 +-- .../acp/session/start/acp-startup-outcome.ts | 15 +-- .../session/start/acp-structured-handshake.ts | 51 +++++--- src/main/acp/transport/acp-errors.ts | 14 ++ 8 files changed, 327 insertions(+), 59 deletions(-) create mode 100644 src/main/acp/protocol/acp-request-retry.ts create mode 100644 src/main/acp/session/acp-structured-restore-retry.test.ts diff --git a/src/main/acp/discovery/acp-listing-session-models.ts b/src/main/acp/discovery/acp-listing-session-models.ts index d3e2b519830..3ac57e32ed5 100644 --- a/src/main/acp/discovery/acp-listing-session-models.ts +++ b/src/main/acp/discovery/acp-listing-session-models.ts @@ -1,4 +1,4 @@ -import { setTimeout as delay } from 'node:timers/promises' +import { retryAcpRequest } from '../protocol/acp-request-retry' import { observeAcpForkSupport } from '../capabilities/acp-fork-support' import { assertStructuredAgentResolutionCurrent, @@ -39,18 +39,14 @@ async function startListingSession( workspace: string, operation?: StructuredAgentResolutionOperation ): Promise { - for (const backoffMs of [250, 500]) { - try { - return await connection.start({ cwd: workspace, mcpServers: [] }) - } catch (error) { - if (!(error instanceof AcpAgentError && error.code === -32603)) { - throw error - } - } - await delay(backoffMs, undefined, { signal: operation?.signal }) - assertStructuredAgentResolutionCurrent(operation) - } - return connection.start({ cwd: workspace, mcpServers: [] }) + return retryAcpRequest( + () => connection.start({ cwd: workspace, mcpServers: [] }), + { + signal: operation?.signal, + assertCurrent: () => assertStructuredAgentResolutionCurrent(operation) + }, + (error) => error instanceof AcpAgentError && error.code === -32603 + ) } /** Lists models from a disposable session, learning menus for the models it starts on or switches to. */ diff --git a/src/main/acp/protocol/acp-request-retry.ts b/src/main/acp/protocol/acp-request-retry.ts new file mode 100644 index 00000000000..104562e799f --- /dev/null +++ b/src/main/acp/protocol/acp-request-retry.ts @@ -0,0 +1,27 @@ +import { setTimeout as delay } from 'node:timers/promises' +import { temporaryAcpProviderAnswer } from '../transport/acp-errors' + +/** A bounded retry, owned by the operation that can still use the answer. */ +export async function retryAcpRequest( + request: () => Promise, + operation: { signal?: AbortSignal; assertCurrent: () => void }, + retryable: (error: unknown) => boolean = temporaryAcpProviderAnswer +): Promise { + const assertCurrent = () => { + operation.signal?.throwIfAborted() + operation.assertCurrent() + } + for (const backoffMs of [250, 500]) { + assertCurrent() + try { + return await request() + } catch (error) { + if (!retryable(error)) { + throw error + } + } + await delay(backoffMs, undefined, { signal: operation.signal }) + } + assertCurrent() + return request() +} diff --git a/src/main/acp/session/acp-structured-restore-retry.test.ts b/src/main/acp/session/acp-structured-restore-retry.test.ts new file mode 100644 index 00000000000..350bb4647dc --- /dev/null +++ b/src/main/acp/session/acp-structured-restore-retry.test.ts @@ -0,0 +1,114 @@ +import { expect, it } from 'vitest' +import { readTestHostJournal } from '../../native-chat/agent-session-journal/journal-observation.test-support' +import { closeProviderTimelineRigs } from '../../native-chat/agent-session-timeline/provider-timeline-assembler-test-support' +import { + HOST_TEST_SESSION as SESSION, + hostTestAttachParams +} from '../../native-chat/agent-session-wire/host/testing/structured-agent-session-host-test-data.test-fixture' +import { + createRestTestRig, + REST_TEST_CALLER, + restTestSend, + type RestTestRig +} from '../../native-chat/agent-session-wire/host/testing/structured-agent-session-rest-test-rig.test-fixture' +import { StructuredAgentRegistry } from '../../native-chat/agent-session-wire/providers/structured-agent-registry' +import { acpStructuredAgentDefinition } from './acp-structured-agent-definitions' +import { + GROK, + GROK_CONFIG_OPTIONS, + openAcpAdapterRig, + waitFor +} from './testing/acp-structured-adapter.test-support' +import { launch } from './testing/acp-structured-host.test-support' + +it('ends an exhausted restore flight without multiplying retries and allows a new same-chat send', async () => { + let rig: RestTestRig | undefined + let restoring = false + let recovered = false + const requests: string[] = [] + const acp = await openAcpAdapterRig({ + deps: { + resolveLaunch: launch(() => restoring), + onEvent: (event) => rig?.host.handleAdapterEvent(event), + onDispatchSettledLate: (settlement) => void rig?.host.settleLateDispatch(settlement) + }, + script: (agent) => { + agent.on('session/new', (frame) => { + requests.push('session/new') + agent.reply(frame, { sessionId: 'acp-session-1', configOptions: GROK_CONFIG_OPTIONS }) + }) + agent.on('session/load', (frame) => { + requests.push('session/load') + if (recovered) { + agent.reply(frame, { configOptions: GROK_CONFIG_OPTIONS }) + } else { + agent.fail(frame, -32603, 'Internal service failure', { + errorName: 'ServiceUnavailableError' + }) + } + }) + agent.on('session/prompt', (frame) => agent.reply(frame, { stopReason: 'end_turn' })) + } + }) + rig = await createRestTestRig({ + adapter: acp.adapter, + agents: new StructuredAgentRegistry([ + { definition: acpStructuredAgentDefinition(GROK), adapter: acp.adapter } + ]) + }) + rig.host.collaboratorsForTests().lifetime.idleSweep.dispose() + try { + const attached = await rig.host.attach( + REST_TEST_CALLER, + hostTestAttachParams(null, { + provider: 'grok', + agent: 'grok', + accountHome: { variable: 'GROK_HOME', path: rig.root }, + providerHandle: undefined + }) + ) + expect(attached.ok).toBe(true) + await waitFor(() => expect(rig!.store.getRecord(SESSION)?.providerHandleChain).toHaveLength(1)) + await rig.host.close(SESSION, 'evict') + restoring = true + const failed = await rig.host.send( + REST_TEST_CALLER, + restTestSend('during outage', rig.store.getRecord(SESSION)!.lease.runtimeFence) + ) + expect(failed.ok).toBe(true) + await waitFor(async () => { + const snapshot = await readTestHostJournal(rig!.host, SESSION) + expect(snapshot.submissions[0]?.dispatchState).toBe('rejected') + }) + expect(requests).toEqual(['session/new', 'session/load', 'session/load', 'session/load']) + expect(acp.spawned.filter((step) => step === 'spawn')).toHaveLength(2) + expect(rig.store.getRecord(SESSION)?.providerHandleChain).toHaveLength(1) + + recovered = true + const retry = await rig.host.send( + REST_TEST_CALLER, + restTestSend('after recovery', rig.store.getRecord(SESSION)!.lease.runtimeFence) + ) + expect(retry.ok).toBe(true) + await acp.frame('session/prompt') + expect(requests).toEqual([ + 'session/new', + 'session/load', + 'session/load', + 'session/load', + 'session/load' + ]) + expect((await rig.host.readOptions(SESSION)).models.map((model) => model.id)).toEqual([ + 'grok-4.7', + 'grok-4.6' + ]) + expect(rig.store.getRecord(SESSION)?.providerHandleChain).toHaveLength(2) + expect( + rig.store.getRecord(SESSION)?.providerHandleChain.map((link) => link.handle.nativeId) + ).toEqual(['acp-session-1', 'acp-session-1']) + } finally { + await acp.adapter.closeAll() + await closeProviderTimelineRigs() + await rig.dispose() + } +}) diff --git a/src/main/acp/session/acp-structured-session-adapter-resume.test.ts b/src/main/acp/session/acp-structured-session-adapter-resume.test.ts index d1a6614b1f5..eb1a36e074e 100644 --- a/src/main/acp/session/acp-structured-session-adapter-resume.test.ts +++ b/src/main/acp/session/acp-structured-session-adapter-resume.test.ts @@ -8,6 +8,7 @@ import { messageText, SESSION } from '../../native-chat/agent-session-timeline/provider-timeline-assembler-test-support' +import { tick } from '../process/testing/acp-scripted-agent.test-support' import type { AcpScriptedAgent } from '../process/testing/acp-scripted-agent.test-support' import { acpLaunchSpecFor } from '../launch/acp-launch-registry' import { @@ -16,6 +17,7 @@ import { PROVIDER_SESSION, replyChunk, sendHello, + GROK, type AcpAdapterRig } from './testing/acp-structured-adapter.test-support' @@ -224,7 +226,8 @@ describe('reattaching a Grok chat the journal holds', () => { }) describe('reopening an OpenCode chat whose conversation the agent moved to another folder', () => { - it('reopens it in the folder OpenCode lists, starting no new session', async () => { + it('reopens it in the folder OpenCode lists after a temporary listing failure, starting no new session', async () => { + let listingFailed = false const rig = await openAcpAdapterRig({ spec: acpLaunchSpecFor('opencode2')!, launch: { @@ -251,14 +254,21 @@ describe('reopening an OpenCode chat whose conversation the agent moved to anoth } agent.reply(frame, { configOptions: GROK_CONFIG_OPTIONS }) }) - agent.on('session/list', (frame) => + agent.on('session/list', (frame) => { + if (!listingFailed) { + listingFailed = true + agent.fail(frame, -32603, 'Internal service failure', { + errorName: 'ServiceUnavailableError' + }) + return + } agent.reply(frame, { sessions: [ { sessionId: 'ses_other', cwd: '/workspace/project' }, { sessionId: 'ses_moved', cwd: '/workspace/project-worktree' } ] }) - ) + }) } }) await rig.acquire() @@ -275,3 +285,109 @@ describe('reopening an OpenCode chat whose conversation the agent moved to anoth expect(await notRestoredRows(rig)).toHaveLength(0) }) }) + +describe('temporary saved-session restore failures', () => { + it.each(['load', 'resume'] as const)( + 'keeps the saved session after exhausted %s retries and restores options and sending on retry', + async (method) => { + let recovering = false + let failedRecovery = false + const requests: string[] = [] + const rig = await openAcpAdapterRig({ + ...resume, + spec: { ...GROK, resumePreference: method === 'resume' ? 'resume' : undefined }, + initialize: RESUMES, + script: (agent) => { + agent.on('session/new', (frame) => { + requests.push('session/new') + agent.reply(frame, { sessionId: 'replacement' }) + }) + agent.on(`session/${method}`, (frame) => { + requests.push(`session/${method}`) + if (method === 'load') { + agent.notify('session/update', { + sessionId: PROVIDER_SESSION, + update: { + sessionUpdate: 'user_message_chunk', + content: { type: 'text', text: 'saved question' } + }, + _meta: { promptId: 'saved-turn', isReplay: true } + }) + agent.notify( + 'session/update', + replyChunk('saved-turn', 'saved answer', { isReplay: true }) + ) + } + if (!recovering || !failedRecovery) { + failedRecovery = recovering + agent.fail(frame, -32603, 'Internal service failure', { + errorName: 'ServiceUnavailableError' + }) + } else { + agent.reply(frame, { configOptions: GROK_CONFIG_OPTIONS }) + } + }) + } + }) + await rig.acquire() + expect(rig.openVerdict()).toMatchObject({ retry: 'never', fact: { kind: 'startFailed' } }) + expect(requests).toEqual([`session/${method}`, `session/${method}`, `session/${method}`]) + expect(await texts(rig)).toEqual([]) + expect(await notRestoredRows(rig)).toEqual([]) + + recovering = true + await rig.acquire({ fence: 2 }) + expect(rig.openVerdict()).toBeUndefined() + expect(rig.started().link).toMatchObject({ + origin: 'resumed', + handle: { nativeId: 'acp-session-1' } + }) + expect(rig.started().link?.replaces).toBeUndefined() + expect(rig.started().catalogListing?.models.map((model) => model.id)).toEqual([ + 'grok-4.7', + 'grok-4.6' + ]) + expect(await texts(rig)).toEqual(method === 'load' ? ['saved question', 'saved answer'] : []) + expect(requests).toEqual([ + `session/${method}`, + `session/${method}`, + `session/${method}`, + `session/${method}`, + `session/${method}` + ]) + expect(await sendHello(rig, 'after-retry', 2)).toMatchObject({ state: 'admitted' }) + const prompt = await rig.frame('session/prompt') + expect(prompt.params).toMatchObject({ sessionId: 'acp-session-1' }) + rig.child().agent.reply(prompt, { stopReason: 'end_turn' }) + await rig.settle() + expect(await notRestoredRows(rig)).toEqual([]) + } + ) + + it('sends no further restore or new-session request after abort during backoff', async () => { + const controller = new AbortController() + const requests: string[] = [] + const rig = await openAcpAdapterRig({ + ...resume, + script: (agent) => { + agent.on('session/new', (frame) => { + requests.push('session/new') + agent.reply(frame, { sessionId: 'replacement' }) + }) + agent.on('session/load', (frame) => { + requests.push('session/load') + agent.fail(frame, -32603, 'Internal service failure', { + errorName: 'ServiceUnavailableError' + }) + }) + } + }) + const opening = rig.acquire({ signal: controller.signal }) + await rig.frame('session/load') + await tick() + controller.abort() + await opening + expect(requests).toEqual(['session/load']) + expect(await texts(rig)).toEqual([]) + }) +}) diff --git a/src/main/acp/session/start/acp-session-reopen-failure.ts b/src/main/acp/session/start/acp-session-reopen-failure.ts index aebdc590c76..3bf3eeab545 100644 --- a/src/main/acp/session/start/acp-session-reopen-failure.ts +++ b/src/main/acp/session/start/acp-session-reopen-failure.ts @@ -1,17 +1,21 @@ import { z } from 'zod' +import { AgentSessionStartFailure } from '../../../native-chat/agent-session-wire/startup/structured-agent-session-start-failure' import { agentSessionFailureFact } from '../../../../shared/agent-session/failures/agent-session-failure' import { agentSessionFailureWords } from '../../../../shared/agent-session/failures/agent-session-failure-words' import type { AgentSessionProviderHandleLink } from '../../../../shared/agent-session/lifecycle/provider-handles/agent-session-provider-handle' import type { ProviderTimelineEvent } from '../../../native-chat/agent-session-timeline/provider-timeline-event' -import { AcpAgentError, AcpAuthRequiredError, AcpRpcError } from '../../transport/acp-errors' +import { + AcpAgentError, + AcpAuthRequiredError, + AcpRpcError, + temporaryAcpProviderAnswer +} from '../../transport/acp-errors' import type { AcpDialect } from '../../acp-dialects/acp-dialect' import type { InitializeResponse } from '../../generated/acp-protocol.generated' import type { AcpStructuredConnection } from '../../process/acp-structured-connection' import type { AcpStructuredLaunch } from '../../launch/acp-structured-launch-resolution' import { providerRequestTimeoutOf } from '../../../provider-process/provider-request-timeout' -import { temporaryAcpStartupAnswer } from './acp-startup-outcome' - const ACP_RESOURCE_NOT_FOUND = -32002 // Without a `cwd`, OpenCode 2.x lists every folder's sessions. const sessionListPage = z.object({ @@ -25,14 +29,13 @@ export async function acpReopenTakeover( resume: NonNullable, start: { over: boolean; now: number; warn: (fields: { scope: string; error: unknown }) => void } ): Promise> { + if (temporaryAcpProviderAnswer(error)) { + // Restore already spent its retry budget; an explicit retry gets a fresh flight. + throw new AgentSessionStartFailure(error, 'never') + } // A replacement session cannot resolve sign-in failure, a missed deadline or an abandoned start; // its own answer would replace the failure the user must see. - if ( - error instanceof AcpAuthRequiredError || - temporaryAcpStartupAnswer(error) || - providerRequestTimeoutOf(error) || - start.over - ) { + if (error instanceof AcpAuthRequiredError || providerRequestTimeoutOf(error) || start.over) { throw error } if ( diff --git a/src/main/acp/session/start/acp-startup-outcome.ts b/src/main/acp/session/start/acp-startup-outcome.ts index 110564b0dea..a5ee98c3ae7 100644 --- a/src/main/acp/session/start/acp-startup-outcome.ts +++ b/src/main/acp/session/start/acp-startup-outcome.ts @@ -5,29 +5,18 @@ import { type AgentSessionStartAnswer } from '../../../native-chat/agent-session-wire/startup/structured-agent-session-start-failure' import { - AcpAgentError, + temporaryAcpProviderAnswer, AcpAgentExitedError, AcpFrameTooLargeError, AcpRpcError } from '../../transport/acp-errors' -export function temporaryAcpStartupAnswer(error: unknown): boolean { - return ( - error instanceof AcpAgentError && - error.code === -32603 && - typeof error.data === 'object' && - error.data !== null && - 'errorName' in error.data && - (error.data.errorName === 'ServerUnavailable' || error.data.errorName === 'CatalogNotReady') - ) -} - /** An agent's answer, sign-in included, is final unless it declared a temporary one; a frame * Orca could not read is final too. Only the process's own drained exit is unanswered: a closed * connection alone may be Orca refusing what the agent wrote. */ export const acpStartupAnswer: AgentSessionStartAnswer = (cause) => { if (cause instanceof AcpRpcError) { - return temporaryAcpStartupAnswer(cause) ? 'temporary' : 'final' + return temporaryAcpProviderAnswer(cause) ? 'temporary' : 'final' } if (cause instanceof AcpFrameTooLargeError) { return 'final' diff --git a/src/main/acp/session/start/acp-structured-handshake.ts b/src/main/acp/session/start/acp-structured-handshake.ts index 2361aebdb9c..d1ffdb41b6a 100644 --- a/src/main/acp/session/start/acp-structured-handshake.ts +++ b/src/main/acp/session/start/acp-structured-handshake.ts @@ -1,4 +1,5 @@ import { withStructuredHistoryInvocation } from '../../../native-chat/agent-session-wire/acquisition/structured-agent-session-history-invocation' +import { retryAcpRequest } from '../../protocol/acp-request-retry' import { observeAcpForkSupport } from '../../capabilities/acp-fork-support' import { withAcpStartupDiagnostic } from './acp-startup-diagnostic' // An ACP child's handshake, run after the host has published it: initialize, reopen the chat's @@ -105,33 +106,41 @@ export async function initializeAcpStructuredSession( options.adoptSession(accepted.response, accepted.kind === 'new' ? 'new' : 'loaded') } if (resume) { - const reopenIn = async (cwd: string, attaching: AcpStructuredLane) => { - const opened = await connection.start( - { cwd, mcpServers: [], sessionId: resume.sessionId, ...setup }, - acceptSession + const reopenIn = (cwd: string) => + retryAcpRequest( + async () => { + liveLane?.dispose() + input.lanes.clear() + const attaching = input.lanes.create(resume.sessionId, true) + liveLane = attaching + const opened = await connection.start( + { cwd, mcpServers: [], sessionId: resume.sessionId, ...setup }, + acceptSession + ) + assertStarting() + const frames = input.lanes.finishLoad() + attaching.translator.finishLoad() + return { opened, frames, lane: attaching } + }, + { signal: input.signal, assertCurrent: assertStarting } ) - assertStarting() - const frames = input.lanes.finishLoad() - attaching.translator.finishLoad() - return { opened, frames } - } - liveLane = input.lanes.create(resume.sessionId, true) try { - ;({ opened: started, frames: replayed } = await reopenIn(launch.cwd, liveLane)) + ;({ opened: started, frames: replayed, lane: liveLane } = await reopenIn(launch.cwd)) } catch (refused) { let error: unknown = refused try { - const movedTo = await acpSessionOtherFolder(refused, resume.sessionId, launch.cwd, { - dialect: spec.dialect, - initialized, - connection, - assertStarting - }) + const movedTo = await retryAcpRequest( + () => + acpSessionOtherFolder(refused, resume.sessionId, launch.cwd, { + dialect: spec.dialect, + initialized, + connection, + assertStarting + }), + { signal: input.signal, assertCurrent: assertStarting } + ) if (movedTo) { - liveLane.dispose() - input.lanes.clear() - liveLane = input.lanes.create(resume.sessionId, true) - ;({ opened: started, frames: replayed } = await reopenIn(movedTo, liveLane)) + ;({ opened: started, frames: replayed, lane: liveLane } = await reopenIn(movedTo)) } } catch (followFailed) { // A failed lookup or reopen gets the same takeover policy as the first refusal. diff --git a/src/main/acp/transport/acp-errors.ts b/src/main/acp/transport/acp-errors.ts index 2ee2afbf417..5a6e28334f6 100644 --- a/src/main/acp/transport/acp-errors.ts +++ b/src/main/acp/transport/acp-errors.ts @@ -35,6 +35,20 @@ export class AcpAgentError extends AcpRpcError { } } +/** Only an agent-declared temporary answer permits retrying a saved conversation. */ +export function temporaryAcpProviderAnswer(error: unknown): boolean { + return ( + error instanceof AcpAgentError && + error.code === -32603 && + typeof error.data === 'object' && + error.data !== null && + 'errorName' in error.data && + (error.data.errorName === 'ServerUnavailable' || + error.data.errorName === 'CatalogNotReady' || + error.data.errorName === 'ServiceUnavailableError') + ) +} + export class AcpAuthRequiredError extends AcpAgentError { constructor( message: string,