fix(acp): keep saved chats retryable after temporary restore failures (#42981)

* test(acp): cover temporary restore recovery and cancellation

Co-authored-by: outworkAnton <anton.mazurov@reksoft.com>

* fix(acp): preserve saved conversations through temporary restore failures

Share temporary provider-answer classification and bounded abort-aware retries. Isolate each load attempt so partial failed replay cannot enter successful history catch-up.

Co-authored-by: outworkAnton <anton.mazurov@reksoft.com>

* fix(acp): end automatic startup after the restore retry budget

* test(acp): use the advertised resume preference type

* fix(acp): retry temporary moved-session lookup failures within the same flight

---------

Co-authored-by: outworkAnton <anton.mazurov@reksoft.com>
This commit is contained in:
Brennan Benson
2026-11-16 02:05:16 -08:00
committed by GitHub
co-authored by outworkAnton
parent ba1f8f9eb4
commit cbd75eb008
8 changed files with 327 additions and 59 deletions
@@ -1,4 +1,4 @@
import { setTimeout as delay } from 'node:timers/promises'
import { retryAcpRequest } from '../protocol/acp-request-retry'
import { observeAcpForkSupport } from '../capabilities/acp-fork-support'
import {
assertStructuredAgentResolutionCurrent,
@@ -39,18 +39,14 @@ async function startListingSession(
workspace: string,
operation?: StructuredAgentResolutionOperation
): Promise<AcpSessionStarted> {
for (const backoffMs of [250, 500]) {
try {
return await connection.start({ cwd: workspace, mcpServers: [] })
} catch (error) {
if (!(error instanceof AcpAgentError && error.code === -32603)) {
throw error
}
}
await delay(backoffMs, undefined, { signal: operation?.signal })
assertStructuredAgentResolutionCurrent(operation)
}
return connection.start({ cwd: workspace, mcpServers: [] })
return retryAcpRequest(
() => connection.start({ cwd: workspace, mcpServers: [] }),
{
signal: operation?.signal,
assertCurrent: () => assertStructuredAgentResolutionCurrent(operation)
},
(error) => error instanceof AcpAgentError && error.code === -32603
)
}
/** Lists models from a disposable session, learning menus for the models it starts on or switches to. */
@@ -0,0 +1,27 @@
import { setTimeout as delay } from 'node:timers/promises'
import { temporaryAcpProviderAnswer } from '../transport/acp-errors'
/** A bounded retry, owned by the operation that can still use the answer. */
export async function retryAcpRequest<T>(
request: () => Promise<T>,
operation: { signal?: AbortSignal; assertCurrent: () => void },
retryable: (error: unknown) => boolean = temporaryAcpProviderAnswer
): Promise<T> {
const assertCurrent = () => {
operation.signal?.throwIfAborted()
operation.assertCurrent()
}
for (const backoffMs of [250, 500]) {
assertCurrent()
try {
return await request()
} catch (error) {
if (!retryable(error)) {
throw error
}
}
await delay(backoffMs, undefined, { signal: operation.signal })
}
assertCurrent()
return request()
}
@@ -0,0 +1,114 @@
import { expect, it } from 'vitest'
import { readTestHostJournal } from '../../native-chat/agent-session-journal/journal-observation.test-support'
import { closeProviderTimelineRigs } from '../../native-chat/agent-session-timeline/provider-timeline-assembler-test-support'
import {
HOST_TEST_SESSION as SESSION,
hostTestAttachParams
} from '../../native-chat/agent-session-wire/host/testing/structured-agent-session-host-test-data.test-fixture'
import {
createRestTestRig,
REST_TEST_CALLER,
restTestSend,
type RestTestRig
} from '../../native-chat/agent-session-wire/host/testing/structured-agent-session-rest-test-rig.test-fixture'
import { StructuredAgentRegistry } from '../../native-chat/agent-session-wire/providers/structured-agent-registry'
import { acpStructuredAgentDefinition } from './acp-structured-agent-definitions'
import {
GROK,
GROK_CONFIG_OPTIONS,
openAcpAdapterRig,
waitFor
} from './testing/acp-structured-adapter.test-support'
import { launch } from './testing/acp-structured-host.test-support'
it('ends an exhausted restore flight without multiplying retries and allows a new same-chat send', async () => {
let rig: RestTestRig | undefined
let restoring = false
let recovered = false
const requests: string[] = []
const acp = await openAcpAdapterRig({
deps: {
resolveLaunch: launch(() => restoring),
onEvent: (event) => rig?.host.handleAdapterEvent(event),
onDispatchSettledLate: (settlement) => void rig?.host.settleLateDispatch(settlement)
},
script: (agent) => {
agent.on('session/new', (frame) => {
requests.push('session/new')
agent.reply(frame, { sessionId: 'acp-session-1', configOptions: GROK_CONFIG_OPTIONS })
})
agent.on('session/load', (frame) => {
requests.push('session/load')
if (recovered) {
agent.reply(frame, { configOptions: GROK_CONFIG_OPTIONS })
} else {
agent.fail(frame, -32603, 'Internal service failure', {
errorName: 'ServiceUnavailableError'
})
}
})
agent.on('session/prompt', (frame) => agent.reply(frame, { stopReason: 'end_turn' }))
}
})
rig = await createRestTestRig({
adapter: acp.adapter,
agents: new StructuredAgentRegistry([
{ definition: acpStructuredAgentDefinition(GROK), adapter: acp.adapter }
])
})
rig.host.collaboratorsForTests().lifetime.idleSweep.dispose()
try {
const attached = await rig.host.attach(
REST_TEST_CALLER,
hostTestAttachParams(null, {
provider: 'grok',
agent: 'grok',
accountHome: { variable: 'GROK_HOME', path: rig.root },
providerHandle: undefined
})
)
expect(attached.ok).toBe(true)
await waitFor(() => expect(rig!.store.getRecord(SESSION)?.providerHandleChain).toHaveLength(1))
await rig.host.close(SESSION, 'evict')
restoring = true
const failed = await rig.host.send(
REST_TEST_CALLER,
restTestSend('during outage', rig.store.getRecord(SESSION)!.lease.runtimeFence)
)
expect(failed.ok).toBe(true)
await waitFor(async () => {
const snapshot = await readTestHostJournal(rig!.host, SESSION)
expect(snapshot.submissions[0]?.dispatchState).toBe('rejected')
})
expect(requests).toEqual(['session/new', 'session/load', 'session/load', 'session/load'])
expect(acp.spawned.filter((step) => step === 'spawn')).toHaveLength(2)
expect(rig.store.getRecord(SESSION)?.providerHandleChain).toHaveLength(1)
recovered = true
const retry = await rig.host.send(
REST_TEST_CALLER,
restTestSend('after recovery', rig.store.getRecord(SESSION)!.lease.runtimeFence)
)
expect(retry.ok).toBe(true)
await acp.frame('session/prompt')
expect(requests).toEqual([
'session/new',
'session/load',
'session/load',
'session/load',
'session/load'
])
expect((await rig.host.readOptions(SESSION)).models.map((model) => model.id)).toEqual([
'grok-4.7',
'grok-4.6'
])
expect(rig.store.getRecord(SESSION)?.providerHandleChain).toHaveLength(2)
expect(
rig.store.getRecord(SESSION)?.providerHandleChain.map((link) => link.handle.nativeId)
).toEqual(['acp-session-1', 'acp-session-1'])
} finally {
await acp.adapter.closeAll()
await closeProviderTimelineRigs()
await rig.dispose()
}
})
@@ -8,6 +8,7 @@ import {
messageText,
SESSION
} from '../../native-chat/agent-session-timeline/provider-timeline-assembler-test-support'
import { tick } from '../process/testing/acp-scripted-agent.test-support'
import type { AcpScriptedAgent } from '../process/testing/acp-scripted-agent.test-support'
import { acpLaunchSpecFor } from '../launch/acp-launch-registry'
import {
@@ -16,6 +17,7 @@ import {
PROVIDER_SESSION,
replyChunk,
sendHello,
GROK,
type AcpAdapterRig
} from './testing/acp-structured-adapter.test-support'
@@ -224,7 +226,8 @@ describe('reattaching a Grok chat the journal holds', () => {
})
describe('reopening an OpenCode chat whose conversation the agent moved to another folder', () => {
it('reopens it in the folder OpenCode lists, starting no new session', async () => {
it('reopens it in the folder OpenCode lists after a temporary listing failure, starting no new session', async () => {
let listingFailed = false
const rig = await openAcpAdapterRig({
spec: acpLaunchSpecFor('opencode2')!,
launch: {
@@ -251,14 +254,21 @@ describe('reopening an OpenCode chat whose conversation the agent moved to anoth
}
agent.reply(frame, { configOptions: GROK_CONFIG_OPTIONS })
})
agent.on('session/list', (frame) =>
agent.on('session/list', (frame) => {
if (!listingFailed) {
listingFailed = true
agent.fail(frame, -32603, 'Internal service failure', {
errorName: 'ServiceUnavailableError'
})
return
}
agent.reply(frame, {
sessions: [
{ sessionId: 'ses_other', cwd: '/workspace/project' },
{ sessionId: 'ses_moved', cwd: '/workspace/project-worktree' }
]
})
)
})
}
})
await rig.acquire()
@@ -275,3 +285,109 @@ describe('reopening an OpenCode chat whose conversation the agent moved to anoth
expect(await notRestoredRows(rig)).toHaveLength(0)
})
})
describe('temporary saved-session restore failures', () => {
it.each(['load', 'resume'] as const)(
'keeps the saved session after exhausted %s retries and restores options and sending on retry',
async (method) => {
let recovering = false
let failedRecovery = false
const requests: string[] = []
const rig = await openAcpAdapterRig({
...resume,
spec: { ...GROK, resumePreference: method === 'resume' ? 'resume' : undefined },
initialize: RESUMES,
script: (agent) => {
agent.on('session/new', (frame) => {
requests.push('session/new')
agent.reply(frame, { sessionId: 'replacement' })
})
agent.on(`session/${method}`, (frame) => {
requests.push(`session/${method}`)
if (method === 'load') {
agent.notify('session/update', {
sessionId: PROVIDER_SESSION,
update: {
sessionUpdate: 'user_message_chunk',
content: { type: 'text', text: 'saved question' }
},
_meta: { promptId: 'saved-turn', isReplay: true }
})
agent.notify(
'session/update',
replyChunk('saved-turn', 'saved answer', { isReplay: true })
)
}
if (!recovering || !failedRecovery) {
failedRecovery = recovering
agent.fail(frame, -32603, 'Internal service failure', {
errorName: 'ServiceUnavailableError'
})
} else {
agent.reply(frame, { configOptions: GROK_CONFIG_OPTIONS })
}
})
}
})
await rig.acquire()
expect(rig.openVerdict()).toMatchObject({ retry: 'never', fact: { kind: 'startFailed' } })
expect(requests).toEqual([`session/${method}`, `session/${method}`, `session/${method}`])
expect(await texts(rig)).toEqual([])
expect(await notRestoredRows(rig)).toEqual([])
recovering = true
await rig.acquire({ fence: 2 })
expect(rig.openVerdict()).toBeUndefined()
expect(rig.started().link).toMatchObject({
origin: 'resumed',
handle: { nativeId: 'acp-session-1' }
})
expect(rig.started().link?.replaces).toBeUndefined()
expect(rig.started().catalogListing?.models.map((model) => model.id)).toEqual([
'grok-4.7',
'grok-4.6'
])
expect(await texts(rig)).toEqual(method === 'load' ? ['saved question', 'saved answer'] : [])
expect(requests).toEqual([
`session/${method}`,
`session/${method}`,
`session/${method}`,
`session/${method}`,
`session/${method}`
])
expect(await sendHello(rig, 'after-retry', 2)).toMatchObject({ state: 'admitted' })
const prompt = await rig.frame('session/prompt')
expect(prompt.params).toMatchObject({ sessionId: 'acp-session-1' })
rig.child().agent.reply(prompt, { stopReason: 'end_turn' })
await rig.settle()
expect(await notRestoredRows(rig)).toEqual([])
}
)
it('sends no further restore or new-session request after abort during backoff', async () => {
const controller = new AbortController()
const requests: string[] = []
const rig = await openAcpAdapterRig({
...resume,
script: (agent) => {
agent.on('session/new', (frame) => {
requests.push('session/new')
agent.reply(frame, { sessionId: 'replacement' })
})
agent.on('session/load', (frame) => {
requests.push('session/load')
agent.fail(frame, -32603, 'Internal service failure', {
errorName: 'ServiceUnavailableError'
})
})
}
})
const opening = rig.acquire({ signal: controller.signal })
await rig.frame('session/load')
await tick()
controller.abort()
await opening
expect(requests).toEqual(['session/load'])
expect(await texts(rig)).toEqual([])
})
})
@@ -1,17 +1,21 @@
import { z } from 'zod'
import { AgentSessionStartFailure } from '../../../native-chat/agent-session-wire/startup/structured-agent-session-start-failure'
import { agentSessionFailureFact } from '../../../../shared/agent-session/failures/agent-session-failure'
import { agentSessionFailureWords } from '../../../../shared/agent-session/failures/agent-session-failure-words'
import type { AgentSessionProviderHandleLink } from '../../../../shared/agent-session/lifecycle/provider-handles/agent-session-provider-handle'
import type { ProviderTimelineEvent } from '../../../native-chat/agent-session-timeline/provider-timeline-event'
import { AcpAgentError, AcpAuthRequiredError, AcpRpcError } from '../../transport/acp-errors'
import {
AcpAgentError,
AcpAuthRequiredError,
AcpRpcError,
temporaryAcpProviderAnswer
} from '../../transport/acp-errors'
import type { AcpDialect } from '../../acp-dialects/acp-dialect'
import type { InitializeResponse } from '../../generated/acp-protocol.generated'
import type { AcpStructuredConnection } from '../../process/acp-structured-connection'
import type { AcpStructuredLaunch } from '../../launch/acp-structured-launch-resolution'
import { providerRequestTimeoutOf } from '../../../provider-process/provider-request-timeout'
import { temporaryAcpStartupAnswer } from './acp-startup-outcome'
const ACP_RESOURCE_NOT_FOUND = -32002
// Without a `cwd`, OpenCode 2.x lists every folder's sessions.
const sessionListPage = z.object({
@@ -25,14 +29,13 @@ export async function acpReopenTakeover(
resume: NonNullable<AcpStructuredLaunch['resume']>,
start: { over: boolean; now: number; warn: (fields: { scope: string; error: unknown }) => void }
): Promise<Pick<AgentSessionProviderHandleLink, 'supersedesKey' | 'replaces'>> {
if (temporaryAcpProviderAnswer(error)) {
// Restore already spent its retry budget; an explicit retry gets a fresh flight.
throw new AgentSessionStartFailure(error, 'never')
}
// A replacement session cannot resolve sign-in failure, a missed deadline or an abandoned start;
// its own answer would replace the failure the user must see.
if (
error instanceof AcpAuthRequiredError ||
temporaryAcpStartupAnswer(error) ||
providerRequestTimeoutOf(error) ||
start.over
) {
if (error instanceof AcpAuthRequiredError || providerRequestTimeoutOf(error) || start.over) {
throw error
}
if (
@@ -5,29 +5,18 @@ import {
type AgentSessionStartAnswer
} from '../../../native-chat/agent-session-wire/startup/structured-agent-session-start-failure'
import {
AcpAgentError,
temporaryAcpProviderAnswer,
AcpAgentExitedError,
AcpFrameTooLargeError,
AcpRpcError
} from '../../transport/acp-errors'
export function temporaryAcpStartupAnswer(error: unknown): boolean {
return (
error instanceof AcpAgentError &&
error.code === -32603 &&
typeof error.data === 'object' &&
error.data !== null &&
'errorName' in error.data &&
(error.data.errorName === 'ServerUnavailable' || error.data.errorName === 'CatalogNotReady')
)
}
/** An agent's answer, sign-in included, is final unless it declared a temporary one; a frame
* Orca could not read is final too. Only the process's own drained exit is unanswered: a closed
* connection alone may be Orca refusing what the agent wrote. */
export const acpStartupAnswer: AgentSessionStartAnswer = (cause) => {
if (cause instanceof AcpRpcError) {
return temporaryAcpStartupAnswer(cause) ? 'temporary' : 'final'
return temporaryAcpProviderAnswer(cause) ? 'temporary' : 'final'
}
if (cause instanceof AcpFrameTooLargeError) {
return 'final'
@@ -1,4 +1,5 @@
import { withStructuredHistoryInvocation } from '../../../native-chat/agent-session-wire/acquisition/structured-agent-session-history-invocation'
import { retryAcpRequest } from '../../protocol/acp-request-retry'
import { observeAcpForkSupport } from '../../capabilities/acp-fork-support'
import { withAcpStartupDiagnostic } from './acp-startup-diagnostic'
// An ACP child's handshake, run after the host has published it: initialize, reopen the chat's
@@ -105,33 +106,41 @@ export async function initializeAcpStructuredSession(
options.adoptSession(accepted.response, accepted.kind === 'new' ? 'new' : 'loaded')
}
if (resume) {
const reopenIn = async (cwd: string, attaching: AcpStructuredLane) => {
const opened = await connection.start(
{ cwd, mcpServers: [], sessionId: resume.sessionId, ...setup },
acceptSession
const reopenIn = (cwd: string) =>
retryAcpRequest(
async () => {
liveLane?.dispose()
input.lanes.clear()
const attaching = input.lanes.create(resume.sessionId, true)
liveLane = attaching
const opened = await connection.start(
{ cwd, mcpServers: [], sessionId: resume.sessionId, ...setup },
acceptSession
)
assertStarting()
const frames = input.lanes.finishLoad()
attaching.translator.finishLoad()
return { opened, frames, lane: attaching }
},
{ signal: input.signal, assertCurrent: assertStarting }
)
assertStarting()
const frames = input.lanes.finishLoad()
attaching.translator.finishLoad()
return { opened, frames }
}
liveLane = input.lanes.create(resume.sessionId, true)
try {
;({ opened: started, frames: replayed } = await reopenIn(launch.cwd, liveLane))
;({ opened: started, frames: replayed, lane: liveLane } = await reopenIn(launch.cwd))
} catch (refused) {
let error: unknown = refused
try {
const movedTo = await acpSessionOtherFolder(refused, resume.sessionId, launch.cwd, {
dialect: spec.dialect,
initialized,
connection,
assertStarting
})
const movedTo = await retryAcpRequest(
() =>
acpSessionOtherFolder(refused, resume.sessionId, launch.cwd, {
dialect: spec.dialect,
initialized,
connection,
assertStarting
}),
{ signal: input.signal, assertCurrent: assertStarting }
)
if (movedTo) {
liveLane.dispose()
input.lanes.clear()
liveLane = input.lanes.create(resume.sessionId, true)
;({ opened: started, frames: replayed } = await reopenIn(movedTo, liveLane))
;({ opened: started, frames: replayed, lane: liveLane } = await reopenIn(movedTo))
}
} catch (followFailed) {
// A failed lookup or reopen gets the same takeover policy as the first refusal.
+14
View File
@@ -35,6 +35,20 @@ export class AcpAgentError extends AcpRpcError {
}
}
/** Only an agent-declared temporary answer permits retrying a saved conversation. */
export function temporaryAcpProviderAnswer(error: unknown): boolean {
return (
error instanceof AcpAgentError &&
error.code === -32603 &&
typeof error.data === 'object' &&
error.data !== null &&
'errorName' in error.data &&
(error.data.errorName === 'ServerUnavailable' ||
error.data.errorName === 'CatalogNotReady' ||
error.data.errorName === 'ServiceUnavailableError')
)
}
export class AcpAuthRequiredError extends AcpAgentError {
constructor(
message: string,