diff --git a/src/main/claude/claude-structured-dispatch-test-support.ts b/src/main/claude/claude-structured-dispatch-test-support.ts index a309bd9d0b9..5af0f17243b 100644 --- a/src/main/claude/claude-structured-dispatch-test-support.ts +++ b/src/main/claude/claude-structured-dispatch-test-support.ts @@ -5,7 +5,7 @@ import type { ClaudeSession } from './claude-structured-session-state' import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import { ClaudeChildWorkDecoder } from './claude-child-work-decoder' import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' -import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate' +import { createClaudeSessionStartup } from './claude-structured-session-startup-state' export function sessionFor(send: Mock = vi.fn().mockResolvedValue(undefined)): ClaudeSession { return { @@ -32,7 +32,7 @@ export function sessionFor(send: Mock = vi.fn().mockResolvedValue(undefined)): C capabilities: [], events: undefined, translator: null, - startup: { ...createClaudeSessionStartupGate(), state: 'proven' } + startup: { ...createClaudeSessionStartup(), state: 'proven' } } } diff --git a/src/main/claude/claude-structured-dispatch.ts b/src/main/claude/claude-structured-dispatch.ts index d93a8ecab0a..5a5b2796595 100644 --- a/src/main/claude/claude-structured-dispatch.ts +++ b/src/main/claude/claude-structured-dispatch.ts @@ -35,10 +35,8 @@ import { import { AgentSessionPreDispatchError } from '../native-chat/agent-session-wire/structured-agent-session-operation-settlement' import { claudeStartupFailureReason, - claudeStartupHoldsWrites, - failClaudeStartupGate, - holdClaudeStartupWrite -} from './claude-structured-session-startup-gate' + failClaudeStartup +} from './claude-structured-session-startup-state' const MAX_ACTIVE_DISPATCH_WAITERS = 64 @@ -226,7 +224,7 @@ export function settleCancelledClaudeDispatchWaiters( * Retired rather than dropped: their identities stay joinable, bounded by * `MAX_RETIRED_DISPATCH_WAITERS`. */ export function retireClaudeDispatchWaiters(session: ClaudeSession): void { - failClaudeStartupGate(session, new Error('claude stream-json ended before startup completed')) + failClaudeStartup(session, new Error('claude stream-json ended before startup completed')) for (const waiter of session.dispatchWaiters.splice(0)) { retireWaiter(session, waiter) waiter.resolve(null) @@ -236,8 +234,7 @@ export function retireClaudeDispatchWaiters(session: ClaudeSession): void { export async function dispatchClaudeTurn( session: ClaudeSession, input: { clientMessageId?: string; body: AgentJournalMessageItem; requestedAt?: number }, - beforeDispatch?: () => Promise, - onSettledLate?: ClaudeLateDispatchSettlement + beforeDispatch?: () => Promise ): Promise { let content: unknown[] try { @@ -276,14 +273,6 @@ export async function dispatchClaudeTurn( parent_tool_use_id: null, session_id: session.providerSessionId } - if (claudeStartupHoldsWrites(session)) { - return holdClaudeStartupWrite(session, { - message, - arm, - ...(beforeDispatch ? { beforeDispatch } : {}), - ...(onSettledLate ? { settleLate: onSettledLate } : {}) - }) - } const pending = { replay: beforeDispatch ? undefined : arm() } const authorize = beforeDispatch ? async () => { diff --git a/src/main/claude/claude-structured-options.test.ts b/src/main/claude/claude-structured-options.test.ts index e02647f3625..a92149a2cdd 100644 --- a/src/main/claude/claude-structured-options.test.ts +++ b/src/main/claude/claude-structured-options.test.ts @@ -11,7 +11,7 @@ import { import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import { ClaudeChildWorkDecoder } from './claude-child-work-decoder' import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' -import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate' +import { createClaudeSessionStartup } from './claude-structured-session-startup-state' import { claudeStructuredSessionOptionsFrom, observeClaudeFastModeFacts, @@ -48,7 +48,7 @@ function sessionFor(setModel: ClaudeSession['connection']['setModel']): ClaudeSe capabilities: [], events: undefined, translator: null, - startup: { ...createClaudeSessionStartupGate(), state: 'proven' } + startup: { ...createClaudeSessionStartup(), state: 'proven' } } } diff --git a/src/main/claude/claude-structured-prompt-ownership.ts b/src/main/claude/claude-structured-prompt-ownership.ts index 885ea2dfa74..864a99a0b77 100644 --- a/src/main/claude/claude-structured-prompt-ownership.ts +++ b/src/main/claude/claude-structured-prompt-ownership.ts @@ -15,11 +15,6 @@ import { buildClaudePromptReply } from './claude-structured-prompt-replies' import type { ClaudeSession } from './claude-structured-session-state' import type { ClaudePendingPrompt } from './claude-prompt-registry' import type { PermissionResult } from '@anthropic-ai/claude-agent-sdk' -import { - claudeStartupHoldsWrites, - rejectClaudeStartupWrites -} from './claude-structured-session-startup-gate' -import { DISPATCH_REJECTED_CANCELLED } from '../../shared/structured-agent-session-dispatch-rejection' /** Conservative user-facing window: below the 30s control deadline, trading * residual slow-pump risk for ensuring delivery bookkeeping cannot block Stop indefinitely. */ @@ -107,14 +102,9 @@ export async function cancelClaudeStructuredTurn(input: { const session = requireSession(sessions, request.sessionId) const acquisitionGeneration = session.acquisitionGeneration const prompt = request.prompt - // A held prompt was never written, so Stop withdraws it; the drain only writes what it still - // holds. Before startup lands nothing was written, so there is nothing to interrupt either. - let withdrewHeld = false - if (!prompt && claudeStartupHoldsWrites(session) && session.fence === request.fence) { - withdrewHeld = rejectClaudeStartupWrites(session, DISPATCH_REJECTED_CANCELLED) - if (session.startup.state === 'pending') { - return { cancelled: withdrewHeld } - } + // Before startup lands nothing was written, so there is nothing to interrupt. + if (!prompt && session.startup.state === 'pending') { + return { cancelled: false } } if (prompt && session.fence !== request.fence) { return { cancelled: false } @@ -197,7 +187,7 @@ export async function cancelClaudeStructuredTurn(input: { } else if (claim) { session.prompts.releaseClaim(claim) } - return withdrewHeld ? { ...result, cancelled: true } : result + return result } catch (error) { if (claim && !interruptConfirmed) { session.prompts.releaseClaim(claim) diff --git a/src/main/claude/claude-structured-real-cli.test.ts b/src/main/claude/claude-structured-real-cli.test.ts index 4181dab43c7..c117e3e863f 100644 --- a/src/main/claude/claude-structured-real-cli.test.ts +++ b/src/main/claude/claude-structured-real-cli.test.ts @@ -70,7 +70,7 @@ function realAdapter( const acquire = adapter.acquire adapter.acquire = async (input) => { const acquisition = await acquire(input) - await adapter.drainStartup(input.identity.sessionId) + await adapter.awaitStarted(input.identity.sessionId) return acquisition } return adapter diff --git a/src/main/claude/claude-structured-session-acquisition.ts b/src/main/claude/claude-structured-session-acquisition.ts index b2efed691ee..6728f960ef0 100644 --- a/src/main/claude/claude-structured-session-acquisition.ts +++ b/src/main/claude/claude-structured-session-acquisition.ts @@ -248,30 +248,34 @@ export async function acquireClaudeSession({ event() } }) - session.startup.settled = settleClaudeSessionStartup({ - session, - facts: readClaudeStartupFacts({ - connection, - initProof, - sessionId, - providerSessionId: launch.providerSessionId, - resumesTranscript: launch.resumesTranscript, - inputOptions: input.options, - requestTimeoutMs: deps.requestTimeoutMs, - emit - }), - isCurrent: () => sessions.get(sessionId) === session, - requestTimeoutMs: deps.requestTimeoutMs, - fault: (error) => callbacks.handleExit(sessionId, attempt, error), - onStarted: (options) => - emit({ - type: 'started', + // Whichever comes first: the start landing or faulting, or the child being ended. + session.startup.settled = Promise.race([ + session.startup.settled, + settleClaudeSessionStartup({ + session, + facts: readClaudeStartupFacts({ + connection, + initProof, sessionId, - fence: input.fence, - acquisitionGeneration: session.acquisitionGeneration, - ...options - }) - }) + providerSessionId: launch.providerSessionId, + resumesTranscript: launch.resumesTranscript, + inputOptions: input.options, + requestTimeoutMs: deps.requestTimeoutMs, + emit + }), + isCurrent: () => sessions.get(sessionId) === session, + requestTimeoutMs: deps.requestTimeoutMs, + fault: (error) => callbacks.handleExit(sessionId, attempt, error), + onStarted: (options) => + emit({ + type: 'started', + sessionId, + fence: input.fence, + acquisitionGeneration: session.acquisitionGeneration, + ...options + }) + }) + ]) // A child whose exit already reached `handleExit` is not handed over as live: the create // fails with the CLI's own diagnostic, as one that died before publish does. if (sessions.get(sessionId) !== session) { diff --git a/src/main/claude/claude-structured-session-adapter.ts b/src/main/claude/claude-structured-session-adapter.ts index b8cc5c1dec5..5234606a1cc 100644 --- a/src/main/claude/claude-structured-session-adapter.ts +++ b/src/main/claude/claude-structured-session-adapter.ts @@ -12,7 +12,10 @@ import { acquireClaudeSession } from './claude-structured-session-acquisition' import { supportsClaudeStructuredLocation } from './claude-structured-location-support' import { setClaudeStructuredSessionOption } from './claude-structured-options' import { readClaudeStructuredSessionOptions } from './claude-structured-session-options' -import { claudeStartupSettledWithin } from './claude-structured-session-startup-gate' +import { + claudeStartupFailureReason, + claudeStartupSettledWithin +} from './claude-structured-session-startup-state' import { CLAUDE_DEFAULT_REQUEST_TIMEOUT_MS } from './claude-agent-sdk-control-requests' import { ClaudeAcquisitionRegistry, @@ -115,9 +118,16 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda * apart without guessing at wall-clock. */ drainObservedExits = (): Promise => drainClaudeObservedExits(this.exits) - /** Resolves once a published session's startup has landed or faulted it. */ - drainStartup = (sessionId: string): Promise => - this.sessions.get(sessionId)?.startup.settled ?? Promise.resolve() + /** Resolves once a published session's startup has landed, faulted, or been ended by a close; + * with the reason when it did not land. */ + awaitStarted = async (sessionId: string): Promise => { + const session = this.sessions.get(sessionId) + if (!session) { + return + } + await session.startup.settled + return claudeStartupFailureReason(session) ?? undefined + } /** Restart reconciliation reads the transcript a resume replays; these maps track liveness. */ providerHistoryWindow: NonNullable = ( @@ -179,9 +189,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda } dispatch: StructuredAgentSessionAdapter['dispatch'] = (input) => - dispatchClaudeTurn(this.session(input.sessionId), input, input.beforeDispatch, (settlement) => - this.deps.onDispatchSettledLate?.({ sessionId: input.sessionId, ...settlement }) - ) + dispatchClaudeTurn(this.session(input.sessionId), input, input.beforeDispatch) compact: NonNullable = (input) => compactClaudeSession(this.session(input.sessionId), this.compactions, input) diff --git a/src/main/claude/claude-structured-session-exit-lifecycle.ts b/src/main/claude/claude-structured-session-exit-lifecycle.ts index 4f39a8dc467..d0bddaf0a11 100644 --- a/src/main/claude/claude-structured-session-exit-lifecycle.ts +++ b/src/main/claude/claude-structured-session-exit-lifecycle.ts @@ -3,7 +3,7 @@ import { claudeRootExitObserved, settleClaudeExitedSession } from './claude-structured-session-close' -import { failClaudeStartupGate } from './claude-structured-session-startup-gate' +import { failClaudeStartup } from './claude-structured-session-startup-state' import type { ClaudeAcquisitionAttempt, ClaudeSession, @@ -32,7 +32,7 @@ export function observeClaudeSessionExit( return } lifecycle.sessions.delete(sessionId) - failClaudeStartupGate(session, error) + failClaudeStartup(session, error) // Re-enter the provider's close ladder before publishing lifecycle recovery. // An exit callback is root evidence only; the retained tree proof must run // before the host releases and reacquires this exact child. diff --git a/src/main/claude/claude-structured-session-publication.ts b/src/main/claude/claude-structured-session-publication.ts index d64273f9b8b..f20bc0c7907 100644 --- a/src/main/claude/claude-structured-session-publication.ts +++ b/src/main/claude/claude-structured-session-publication.ts @@ -6,7 +6,7 @@ import type { ClaudeSession } from './claude-structured-session-state' import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import { ClaudeChildWorkDecoder } from './claude-child-work-decoder' import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' -import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate' +import { createClaudeSessionStartup } from './claude-structured-session-startup-state' /** The session as published at spawn: nothing the CLI reports at init is assumed yet. */ export function createClaudeSessionPublication(input: { @@ -67,7 +67,7 @@ export function createClaudeSessionPublication(input: { translator: input.translator, events: input.events, ...(input.unbindReadingControl ? { unbindReadingControl: input.unbindReadingControl } : {}), - startup: createClaudeSessionStartupGate() + startup: createClaudeSessionStartup() } } } diff --git a/src/main/claude/claude-structured-session-recovery.test.ts b/src/main/claude/claude-structured-session-recovery.test.ts index ef8e0c65469..8ddc29a2e67 100644 --- a/src/main/claude/claude-structured-session-recovery.test.ts +++ b/src/main/claude/claude-structured-session-recovery.test.ts @@ -338,7 +338,7 @@ describe('ClaudeStructuredSessionAdapter close and exit recovery', () => { spawnToken: 'spawn-9', events: journalSink }) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') const first = claude.connections[0] const oldPrompt = invokeCanUseTool(first, 'Bash', 'permission-retained', 'tool-retained') const oldSession = ( diff --git a/src/main/claude/claude-structured-session-startup-gate.ts b/src/main/claude/claude-structured-session-startup-gate.ts deleted file mode 100644 index 0df78936191..00000000000 --- a/src/main/claude/claude-structured-session-startup-gate.ts +++ /dev/null @@ -1,172 +0,0 @@ -// A Claude session is published once its child is spawned, before the CLI has answered -// initialize. Prompts sent in that window are held here and written, in order, once startup -// lands (init facts read and saved options restored), so a first turn never runs under -// defaults the restore was about to replace. A held prompt was never written, so a startup -// that fails rejects it rather than leaving its delivery in doubt. - -import type { AgentSessionDispatchOutcome } from '../native-chat/agent-session-wire/structured-agent-session-adapter' -import { AgentSessionPreDispatchError } from '../native-chat/agent-session-wire/structured-agent-session-operation-settlement' -import { dispatchWriteFailureReason } from '../../shared/structured-agent-session-dispatch-rejection' -import { providerStartupFailureRejection } from '../native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement' -import { claudeUserMessageWasProvablyUnwritten } from './claude-agent-sdk-user-message-queue' -import { - forgetRetiredWaiter, - forgetWaiter, - retireWaiter -} from './claude-structured-dispatch-waiters' -import type { - ClaudeDispatchWaiter, - ClaudeLateDispatchOutcome, - ClaudeSession -} from './claude-structured-session-state' - -type ClaudeStartupHeldWrite = { - waiter: ClaudeDispatchWaiter - message: Record - settleLate?: (outcome: ClaudeLateDispatchOutcome) => void -} - -export type ClaudeSessionStartupGate = { - state: 'pending' | 'proven' | 'failed' - held: ClaudeStartupHeldWrite[] - /** Held prompts are still being written; later prompts must queue behind them. */ - draining: boolean - failure: Error | null - /** Resolves once startup has landed or faulted the session; never rejects. */ - settled: Promise -} - -export function createClaudeSessionStartupGate(): ClaudeSessionStartupGate { - return { state: 'pending', held: [], draining: false, failure: null, settled: Promise.resolve() } -} - -export function claudeStartupFailureReason(session: ClaudeSession): string | null { - return session.startup.state === 'failed' - ? providerStartupFailureRejection(session.startup.failure ?? undefined) - : null -} - -/** Resolves when startup lands or `timeoutMs` passes; a stuck start then refuses the write as before. */ -export function claudeStartupSettledWithin( - session: ClaudeSession | undefined, - timeoutMs: number -): Promise { - if (session?.startup.state !== 'pending') { - return Promise.resolve() - } - let timer: ReturnType | undefined - return Promise.race([ - session.startup.settled, - new Promise((resolve) => { - timer = setTimeout(resolve, timeoutMs) - }) - ]).finally(() => clearTimeout(timer)) -} - -export function claudeStartupHoldsWrites(session: ClaudeSession): boolean { - return session.startup.state === 'pending' || session.startup.draining -} - -/** Admits a prompt while startup is pending; it is written when `openClaudeStartupGate` runs. */ -export async function holdClaudeStartupWrite( - session: ClaudeSession, - input: { - message: Record - arm: () => { waiter: ClaudeDispatchWaiter } - beforeDispatch?: () => Promise - settleLate?: (outcome: ClaudeLateDispatchOutcome) => void - } -): Promise { - if (input.beforeDispatch) { - try { - await input.beforeDispatch() - } catch (error) { - if (error instanceof AgentSessionPreDispatchError) { - throw error - } - return { state: 'rejected', reason: dispatchWriteFailureReason(error) } - } - } - // Startup may have failed while the admission barrier ran. - const failed = claudeStartupFailureReason(session) - if (failed) { - return { state: 'rejected', reason: failed } - } - const { waiter } = input.arm() - session.startup.held.push({ - waiter, - message: input.message, - ...(input.settleLate ? { settleLate: input.settleLate } : {}) - }) - // Startup finished writing what it held while the barrier ran; nothing else would drain this. - if (!claudeStartupHoldsWrites(session)) { - void drainClaudeStartupWrites(session) - } - return { state: 'admitted' } -} - -export async function openClaudeStartupGate(session: ClaudeSession): Promise { - const gate = session.startup - if (gate.state !== 'pending') { - return - } - gate.state = 'proven' - await drainClaudeStartupWrites(session) -} - -async function drainClaudeStartupWrites(session: ClaudeSession): Promise { - const gate = session.startup - gate.draining = true - try { - for (let held = gate.held.shift(); held; held = gate.held.shift()) { - await writeHeld(session, held) - } - } finally { - gate.draining = false - } -} - -async function writeHeld(session: ClaudeSession, held: ClaudeStartupHeldWrite): Promise { - try { - await session.connection.send(held.message) - } catch (error) { - if (held.waiter.settledUuid) { - return - } - if (claudeUserMessageWasProvablyUnwritten(error)) { - rejectHeld(session, held, dispatchWriteFailureReason(error)) - return - } - // Possibly written: only a replay or the child's exit can settle it now. - retireWaiter(session, held.waiter) - held.waiter.resolve(null) - } -} - -function rejectHeld(session: ClaudeSession, held: ClaudeStartupHeldWrite, reason: string): void { - forgetWaiter(session, held.waiter) - forgetRetiredWaiter(session, held.waiter) - held.waiter.resolve(null) - if (held.waiter.clientMessageId) { - held.settleLate?.({ clientMessageId: held.waiter.clientMessageId, state: 'rejected', reason }) - } -} - -/** Rejects every held prompt with `reason`; true when any was held. */ -export function rejectClaudeStartupWrites(session: ClaudeSession, reason: string): boolean { - const held = session.startup.held.splice(0) - for (const entry of held) { - rejectHeld(session, entry, reason) - } - return held.length > 0 -} - -/** Startup cannot land any more; nothing held was written, so all of it is rejected. */ -export function failClaudeStartupGate(session: ClaudeSession, error: Error): void { - const gate = session.startup - if (gate.state === 'pending') { - gate.state = 'failed' - gate.failure = error - } - rejectClaudeStartupWrites(session, providerStartupFailureRejection(error)) -} diff --git a/src/main/claude/claude-structured-session-startup-state.ts b/src/main/claude/claude-structured-session-startup-state.ts new file mode 100644 index 00000000000..1bda560377a --- /dev/null +++ b/src/main/claude/claude-structured-session-startup-state.ts @@ -0,0 +1,58 @@ +// Where a Claude start stands. A session is published once its child is spawned, before the CLI +// has answered initialize. Nothing is written to it until startup lands (init facts read and saved +// options restored): the host's delivery loop waits on `settled` before it hands a message over, +// so a first turn never runs under defaults the restore was about to replace. + +import { providerStartupFailureRejection } from '../native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement' +import type { ClaudeSession } from './claude-structured-session-state' + +export type ClaudeSessionStartup = { + state: 'pending' | 'proven' | 'failed' + failure: Error | null + /** Resolves once startup has landed or faulted, or the child exited or was closed; never + * rejects. A close must end it: the delivery loop waits here, and a start Stop cut short + * would otherwise hold that loop forever. */ + settled: Promise + end: () => void +} + +export function createClaudeSessionStartup(): ClaudeSessionStartup { + let end: () => void = () => undefined + const ended = new Promise((resolve) => { + end = resolve + }) + return { state: 'pending', failure: null, settled: ended, end } +} + +export function claudeStartupFailureReason(session: ClaudeSession): string | null { + return session.startup.state === 'failed' + ? providerStartupFailureRejection(session.startup.failure ?? undefined) + : null +} + +/** Resolves when startup lands or `timeoutMs` passes; a stuck start then refuses the write as before. */ +export function claudeStartupSettledWithin( + session: ClaudeSession | undefined, + timeoutMs: number +): Promise { + if (session?.startup.state !== 'pending') { + return Promise.resolve() + } + let timer: ReturnType | undefined + return Promise.race([ + session.startup.settled, + new Promise((resolve) => { + timer = setTimeout(resolve, timeoutMs) + }) + ]).finally(() => clearTimeout(timer)) +} + +/** Startup cannot land any more: the child exited, was closed, or its start faulted. */ +export function failClaudeStartup(session: ClaudeSession, error: Error): void { + const startup = session.startup + if (startup.state === 'pending') { + startup.state = 'failed' + startup.failure = error + } + startup.end() +} diff --git a/src/main/claude/claude-structured-session-startup.test.ts b/src/main/claude/claude-structured-session-startup.test.ts index 362f4d691d0..ceb9e9cdf6d 100644 --- a/src/main/claude/claude-structured-session-startup.test.ts +++ b/src/main/claude/claude-structured-session-startup.test.ts @@ -55,7 +55,7 @@ describe('Claude structured session publishes before the CLI answers initialize' expect(adapter.readCommands('session-1')).toBeUndefined() await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') expect(events.find((event) => event.type === 'options')).toMatchObject({ models: [{ value: 'claude-sonnet' }] @@ -65,7 +65,7 @@ describe('Claude structured session publishes before the CLI answers initialize' await adapter.closeAll() }) - it('reports `started` once saved options are restored, before any held prompt is written', async () => { + it('reports `started` once saved options are restored, having written no prompt of its own', async () => { const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS, initModel: 'claude-opus-9' }) const { adapter, events } = startingAdapter(claude) const order: string[] = [] @@ -74,11 +74,10 @@ describe('Claude structured session publishes before the CLI answers initialize' return undefined } await adapter.acquire({ ...ACQUIRE, options: { model: 'opus' } }) - await adapter.dispatch(PROMPT) expect(events.some((event) => event.type === 'started')).toBe(false) await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') const startedAt = events.findIndex((event) => event.type === 'started') expect(events[startedAt]).toEqual({ @@ -90,9 +89,9 @@ describe('Claude structured session publishes before the CLI answers initialize' reportedOptions: expect.objectContaining({ model: 'opus' }), restoreSkippedOptions: [] }) - // The restore wrote the saved model before `started`, and the held prompt only after it. + // The restore wrote the saved model before `started`; no message waits inside the adapter. expect(order).toEqual(['set_model']) - expect(claude.connections[0].sent).toHaveLength(1) + expect(claude.connections[0].sent).toEqual([]) expect(events.slice(0, startedAt).some((event) => event.type === 'options')).toBe(true) await adapter.closeAll() }) @@ -133,52 +132,37 @@ describe('Claude structured session publishes before the CLI answers initialize' await adapter.closeAll() }) - it('holds a prompt sent before init and writes it once startup lands', async () => { + // The host's delivery loop waits here before it hands a message over, so the adapter no longer + // holds prompts of its own: nothing is written until startup lands because nothing is sent. + it('resolves awaitStarted only once startup lands', async () => { const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) const { adapter } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - - await expect(adapter.dispatch(PROMPT)).resolves.toEqual({ state: 'admitted' }) - expect(claude.connections[0].sent).toEqual([]) - - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') - - expect(claude.connections[0].sent).toHaveLength(1) - expect(claude.connections[0].sent[0]).toMatchObject({ type: 'user' }) - await adapter.closeAll() - }) - - it('writes a prompt whose admission barrier was still running when startup landed', async () => { - const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) - const { adapter } = startingAdapter(claude) - await adapter.acquire(ACQUIRE) - let passBarrier = (): void => {} - const barrier = new Promise((resolve) => { - passBarrier = resolve + let started = false + const waited = adapter.awaitStarted('session-1').then(() => { + started = true }) - const dispatched = adapter.dispatch({ ...PROMPT, beforeDispatch: () => barrier }) - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') - passBarrier() + await vi.advanceTimersByTimeAsync(SLOW_INIT_MS - 1) + expect(started).toBe(false) + await vi.advanceTimersByTimeAsync(1) + await waited - await expect(dispatched).resolves.toEqual({ state: 'admitted' }) - expect(claude.connections[0].sent.filter((message) => message.type === 'user')).toHaveLength(1) + await expect(adapter.dispatch(PROMPT)).resolves.toEqual({ state: 'admitted' }) + expect(claude.connections[0].sent).toEqual([expect.objectContaining({ type: 'user' })]) await adapter.closeAll() }) - it('ends the session with the exit reason when the CLI dies before init, and rejects held prompts', async () => { + it('ends the session with the exit reason when the CLI dies before init', async () => { const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS, exitBeforeInit: 'claude stream-json exited (code 1): stderr says no' }) - const { adapter, events, late } = startingAdapter(claude) + const { adapter, events } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - await adapter.dispatch(PROMPT) await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') await adapter.drainObservedExits() expect(events.find((event) => event.type === 'ended')).toMatchObject({ @@ -186,9 +170,6 @@ describe('Claude structured session publishes before the CLI answers initialize' cause: 'unexpected-exit', startupUnproven: true }) - expect(late).toEqual([ - expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' }) - ]) expect(claude.connections[0].sent).toEqual([]) expect(claude.connections[0].closeCount).toBe(1) }) @@ -203,7 +184,7 @@ describe('Claude structured session publishes before the CLI answers initialize' await adapter.acquire(ACQUIRE) await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') await adapter.drainObservedExits() // A failed start is released on the same evidence a failed create is. @@ -217,7 +198,7 @@ describe('Claude structured session publishes before the CLI answers initialize' const claude = fakeClaude({ initAccount: { apiProvider: 'firstParty', tokenSource: 'none' } }) const { adapter, events } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') await adapter.drainObservedExits() expect(events.find((event) => event.type === 'ended')).toMatchObject({ @@ -226,82 +207,41 @@ describe('Claude structured session publishes before the CLI answers initialize' }) }) - it('closes a session stopped before init without faulting it or writing held prompts', async () => { - const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) - const { adapter, events, late } = startingAdapter(claude) + // A Stop that closes a child still starting must end the wait the host's delivery loop is in, + // though initialize never answers; otherwise every later send joins a loop that never moves. + it('ends the wait on a start closed before init, without faulting it', async () => { + const claude = fakeClaude({ initDelayMs: 10 * SLOW_INIT_MS }) + const { adapter, events } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - await adapter.dispatch(PROMPT) + let ended = false + const waited = adapter.awaitStarted('session-1').then(() => { + ended = true + }) await expect(adapter.closeSession('session-1')).resolves.toBe(true) - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await vi.advanceTimersByTimeAsync(0) + await waited + expect(ended).toBe(true) const connection = claude.connections[0] expect(connection.closeCount).toBe(1) expect(connection.sent).toEqual([]) expect(connection.calls.map(({ subtype }) => subtype)).not.toContain('get_settings') - expect(late).toEqual([ - expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' }) - ]) expect(events.some((event) => event.type === 'ended' && event.startupUnproven)).toBe(false) expect(events.some((event) => event.type === 'started')).toBe(false) }) - it('withdraws a held prompt when the turn is cancelled before init', async () => { + it('interrupts nothing when Stop lands before init: nothing was written', async () => { const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) - const { adapter, late } = startingAdapter(claude) + const { adapter } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - await adapter.dispatch(PROMPT) await expect( adapter.cancelTurn({ sessionId: 'session-1', turnId: 'turn-1', fence: 7 }) - ).resolves.toEqual({ cancelled: true }) - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + ).resolves.toEqual({ cancelled: false }) + expect(claude.connections[0].calls.map(({ subtype }) => subtype)).not.toContain('interrupt') expect(claude.connections[0].sent).toEqual([]) - expect(late).toEqual([ - expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' }) - ]) - await adapter.closeAll() - }) - - it('withdraws the prompts still held when Stop lands while startup is writing them', async () => { - const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) - const { adapter, late } = startingAdapter(claude) - await adapter.acquire(ACQUIRE) - const connection = claude.connections[0] - const send = connection.send - let landFirstWrite = (): void => {} - const firstWrite = new Promise((resolve) => { - landFirstWrite = resolve - }) - let writes = 0 - connection.send = async (message, beforeDispatch) => { - writes += 1 - if (writes === 1) { - await firstWrite - } - return send(message, beforeDispatch) - } - await adapter.dispatch(PROMPT) - await adapter.dispatch({ ...PROMPT, clientMessageId: 'client-2' }) - - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - // Startup has landed and is writing the first held prompt. - expect(writes).toBe(1) - const cancelled = adapter.cancelTurn({ sessionId: 'session-1', turnId: 'turn-1', fence: 7 }) - await vi.advanceTimersByTimeAsync(0) - landFirstWrite() - await vi.advanceTimersByTimeAsync(5_000) - await adapter.drainStartup('session-1') - - expect(connection.sent.filter((message) => message.type === 'user')).toHaveLength(1) - expect(late).toContainEqual( - expect.objectContaining({ clientMessageId: 'client-2', state: 'rejected' }) - ) - // Stop withdrew something, so it answers as a cancel whatever the interrupt made of the turn. - await expect(cancelled).resolves.toEqual({ cancelled: true }) await adapter.closeAll() }) }) diff --git a/src/main/claude/claude-structured-session-startup.ts b/src/main/claude/claude-structured-session-startup.ts index aa1f9a02ecc..d486a670a13 100644 --- a/src/main/claude/claude-structured-session-startup.ts +++ b/src/main/claude/claude-structured-session-startup.ts @@ -26,10 +26,7 @@ import { observeClaudeSettingsApplied, readClaudeSettingsEffort } from './claude-structured-session-options' -import { - failClaudeStartupGate, - openClaudeStartupGate -} from './claude-structured-session-startup-gate' +import { failClaudeStartup } from './claude-structured-session-startup-state' import type { ClaudeSession, ClaudeStructuredSessionEvent } from './claude-structured-session-state' export type ClaudeInitProof = { @@ -163,8 +160,8 @@ function claudeStartedReportedOptions( return persisted } -/** Applies startup facts to the published session, restores saved options, then releases - * held prompts. Any failure faults the session so the user sees why it never started. */ +/** Applies startup facts to the published session and restores saved options; only then does the + * session take input. Any failure faults the session so the user sees why it never started. */ export async function settleClaudeSessionStartup(input: { session: ClaudeSession facts: Promise @@ -179,7 +176,7 @@ export async function settleClaudeSessionStartup(input: { if (input.isCurrent()) { return false } - failClaudeStartupGate(session, new Error('claude session closed before startup completed')) + failClaudeStartup(session, new Error('claude session closed before startup completed')) return true } try { @@ -198,13 +195,15 @@ export async function settleClaudeSessionStartup(input: { ), restoreSkippedOptions: [...session.restoreSkippedOptions] }) - await openClaudeStartupGate(session) + if (session.startup.state === 'pending') { + session.startup.state = 'proven' + } } } catch (caught) { const error = caught instanceof Error ? caught : new Error(String(caught)) // A close or exit that already ended startup owns how the session ends. const endedElsewhere = session.startup.state !== 'pending' - failClaudeStartupGate(session, error) + failClaudeStartup(session, error) if (!endedElsewhere && input.isCurrent()) { input.fault(error) } diff --git a/src/main/claude/claude-structured-session-state.ts b/src/main/claude/claude-structured-session-state.ts index 60055874b37..b671dd8b632 100644 --- a/src/main/claude/claude-structured-session-state.ts +++ b/src/main/claude/claude-structured-session-state.ts @@ -25,7 +25,7 @@ import type { AgentChildWorkEvidence } from '../../shared/agent-status-child-wor import type { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import type { ClaudeChildWorkDecoder } from './claude-child-work-decoder' import type { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' -import type { ClaudeSessionStartupGate } from './claude-structured-session-startup-gate' +import type { ClaudeSessionStartup } from './claude-structured-session-startup-state' export type ClaudeAuthDiagnostic = { apiKeySourceConfigured: boolean @@ -195,7 +195,7 @@ export type ClaudeSession = { events: StructuredAgentSessionEventSink | undefined unbindReadingControl?: () => void /** Published at spawn; init facts, option restore and queued prompts land when startup does. */ - startup: ClaudeSessionStartupGate + startup: ClaudeSessionStartup } export function mintClaudeAcquisitionGeneration(deps: ClaudeStructuredSessionAdapterDeps): string { diff --git a/src/main/claude/claude-structured-session-test-support.ts b/src/main/claude/claude-structured-session-test-support.ts index 023e0ba46b7..c8e3d86b0e8 100644 --- a/src/main/claude/claude-structured-session-test-support.ts +++ b/src/main/claude/claude-structured-session-test-support.ts @@ -220,7 +220,7 @@ export function adapterFor( const acquire = adapter.acquire adapter.acquire = async (input) => { const acquisition = await acquire(input) - await adapter.drainStartup(input.identity.sessionId) + await adapter.awaitStarted(input.identity.sessionId) return acquisition } return adapter diff --git a/src/main/ipc/desktop-renderer-runtime-capabilities.ts b/src/main/ipc/desktop-renderer-runtime-capabilities.ts index 63ba9b1d0b1..0bcfd1bd686 100644 --- a/src/main/ipc/desktop-renderer-runtime-capabilities.ts +++ b/src/main/ipc/desktop-renderer-runtime-capabilities.ts @@ -1,5 +1,6 @@ import { AGENT_LAUNCH_RUNTIME_CAPABILITY, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, AGENT_SESSION_BACKGROUND_TASK_ROW_STOP_CAPABILITY, AGENT_SESSION_BACKGROUND_TASK_STOP_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, @@ -25,6 +26,7 @@ import { export const DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES: readonly RuntimeCapability[] = [ AGENT_SESSION_BACKGROUND_TASK_STOP_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, AGENT_SESSION_TURN_ITEM_CAPABILITY, AGENT_SESSION_BACKGROUND_TASK_ROW_STOP_CAPABILITY, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY, diff --git a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts index c4138ba2241..c356d7904b2 100644 --- a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts +++ b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts @@ -1,8 +1,11 @@ +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { DISPATCH_DOUBT_HOST_RESTARTED } from './journal-dispatch-doubt-reasons' import type { AgentSessionJournal } from './journal-store' /** Settles every submission a process fact left unanswerable. Doubt is never - * proof of non-delivery, so nothing here ever becomes re-deliverable. */ + * proof of non-delivery, so nothing here ever becomes re-deliverable. A queued + * submission was never handed over, so it is not in doubt and is left alone. */ export async function markJournalPendingSubmissionsUnknown( journal: AgentSessionJournal, fence: number, @@ -12,8 +15,9 @@ export async function markJournalPendingSubmissionsUnknown( .submissions() .filter( (entry) => - entry.dispatchState === 'pending' || - (entry.dispatchState === 'unknown' && entry.recovered !== true) + !isQueuedAgentJournalSubmission(entry) && + (entry.dispatchState === 'pending' || + (entry.dispatchState === 'unknown' && entry.recovered !== true)) ) for (const entry of unresolved) { // An earlier reason already names a sharper fact than "the host restarted". @@ -31,7 +35,8 @@ export async function markJournalPendingSubmissionsUnknown( } /** Settles every submission a child that never proved its start left unanswered as `rejected`: - * such a child accepted nothing, so each is provably unwritten and safe to send again. */ + * such a child accepted nothing, so each is provably unwritten and safe to send again. A queued + * submission was never handed to that child; the delivery loop settles it. */ export async function rejectJournalPendingSubmissions( journal: AgentSessionJournal, fence: number, @@ -41,8 +46,9 @@ export async function rejectJournalPendingSubmissions( .submissions() .filter( (entry) => - entry.dispatchState === 'pending' || - (entry.dispatchState === 'unknown' && entry.recovered !== true) + !isQueuedAgentJournalSubmission(entry) && + (entry.dispatchState === 'pending' || + (entry.dispatchState === 'unknown' && entry.recovered !== true)) ) for (const entry of unwritten) { await journal.resolveDispatch({ @@ -55,3 +61,25 @@ export async function rejectJournalPendingSubmissions( } return unwritten.map((entry) => entry.clientMessageId) } + +/** Rejects queued submissions — accepted, never handed over, so provably unwritten. */ +export async function rejectJournalQueuedSubmissions( + journal: AgentSessionJournal, + fence: number, + reason: string, + which: (submission: AgentJournalSubmission) => boolean = () => true +): Promise { + const queued = journal + .submissions() + .filter((entry) => isQueuedAgentJournalSubmission(entry) && which(entry)) + for (const entry of queued) { + await journal.resolveDispatch({ + clientMessageId: entry.clientMessageId, + state: 'rejected', + reason, + fence, + recovered: true + }) + } + return queued.map((entry) => entry.clientMessageId) +} diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index 273502c8cf2..3fbee85a417 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -254,7 +254,8 @@ function applySubmission( providerItemId: null, reason: null, submittedAt: row.ts, - resolvedAt: null + resolvedAt: null, + ...(row.handoverRecorded ? { handoverRecorded: true, acceptedSequence: row.seq } : {}) }) const itemId = agentJournalSubmissionKey(row.clientMessageId) upsertItem(state, itemId, 0, journalRenderItem(itemId, 0, row.body, row)) @@ -277,6 +278,9 @@ function applyDispatch( submission.providerItemId = row.providerItemId submission.reason = row.reason submission.resolvedAt = row.state === 'pending' ? null : row.ts + if (row.state === 'pending') { + submission.handedOverAt = row.ts + } if (row.recovered) { submission.recovered = row.recovered } else { diff --git a/src/main/native-chat/agent-session-journal/journal-restart-reconciliation.ts b/src/main/native-chat/agent-session-journal/journal-restart-reconciliation.ts index 626baa9820a..f37571814bb 100644 --- a/src/main/native-chat/agent-session-journal/journal-restart-reconciliation.ts +++ b/src/main/native-chat/agent-session-journal/journal-restart-reconciliation.ts @@ -18,6 +18,7 @@ import { agentJournalItemKey, agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import type { AgentSessionJournal } from './journal-store' import { reconcileSubmissions, type ProviderHistoryWindow } from './journal-submission-reconciler' @@ -42,7 +43,11 @@ function comparableSubmissions(journal: AgentSessionJournal): AgentJournalSubmis const { items, submissions } = journal.snapshot() const bodies = new Map(items.map((item) => [item.itemId, item.body])) return submissions.filter((submission) => { - if (submission.dispatchState !== 'pending' && submission.dispatchState !== 'unknown') { + if ( + (submission.dispatchState !== 'pending' && submission.dispatchState !== 'unknown') || + // Never handed over, so provider history cannot hold it. + isQueuedAgentJournalSubmission(submission) + ) { return false } const body = bodies.get(agentJournalSubmissionKey(submission.clientMessageId)) diff --git a/src/main/native-chat/agent-session-journal/journal-row-builders.ts b/src/main/native-chat/agent-session-journal/journal-row-builders.ts index e17ccf0cdf8..e56ca86a7fb 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-builders.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-builders.ts @@ -65,6 +65,7 @@ export function journalSubmissionRowBuilder( payloadFingerprint: string body: AgentJournalMessageItem fence: number + handoverRecorded?: true } ): RowBuilder { return (seq, ts) => @@ -267,6 +268,7 @@ export function buildJournalSubmissionRow(input: { seq: number fence: number ts: number + handoverRecorded?: true }): JournalSubmissionRow { return { kind: 'submission', @@ -274,7 +276,8 @@ export function buildJournalSubmissionRow(input: { payloadFingerprint: input.payloadFingerprint, providerHandle: input.providerHandle, body: input.body, - ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts) + ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts), + ...(input.handoverRecorded ? { handoverRecorded: true } : {}) } } diff --git a/src/main/native-chat/agent-session-journal/journal-row-schema.ts b/src/main/native-chat/agent-session-journal/journal-row-schema.ts index 99cb47971e9..0f29f8b3879 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-schema.ts @@ -77,6 +77,9 @@ export type JournalSubmissionRow = JournalRowBase & { payloadFingerprint: string providerHandle: AgentSessionProviderHandle body: AgentJournalMessageItem + /** Accepted to be handed over by a later `dispatch{pending}` row; absent on rows whose writer + * dispatched in the same step. Older readers keep the key and ignore it. */ + handoverRecorded?: true } export type JournalDispatchRow = JournalRowBase & { diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index cea47e381fe..79d86f80dbb 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -59,6 +59,8 @@ export type JournalSubmissionInput = { payloadFingerprint: string body: AgentJournalMessageItem fence: number + /** The send is accepted now and handed over later, by a `dispatch{pending}` row. */ + handoverRecorded?: true } export type JournalItemAppendInput = { diff --git a/src/main/native-chat/agent-session-journal/journal-store.test.ts b/src/main/native-chat/agent-session-journal/journal-store.test.ts index ffaea5c5d6f..3637f03dab6 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.test.ts @@ -424,3 +424,31 @@ async function withJournalDatabase( opened.db.close() } } + +describe('what a handle found on disk when it opened', () => { + const submission = (clientMessageId: string) => ({ + clientMessageId, + payloadFingerprint: 'fp', + body: { kind: 'message' as const, role: 'user' as const, blocks: [] }, + fence: 1, + handoverRecorded: true as const + }) + + it('names rows an earlier handle wrote, and never a row of a later epoch', async () => { + const earlier = await open() + await earlier.appendItem(item(1), body('one'), { fence: 1 }) + await earlier.appendSubmission(submission('earlier')) + await earlier.close() + + const journal = await open() + const leftover = journal.submissions().find((entry) => entry.clientMessageId === 'earlier') + expect(journal.wroteBeforeOpen(leftover?.acceptedSequence)).toBe(true) + + // Sequences restart with an epoch, so a row accepted after it can sit below the open cursor. + await journal.replaceEpochItems('handle_forked', 1, []) + await journal.appendSubmission(submission('later')) + const later = journal.submissions().find((entry) => entry.clientMessageId === 'later') + expect(later?.acceptedSequence).toBeLessThanOrEqual(2) + expect(journal.wroteBeforeOpen(later?.acceptedSequence)).toBe(false) + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 568fbe4d39d..5119f7ef597 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -28,7 +28,8 @@ import { readJournalRowsAfterCursor, type JournalLoad } from './journal-open' import { journalDatabaseFile } from './journal-paths' import { markJournalPendingSubmissionsUnknown, - rejectJournalPendingSubmissions + rejectJournalPendingSubmissions, + rejectJournalQueuedSubmissions } from './journal-pending-submission-recovery' import { applyJournalRow, @@ -75,6 +76,7 @@ export class AgentSessionJournal { private state: JournalReducerState private readOnly = false private malformedRows = 0 + private openedThrough: AgentJournalCursor = { epoch: '', sequence: 0 } private database: OpenJournalDatabase | null = null private onCommitted: (() => void) | null = null private readonly queue: JournalWriteQueue @@ -147,6 +149,16 @@ export class AgentSessionJournal { return this.journalDir } + /** Whether a row at this sequence was on disk when this handle opened, so an earlier handle + * wrote it. Sequences restart with each epoch, so a row of a later epoch never was. */ + wroteBeforeOpen(sequence: number | undefined): boolean { + return ( + sequence !== undefined && + this.state.epoch === this.openedThrough.epoch && + sequence <= this.openedThrough.sequence + ) + } + /** What the last open's repair did. */ get repair(): { malformedRows: number } { return { malformedRows: this.malformedRows } @@ -157,6 +169,7 @@ export class AgentSessionJournal { this.database = openJournalDatabase(this.dbPath) try { await this.restore() + this.openedThrough = this.cursor() } catch (error) { // Nothing else holds a reference to this connection, so a throw here is // the leak site unless the store releases it itself — and a close that @@ -310,6 +323,15 @@ export class AgentSessionJournal { return rejectJournalPendingSubmissions(this, fence, reason) } + /** Reject sends accepted but never handed over, optionally only those `which` names. */ + async rejectQueuedSubmissions( + fence: number, + reason: string, + which?: (submission: AgentJournalSubmission) => boolean + ): Promise { + return rejectJournalQueuedSubmissions(this, fence, reason, which) + } + /** The escape hatch for corruption, an unreconcilable prefix, a forked handle, * and an unreadable schema. It invalidates every cursor; clients reload. */ async rollEpoch(reason: AgentJournalEpochReason, fence: number): Promise { diff --git a/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.test.ts b/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.test.ts index 214c889b5c9..1c60029c7d3 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.test.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.test.ts @@ -6,14 +6,14 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentJournalItemIdentity, AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import { openJournalDatabase } from '../agent-session-journal/journal-database' import { JOURNAL_DB_SCHEMA_VERSION } from '../agent-session-journal/journal-database-schema' -import { loadJournal } from '../agent-session-journal/journal-open' +import { loadJournal, replayJournal } from '../agent-session-journal/journal-open' import { journalDatabaseFile } from '../agent-session-journal/journal-paths' import { readJournalEpochRows } from '../agent-session-journal/journal-row-table' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' @@ -24,6 +24,12 @@ import { recoveryJournalDir } from './agent-session-journal-recovery' +// Only the store's own replay goes through the mock; the probe's, inside the same module, does not. +vi.mock('../agent-session-journal/journal-open', async (importOriginal) => { + const actual = await importOriginal<{ replayJournal: typeof replayJournal }>() + return { ...actual, replayJournal: vi.fn(actual.replayJournal) } +}) + const CODEX_SESSION = '019fd532-7c11-7a90-b6de-4e1a2c3d5f60' const IDENTITY: AgentSessionJournalIdentity = { @@ -160,6 +166,21 @@ describe('openAgentSessionJournalWithRecovery', () => { expect(opened.snapshot().items).toHaveLength(2) }) + it('reads the journal once: the probe is the open', async () => { + await seedJournal(2) + vi.mocked(replayJournal).mockClear() + const opened = journals.track( + await openAgentSessionJournalWithRecovery({ + identity: IDENTITY, + journalDir, + fence: 1, + historyFilePath + }).then((result) => result.journal) + ) + expect(opened.snapshot().items).toHaveLength(2) + expect(replayJournal).not.toHaveBeenCalled() + }) + it('rebuilds a holed journal in place on a fresh epoch', async () => { await seedJournal(3) await deleteRow(3) diff --git a/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts b/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts index 6e771a31809..29d324be971 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts @@ -67,7 +67,9 @@ export async function openAgentSessionJournalWithRecovery(input: { } const journal = await openAgentSessionJournal({ identity: input.identity, - journalDir: input.journalDir + journalDir: input.journalDir, + // The probe is this open's replay; omitted, not `null`, when there was nothing to load. + ...(probe ? { loaded: probe } : {}) }) if (!probe?.corrupt) { return { journal, recovery: null } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts new file mode 100644 index 00000000000..186127847a6 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts @@ -0,0 +1,671 @@ +// A send is accepted, then delivered: the host answers once the message is recorded, and the +// session's delivery loop starts a provider child for it and hands it over. Against the real host, +// store and journal; each assertion reads what an open chat or the journal's next reader sees. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' +import { + DISPATCH_REJECTED_CANCELLED, + DISPATCH_REJECTED_HOST_RESTARTED, + DISPATCH_REJECTED_PROVIDER_CLOSED +} from '../../../shared/structured-agent-session-dispatch-rejection' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { journalDirectoryFor } from '../agent-session-journal/journal-paths' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' +import { + AgentSessionPreSpawnError, + type StructuredAgentSessionAdapter +} from './structured-agent-session-adapter' +import { journalIdentityFor } from './structured-agent-session-attach' +import { attachParamsForRecord } from './structured-agent-session-conversation-open' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { persistRewindRecord } from './structured-rewind-recovery' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } + +function eventually(assertion: () => void | Promise): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let acquire: Mock +let dispatch: Mock +let adapterExtras: Partial +let releaseGraceMs: number + +const spawnChild: StructuredAgentSessionAdapter['acquire'] = async ({ fence, spawnToken }) => ({ + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + acquisitionGeneration: `generation-${acquire.mock.calls.length}`, + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex' as const, threadId: THREAD }, + origin: store.getRecord(SESSION)?.providerHandleChain.length + ? ('resumed' as const) + : ('created' as const), + mintedAtFence: fence, + observedAt: NOW + } +}) + +async function startHost(): Promise { + host = new StructuredAgentSessionHost({ + store, + adapter: { + acquire, + dispatch, + closeSession: vi.fn(async () => true), + releaseAcquisition: vi.fn(async () => true), + cancelTurn: vi.fn(async () => ({ cancelled: false })), + answerPrompt: vi.fn(async () => undefined), + setOption: vi.fn(async () => undefined), + ...adapterExtras + }, + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => `spawn-${acquire.mock.calls.length}`, + releaseGraceMs, + now: () => NOW + }) +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-accept-deliver-')) + resetHostTestOperationIds() + adapterExtras = {} + releaseGraceMs = 60_000 + acquire = vi.fn(spawnChild) + dispatch = vi.fn(async () => ({ + state: 'accepted' as const, + providerIdentity: { + provider: 'codex' as const, + threadId: THREAD, + turnId: `turn-${dispatch.mock.calls.length}`, + ordinal: dispatch.mock.calls.length + } + })) + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + await startHost() + expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) +}) + +afterEach(async () => { + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +function sendParams(text: string) { + const body = hostTestMessage(text) + return { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: 1, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + } +} + +/** Accepted at once, whatever the child is doing; answers the message id. */ +async function accept(text: string): Promise { + const params = sendParams(text) + expect(await host.send(CALLER, params)).toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending', handoverRecorded: true } } + }) + return params.envelope.clientOperationId +} + +function stop() { + const turnId = 'turn-none' + return host.cancel(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.cancel', + sessionId: SESSION, + fields: { turnId } + }) + }, + turnId + }) +} + +function submission(id: string): AgentJournalSubmission | undefined { + return host.journalSnapshot(SESSION).submissions.find((entry) => entry.clientMessageId === id) +} + +/** Read back after the conversation was closed, through the same open any reader takes. */ +async function reopened(id: string): Promise { + await host.revealSession(SESSION) + return submission(id) +} + +function errorRows(): string[] { + return host + .journalSnapshot(SESSION) + .items.flatMap((item) => + item.body.kind === 'status' && item.body.tone === 'error' ? [item.body.text] : [] + ) +} + +let subscriptions = 0 + +function subscribe(): AgentSessionSubscribeEvent[] { + const events: AgentSessionSubscribeEvent[] = [] + subscriptions += 1 + // Cloned as received: a frame shares the journal's live objects, which later rows revise. + host.subscribe({ + id: `sub-${subscriptions}`, + sessionId: SESSION, + emit: (event) => events.push(structuredClone(event)) + }) + return events +} + +/** What an open chat was told about one message, in frame order. */ +function framedStates(events: AgentSessionSubscribeEvent[], id: string): string[] { + return events.flatMap((event) => + event.type === 'batch' + ? event.batch.submissions + .filter((entry) => entry.clientMessageId === id) + .map((entry) => + entry.dispatchState === 'pending' && entry.handedOverAt !== undefined + ? 'handed-over' + : entry.dispatchState + ) + : [] + ) +} + +function deferred() { + let resolve!: (value: T) => void + let reject!: (error: unknown) => void + const promise = new Promise((next, fail) => { + resolve = next + reject = fail + }) + return { promise, resolve, reject } +} + +/** Rows written by an earlier host process that ended before handing them over. */ +async function writeAsEarlierProcess( + write: (journal: AgentSessionJournal, fence: number) => Promise +): Promise { + await host.close(SESSION) + const record = store.getRecord(SESSION)! + const params = attachParamsForRecord(record, { + clientOperationId: 'earlier', + expectedRuntimeFence: record.lease.runtimeFence + }) + const journal = await openAgentSessionJournal({ + identity: journalIdentityFor(record, params), + journalDir: journalDirectoryFor(root, { + workspaceId: record.location.workspaceId, + sessionId: SESSION + }) + }) + await write(journal, record.lease.runtimeFence) + await journal.close() +} + +function earlierSubmission(id: string, text: string, handoverRecorded?: true) { + const body = hostTestMessage(text) + return { + clientMessageId: id, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }), + body, + ...(handoverRecorded ? { handoverRecorded } : {}) + } +} + +describe('a send is answered at acceptance', () => { + it('answers before the child starts, then an open chat sees the handover and the reply (W2)', async () => { + await host.close(SESSION) + const starting = deferred() + acquire.mockImplementationOnce(async (input) => { + await starting.promise + return spawnChild(input) + }) + + const answering = deferred() + const answer = dispatch.getMockImplementation()! + dispatch.mockImplementationOnce(async (input) => { + await answering.promise + return answer(input) + }) + + const id = await accept('hello') + const events = subscribe() + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + expect(dispatch).not.toHaveBeenCalled() + + starting.resolve() + await eventually(() => expect(framedStates(events, id)).toEqual(['handed-over'])) + answering.resolve() + await eventually(() => expect(framedStates(events, id)).toEqual(['handed-over', 'accepted'])) + }) + + it('accepts a second send while the first one starts the child, before handing either over (W6)', async () => { + await host.close(SESSION) + const starting = deferred() + acquire.mockImplementationOnce(async (input) => { + await starting.promise + return spawnChild(input) + }) + const first = await accept('first') + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + const second = host.send(CALLER, sendParams('second')) + + starting.resolve() + const secondResult = await second + if (!secondResult.ok) { + throw new Error('the second send was refused') + } + const secondId = secondResult.value.clientMessageId + await eventually(() => expect(submission(secondId)?.dispatchState).toBe('accepted')) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([first, secondId]) + // The second was accepted while the start held the queue — before the first was handed over. + expect(submission(secondId)!.submittedAt).toBeLessThanOrEqual(submission(first)!.handedOverAt!) + const rows = host.journalSnapshot(SESSION).submissions + expect(rows.map((row) => row.clientMessageId)).toEqual([first, secondId]) + }) +}) + +describe('a start the chat needed and did not get', () => { + it('writes one error row and rejects every queued message with it; the next send starts (W3)', async () => { + await host.close(SESSION) + acquire.mockRejectedValueOnce(new Error('spawn codex ENOENT')) + const first = await accept('first') + const second = await accept('second') + + await eventually(() => expect(submission(second)?.dispatchState).toBe('rejected')) + const rows = errorRows() + expect(rows).toHaveLength(1) + expect(rows[0]).toContain('spawn codex ENOENT') + expect(submission(first)).toMatchObject({ dispatchState: 'rejected', reason: rows[0] }) + expect(submission(second)).toMatchObject({ dispatchState: 'rejected', reason: rows[0] }) + + const next = await accept('after the fix') + await eventually(() => expect(submission(next)?.dispatchState).toBe('accepted')) + expect(errorRows()).toHaveLength(1) + }) + + it.each([ + [ + 'eligibility', + () => { + adapterExtras = { supportsLocation: () => false } + }, + 'cannot resume' + ], + [ + 'spawn', + () => acquire.mockRejectedValueOnce(new Error('spawn codex ENOENT')), + 'spawn codex ENOENT' + ], + [ + 'auth', + () => + acquire.mockRejectedValueOnce( + new AgentSessionPreSpawnError(new Error('Not logged in. Please run /login.')) + ), + 'Not logged in' + ] + ])('writes one row a live chat sees for a %s refusal (W14)', async (_source, arrange, cause) => { + await host.close(SESSION) + arrange() + await host.flushAllStreamedEvents() + await startHost() + const id = await accept('hello') + const events = subscribe() + + await eventually(() => expect(submission(id)?.dispatchState).toBe('rejected')) + expect(errorRows()).toHaveLength(1) + expect(errorRows()[0]).toContain(cause) + const framedRows = events.flatMap((event) => + event.type === 'batch' || event.type === 'snapshot' + ? (event.type === 'batch' ? event.batch.items : event.page.items).filter( + (item) => item.body.kind === 'status' && item.body.tone === 'error' + ) + : [] + ) + expect(framedRows.length).toBeGreaterThan(0) + }) + + it('names the start failure on queued messages when the attach fails after acquiring (W4′a)', async () => { + await host.close(SESSION) + const id = await accept('hello') + // The attach's own success record is the post-acquisition step that fails. + const record = vi.spyOn(store, 'recordOperationOutcome') + record.mockImplementation(async (input) => { + if (input.operationId !== id && input.outcome.status === 'succeeded') { + record.mockRestore() + throw new Error('record store write failed') + } + return AgentSessionRecordStore.prototype.recordOperationOutcome.call(store, input) + }) + + // Read through the open any reader takes, so a conversation the failure dropped is reopened + // and its message read as that reopen settles it. + let settled: AgentJournalSubmission | undefined + await eventually(async () => { + settled = await reopened(id) + expect(settled?.dispatchState).not.toBe('pending') + }) + // The message names the start that failed, not a close or a restart it never met. + expect(settled).toMatchObject({ + dispatchState: 'rejected', + reason: expect.stringContaining('record store write failed') + }) + expect(errorRows()).toEqual([settled?.reason]) + }) +}) + +describe('an attach that fails after indexing its child', () => { + it('leaves no child behind, so the next send starts one and is delivered', async () => { + await host.close(SESSION) + const owned: boolean[] = [] + host.subscribeStatus({ + id: 'list-1', + emit: (event) => { + if (event.type === 'status') { + owned.push(event.session.hostExecutionOwned === true) + } + } + }) + const first = await accept('hello') + // The attach's own success record is the step after `onAttached` indexed the child. + const record = vi.spyOn(store, 'recordOperationOutcome') + record.mockImplementation(async (input) => { + if (input.operationId !== first && input.outcome.status === 'succeeded') { + record.mockRestore() + throw new Error('record store write failed') + } + return AgentSessionRecordStore.prototype.recordOperationOutcome.call(store, input) + }) + await eventually(() => expect(submission(first)?.dispatchState).toBe('rejected')) + // Nothing was indexed, so nothing had to be taken back: no list ever showed a child. + expect(host['sessions'].get(SESSION)?.child).toBeNull() + expect(owned).not.toContain(true) + const acquiresBefore = acquire.mock.calls.length + + const next = await accept('after the failure') + + await eventually(() => expect(submission(next)?.dispatchState).toBe('accepted')) + expect(acquire).toHaveBeenCalledTimes(acquiresBefore + 1) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([next]) + }) +}) + +describe('what an earlier host process left behind', () => { + it('rejects a message it accepted and never handed over, as not sent (W4′b)', async () => { + await writeAsEarlierProcess(async (journal, fence) => { + await journal.appendSubmission({ ...earlierSubmission('queued', 'q', true), fence }) + }) + + await host.revealSession(SESSION) + + expect(submission('queued')).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_HOST_RESTARTED + }) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('leaves a legacy pending message and a handed-over one in doubt, never re-sent (W4′c)', async () => { + const providerHistoryWindow = vi.fn(async () => null) + adapterExtras = { providerHistoryWindow } + await writeAsEarlierProcess(async (journal, fence) => { + await journal.appendSubmission({ ...earlierSubmission('legacy', 'l'), fence }) + await journal.appendSubmission({ ...earlierSubmission('handed', 'h', true), fence }) + await journal.resolveDispatch({ clientMessageId: 'handed', state: 'pending', fence }) + }) + await host.flushAllStreamedEvents() + await startHost() + + await host.revealSession(SESSION) + + expect(submission('legacy')).toMatchObject({ dispatchState: 'unknown', recovered: true }) + expect(submission('handed')).toMatchObject({ dispatchState: 'unknown', recovered: true }) + // Deciding them from provider history waits for a won lease (W4′d). + expect(providerHistoryWindow).not.toHaveBeenCalled() + expect(dispatch).not.toHaveBeenCalled() + }) +}) + +describe('a child that exits before its message is handed over', () => { + it('rejects the message with the exit reason instead of starting another child (W24)', async () => { + // Each child the loop starts dies between its start step and its handover step. + const awaitStarted = vi.fn(async (sessionId: string) => { + await host.handleAdapterEvent({ + type: 'ended', + sessionId, + fence: store.getRecord(sessionId)!.lease.runtimeFence, + acquisitionGeneration: `generation-${acquire.mock.calls.length}`, + reason: 'codex app-server crashed', + cause: 'unexpected-exit' + }) + }) + adapterExtras = { awaitStarted } + await host.close(SESSION) + await startHost() + + const id = await accept('hello') + + await eventually(() => expect(submission(id)?.dispatchState).toBe('rejected')) + expect(submission(id)?.reason).toContain('codex app-server crashed') + expect(acquire).toHaveBeenCalledTimes(2) + expect(dispatch).not.toHaveBeenCalled() + }) +}) + +describe('Stop withdraws what is queued', () => { + it('withdraws a crash leftover ahead of any delivery step (W17a)', async () => { + await writeAsEarlierProcess(async (journal, fence) => { + await journal.appendSubmission({ ...earlierSubmission('leftover', 'l', true), fence }) + }) + + // Stop's own open wakes the delivery loop, whose first step queues behind this Stop. + expect(await stop()).toMatchObject({ ok: true }) + + expect(submission('leftover')).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_CANCELLED + }) + expect(acquire).toHaveBeenCalledTimes(1) + }) + + it('withdraws a message whose start holds the queue: nothing is handed over (W17b)', async () => { + await host.close(SESSION) + const starting = deferred() + acquire.mockImplementationOnce(async (input) => { + await starting.promise + return spawnChild(input) + }) + const id = await accept('hello') + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + const stopped = stop() + + starting.resolve() + expect(await stopped).toMatchObject({ ok: true }) + await eventually(() => expect(submission(id)?.dispatchState).toBe('rejected')) + expect(submission(id)).toMatchObject({ reason: DISPATCH_REJECTED_CANCELLED }) + expect(submission(id)?.handedOverAt).toBeUndefined() + expect(dispatch).not.toHaveBeenCalled() + }) + + it('stops a child still proving its start, and the delivery loop ends with it (W17c)', async () => { + const ended = deferred() + const awaitStarted = vi.fn(() => ended.promise) + const closeSession = vi.fn(async () => { + ended.resolve() + return true + }) + adapterExtras = { awaitStarted, closeSession } + await host.close(SESSION) + await startHost() + acquire.mockImplementationOnce(async (input) => ({ + ...(await spawnChild(input)), + providerChildPhase: 'starting' as const + })) + const id = await accept('hello') + await eventually(() => expect(awaitStarted).toHaveBeenCalled()) + + expect(await stop()).toMatchObject({ ok: true, value: { cancelled: true } }) + + expect(await reopened(id)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_CANCELLED + }) + expect(closeSession).toHaveBeenCalled() + // A loop still waiting on that start would swallow this send; it starts a new child instead. + awaitStarted.mockImplementation(async () => undefined) + const next = await accept('after stop') + await eventually(() => expect(submission(next)?.dispatchState).toBe('accepted')) + expect(dispatch).toHaveBeenCalledTimes(1) + }) +}) + +describe('an eviction between acceptance and handover', () => { + it('rejects the message as not sent, never leaves it in doubt (W24)', async () => { + const started = deferred() + adapterExtras = { awaitStarted: () => started.promise } + await host.close(SESSION) + await startHost() + await host.hold(SESSION, 'surface-1') + const id = await accept('hello') + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + // Between the delivery loop's start step and its handover step. + await host.close(SESSION) + started.resolve() + + expect(await reopened(id)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_PROVIDER_CLOSED + }) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('rejects a queued message behind a handed-over one, which alone stays in doubt (W24)', async () => { + const second = deferred() + const awaitStarted = vi.fn(async (): Promise => undefined) + adapterExtras = { awaitStarted } + await host.close(SESSION) + await startHost() + await host.hold(SESSION, 'surface-1') + dispatch.mockResolvedValueOnce({ state: 'admitted' }) + const handed = await accept('handed over') + await eventually(() => expect(submission(handed)?.handedOverAt).toBeDefined()) + awaitStarted.mockImplementation(() => second.promise) + const queued = await accept('still queued') + await eventually(() => expect(awaitStarted).toHaveBeenCalledTimes(2)) + + await host.close(SESSION) + second.resolve() + + expect(await reopened(queued)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_PROVIDER_CLOSED + }) + expect(submission(handed)).toMatchObject({ dispatchState: 'unknown' }) + expect(dispatch).toHaveBeenCalledTimes(1) + }) + + it('does not evict an idle child while a message is still queued for it (W24)', async () => { + releaseGraceMs = 0 + const started = deferred() + adapterExtras = { awaitStarted: () => started.promise } + await host.close(SESSION) + await startHost() + await host.hold(SESSION, 'surface-1') + const id = await accept('hello') + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + host.release(SESSION, 'surface-1') + await new Promise((resolve) => setTimeout(resolve, 20)) + + started.resolve() + // Handed to the child it was queued for; the eviction may follow once nothing is owed. + await eventually(() => expect(dispatch).toHaveBeenCalledTimes(1)) + expect(dispatch.mock.calls[0]?.[0].clientMessageId).toBe(id) + expect(acquire).toHaveBeenCalledTimes(2) + }) +}) + +describe('a compaction or rewind an earlier child left prepared', () => { + async function leftPrepared(prepare: (fence: number) => Promise): Promise { + await host.close(SESSION) + await prepare(store.getRecord(SESSION)!.lease.runtimeFence) + // A new process: nothing is open and no view attaches. + await host.flushAllStreamedEvents() + await startHost() + } + + it('settles an interrupted compaction at open, so a send is accepted and delivered (R16)', async () => { + await leftPrepared((fence) => + store.setConversationCommand(SESSION, fence, { + command: 'compact', + runtimeFence: fence, + operationId: 'compact-op', + callerKey: 'client-1', + phase: 'prepared', + state: 'unknown' + }) + ) + + const id = await accept('after the compaction') + + await eventually(() => expect(submission(id)?.dispatchState).toBe('accepted')) + expect(store.getRecord(SESSION)?.conversationCommand).toMatchObject({ + phase: 'committed', + state: 'unknown' + }) + }) + + it('completes a rewind the provider already applied at open, so a send is accepted (R16)', async () => { + await leftPrepared((fence) => + persistRewindRecord(store, SESSION, fence, { + operationId: 'rewind-op', + callerKey: 'client-1', + itemId: 'orca:rewound', + providerItemId: `codex:${THREAD}:turn-1:0`, + expectedEpoch: 'epoch-before', + phase: 'provider-succeeded', + hydrationVerified: true, + retained: [] + }) + ) + + const id = await accept('after the rewind') + + await eventually(() => expect(submission(id)?.dispatchState).toBe('accepted')) + expect(store.getRecord(SESSION)?.rewind).toMatchObject({ phase: 'completed' }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts index e3da044d43c..8302b403033 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts @@ -15,6 +15,7 @@ import { attachFingerprintFields, type AgentSessionAttachParams } from './structured-agent-session-attach' +import { openTestAttachConversation } from './structured-agent-session-attach-test-conversation' import { performAttach } from './structured-agent-session-attach-flow' import type { AgentSessionCreatePhaseRecorder } from '../../observability/agent-session-instrumentation' @@ -154,6 +155,7 @@ describe('structured session acquisition options', () => { store: initialStore, adapter: withHistory('created'), journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -194,6 +196,7 @@ describe('structured session acquisition options', () => { store, adapter: withHistory('resumed'), journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-b', claimKeyId: 'key-1', @@ -228,6 +231,7 @@ describe('structured session acquisition options', () => { store, adapter: sessionAdapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -260,6 +264,7 @@ describe('structured session acquisition options', () => { store, adapter: sessionAdapter, journalRoot: root!, + openConversation: openTestAttachConversation(root!), authority: { spawnToken, claimKeyId: 'key-1', @@ -291,6 +296,7 @@ describe('structured session acquisition options', () => { store, adapter: adapter({ origin: 'created' }), journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -330,6 +336,7 @@ describe('structured session acquisition options', () => { } }), journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-b', claimKeyId: 'key-1', @@ -369,6 +376,7 @@ describe('structured session acquisition options', () => { store, adapter: sessionAdapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -408,6 +416,7 @@ describe('structured session acquisition options', () => { store, adapter: failingAdapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -499,6 +508,7 @@ describe('structured session acquisition options', () => { store: target, adapter: failingAdapter, journalRoot: root!, + openConversation: openTestAttachConversation(root!, failingAdapter), authority: { spawnToken: operationId === CREATE_OPERATION ? 'spawn-a' : 'spawn-b', claimKeyId: 'key-1', @@ -596,6 +606,7 @@ describe('the tab a create reserves', () => { store, adapter: adapter({ origin: 'created' }), journalRoot: root!, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts index 7fa1446d6c5..566e41a6635 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts @@ -122,6 +122,8 @@ export class StructuredAgentSessionAdapterRouter implements StructuredAgentSessi awaitOptionWritable = (sessionId: string): Promise => this.liveOwnerOrNull(sessionId)?.awaitOptionWritable?.(sessionId) ?? Promise.resolve() + awaitStarted = (sessionId: string): Promise => + this.liveOwnerOrNull(sessionId)?.awaitStarted?.(sessionId) ?? Promise.resolve() readOptions = (input: { sessionId: string; fence: number }) => { const reader = this.owner(input.sessionId).readOptions diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts index 9174afc7fd2..7a293193375 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts @@ -289,6 +289,10 @@ export type StructuredAgentSessionAdapter = { ): Promise>> /** Resolves once a live session can take an option write, or after a bound; never rejects. */ awaitOptionWritable?(sessionId: string): Promise + /** Resolves once a session published before it proved its start has proven it, failed, or been + * closed; at once for any other. A start that did not land resolves with the chat's words for + * why. Never rejects. */ + awaitStarted?(sessionId: string): Promise readOptions?(input: { sessionId: string; fence: number }): Promise /** Option keys skipped after a provider rejected their persisted restore value. */ readOptionRestoreFailures?(sessionId: string): readonly string[] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.test.ts index 0248799980a..4b8e1d87ba3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.test.ts @@ -11,10 +11,12 @@ import { attachFingerprintFields, type AgentSessionAttachParams } from './structured-agent-session-attach' +import { openTestAttachConversation } from './structured-agent-session-attach-test-conversation' import { performAttach, type AttachFlowInput } from './structured-agent-session-attach-flow' import { AgentSessionJournal } from '../agent-session-journal/journal-store' import { agentSessionJournalCloseRetries } from '../agent-session-journal/journal-close-retry' import * as legacyImport from '../agent-session-journal/journal-legacy-import' +import { StructuredAgentSessionHost } from './structured-agent-session-host' const NOW = 1_800_000_000_000 const SESSION = 'codex_adopting_session' @@ -121,6 +123,7 @@ async function attach( store, adapter: sessionAdapter, journalRoot: root!, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -210,7 +213,7 @@ describe('adopting a provider conversation on create', () => { } ) - it('still releases acquisition and closes the provisional journal on an import write failure', async () => { + it('still releases acquisition on an import write failure, and leaves the conversation open', async () => { root = await mkdtemp(join(tmpdir(), 'orca-adopt-write-failure-')) const transcriptPath = join(root, 'rollout.jsonl') await writeCodexRollout(transcriptPath, 'valid source') @@ -222,7 +225,51 @@ describe('adopting a provider conversation on create', () => { await expect(attach(transcriptPath, sessionAdapter)).rejects.toThrow('disk write failed') expect(sessionAdapter.acquire).toHaveBeenCalledTimes(1) expect(sessionAdapter.releaseAcquisition).toHaveBeenCalledTimes(1) - expect(close).toHaveBeenCalledTimes(1) + // The journal is the conversation's, not the attach's: a failed import closes nothing. + expect(close).not.toHaveBeenCalled() + }) + + it('leaves the conversation writable when the import fails after acquiring', async () => { + root = await mkdtemp(join(tmpdir(), 'orca-adopt-host-failure-')) + const transcriptPath = join(root, 'rollout.jsonl') + await writeCodexRollout(transcriptPath, 'valid source') + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + const host = new StructuredAgentSessionHost({ + store, + adapter: adapter(), + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-a', + now: () => NOW + }) + vi.spyOn(AgentSessionJournal.prototype, 'replaceEpochItems').mockRejectedValueOnce( + new Error('disk write failed') + ) + const attached = await host + .attach({ callerKey: 'client-1' }, attachParams(transcriptPath)) + .catch(() => null) + expect(attached?.ok).not.toBe(true) + + const body = { kind: 'message' as const, role: 'user' as const, blocks: [] } + const sent = await host.send( + { callerKey: 'client-1' }, + { + envelope: { + sessionId: SESSION, + clientOperationId: `${NOW}-${'2'.padStart(32, '0')}`, + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + } + ) + // The failed attach kept the conversation's own journal open, so the send is recorded. + expect(sent).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + await host.flushAllStreamedEvents() }) it('prepares a valid source once before acquisition and imports those exact items', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.ts index 459d21b1f3b..7a85de6a473 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.ts @@ -1,7 +1,6 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionAttachParams, AttachedJournal } from './structured-agent-session-attach' -import { agentSessionJournalCloseRetries } from '../agent-session-journal/journal-close-retry' import type { JournalReplacementItem } from '../agent-session-journal/journal-epoch-replacement' import { importLegacyTranscriptIntoJournal, @@ -62,13 +61,8 @@ export async function importAdoptedTranscript( record: AgentSessionRecord, prepared: JournalReplacementItem[] | null ): Promise { - try { - await applyAdoptedTranscript(params, attached, record, prepared) - } catch (error) { - // Publication has not taken ownership of this provisional journal yet. - await agentSessionJournalCloseRetries.closeOrRetain(attached.journal) - throw error - } + // The journal is the conversation's, which outlives a failed import; nothing here closes it. + await applyAdoptedTranscript(params, attached, record, prepared) } async function applyAdoptedTranscript( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts index c8c908aae70..eef9297e82a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts @@ -50,8 +50,16 @@ function liveReader() { submissions.push(...event.page.submissions) } } + const rows = new Map() + for (const item of items) { + if (item.body.kind === 'status') { + rows.set(item.itemId, item.body.text) + } + } return { statuses: items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])), + /** Each status row as the chat renders it: its latest revision, once. */ + statusRows: [...rows.values()], submissions, batches: events.slice(opened).filter((event) => event.type === 'batch').length } @@ -150,12 +158,16 @@ describe('an open chat receives every row its journal commits', () => { await exitBeforeProof() - expect(pane.received().statuses).toEqual([ + // The exit ends the child; the delivery loop, which reads why, rejects what it had queued. + await vi.waitFor(() => + expect(pane.received().submissions).toContainEqual( + expect.objectContaining({ clientMessageId: held, dispatchState: 'rejected' }) + ) + ) + // One row, however many of its writers reported the start. + expect(pane.received().statusRows).toEqual([ expect.stringMatching(/stopped before it finished starting: .*not signed in/) ]) - expect(pane.received().submissions).toContainEqual( - expect.objectContaining({ clientMessageId: held, dispatchState: 'rejected' }) - ) }) it('shows a revision the provider queued with no publish behind it', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts index 4b9b46716cb..d6e5e9097a2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts @@ -15,6 +15,7 @@ import type { } from './structured-agent-session-host-types' import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' import type { StructuredAgentSessionTaskQueue } from './structured-agent-session-task-queue' +import type { StructuredAgentSessionConversationOpenOptions } from './structured-agent-session-conversation-open' export type StructuredAgentSessionAttachContext = { deps: StructuredAgentSessionHostDeps @@ -38,8 +39,12 @@ export type StructuredAgentSessionAttachContext = { reconcileLeases: (sessionId: string) => Promise serialize: (sessionId: string, task: () => Promise) => Promise now: () => number - /** Paired with `sessions.delete` by `forgetStructuredAgentSession`; a failed attach that only - * deleted would leave the store's row behind. */ + /** Paired with `sessions.delete` by `forgetStructuredAgentSession`, which a close runs. */ forgetStatus: (sessionId: string) => void publishStatus?: (sessionId: string) => void + /** The conversation's one open journal, opened when closed; see `conversation-open`. */ + openConversation: ( + sessionId: string, + options?: StructuredAgentSessionConversationOpenOptions + ) => Promise } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts index 3a77aa2d6cc..6af8eb369d5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts @@ -24,8 +24,7 @@ export async function settlePostAcquisitionAttachFailure( ? 'root-exit-observed' : 'exit-proven' } - // A failed close must not prevent durable failure settlement. - await Promise.resolve(input.onAttachFailed?.()).catch(() => undefined) + input.onAcquisitionReleased?.(cause, { rootGone: exitProof !== 'unproven' }) try { await input.store.settleFailedPostAcquisitionAttachment({ sessionId: record.sessionId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts index cf37a9a6fe2..2969cb62f09 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts @@ -41,6 +41,7 @@ import { type AgentSessionCreatePhaseRecorder } from '../../observability/agent-session-instrumentation' import type { ProviderHistoryWindow } from '../agent-session-journal/journal-submission-reconciler' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' export type AttachFlowInput = { store: AgentSessionRecordStore @@ -66,8 +67,11 @@ export type AttachFlowInput = { onAcquiring?: () => Promise | void /** Settles writes already captured by the superseded journal before opening another. */ beforeJournalOpen?: () => Promise | void - /** Closes and removes partial publication after journal attachment fails. */ - onAttachFailed?: () => Promise + /** The conversation's own open journal, which the attach adopts: it never opens one itself. */ + openConversation: (record: AgentSessionRecord) => Promise + /** A failure after acquisition released the session's acquisition; `cause` is that failure and + * `rootGone` whether the release saw the provider root go. */ + onAcquisitionReleased?: (cause: unknown, verdict: { rootGone: boolean }) => void } export async function performAttach( @@ -208,6 +212,7 @@ export async function performAttach( params, journalRoot: input.journalRoot, adapter: input.adapter, + openConversation: input.openConversation, providerHistoryWindow }) await importAdoptedTranscript(params, attached, record, preparedTranscript.items) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts index bf62c3fffce..591c09d8e32 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts @@ -23,9 +23,16 @@ import { import { refuseAgentSessionMutation } from './structured-agent-session-mutation-admission' import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' -import { forgetStructuredAgentSession } from './structured-agent-session-host-lifetime' +import type { + StructuredAgentSessionProviderChild, + StructuredAgentSessionStopVerdict +} from './structured-agent-session-host-types' +import { + endProviderChild, + indexProviderChild, + structuredAgentSessionConversationFence +} from './structured-agent-session-provider-child' import type { DeferredStructuredAgentSessionEventSink } from './structured-agent-session-event-sink' -import { agentSessionJournalCloseRetries } from '../agent-session-journal/journal-close-retry' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { addAgentSessionCreatePhaseAttributes, @@ -94,6 +101,10 @@ async function runAttach( ): Promise> { const sessionId = params.envelope.sessionId const recordPhase = options.recordPhase + // Readers of a conversation already open are re-baselined when this attach moves its fence. + const fenceBefore = context.sessions.has(sessionId) + ? structuredAgentSessionConversationFence(context.deps.store, sessionId) + : null if (options.admitRecoveryTicket && !options.admitRecoveryTicket()) { return refuseAgentSessionMutation({ code: 'agent_session_checkpoint_stale', @@ -113,17 +124,18 @@ async function runAttach( context.runtimeState.probeOwner(sessionId) ) // A child this attach spawns writes through a sink this attempt owns. Only a successful - // attach makes it the session's; any other exit closes it with whatever the child queued. + // attach makes the child and its sink the session's; any other exit closes the sink with + // whatever the child queued, and leaves the conversation's child as it was. const attemptSink = context.runtimeState.mintEventSink(sessionId) - let attemptSinkAdopted = false // Read before the reserve clears it: how the previous generation ended decides how whatever it // left running is settled. const priorDeathEvidence = context.deps.store.getRecord(sessionId)?.lease.deathEvidence ?? null - const attached = stampFailedCreateOwnerVerdict( - context.deps.store, - callerKey, - params.envelope, - await performAttach({ + const attempt: { candidate: AttachCandidate | null; committed: boolean } = { + candidate: null, + committed: false + } + try { + const attached = await performAttach({ store: context.deps.store, adapter: context.deps.adapter, journalRoot: context.deps.journalRoot, @@ -147,66 +159,47 @@ async function runAttach( params, now: () => context.now(), recordPhase, - // Site 9: this closes the PRIOR map entry it drops, never the provisional - // journal — it has no reference to that one. `onAttached` owns that. - onAttachFailed: async () => { - await forgetStructuredAgentSession(context, sessionId) - context.runtimeState.currentEventSink(sessionId)?.close() - context.runtimeState.discardEventSink(sessionId) + openConversation: async (record) => { + const conversation = await context.openConversation(record.sessionId, { + acquisition: true + }) + if (!conversation) { + throw new Error('agent_session_identity_required') + } + return conversation.journal }, + // The cleanup released the acquisition, which for a re-attach is the live child itself. + onAcquisitionReleased: (cause, verdict) => + endReleasedChild(context, sessionId, cause, verdict), onAttached: async (attached, acquisitionGeneration, acquiredOwner, providerChildPhase) => { - const fence = context.deps.store.getRecord(sessionId)?.lease.runtimeFence ?? 0 - const previous = context.sessions.get(sessionId) - const previousFence = previous?.fence + const fence = structuredAgentSessionConversationFence(context.deps.store, sessionId) + const current = context.sessions.get(sessionId)?.child ?? null // A re-attach to a live child keeps the sink that child already writes through. const eventSink = acquiredOwner ? attemptSink : (context.runtimeState.currentEventSink(sessionId) ?? attemptSink) - // Site 8: the provisional journal has no owner until the map takes it, - // and the barrier below throws by design. - try { - if (acquiredOwner) { - // Before the drain: the buffered events are the new child's, never a stale row's. - await settleStaleStructuredAgentSessionState({ - journal: attached.journal, - sessionId, - fence, - acquisitionGeneration, - deathEvidence: priorDeathEvidence - }) - } - await bindAndDrain(eventSink, attached.journal, fence, (activity) => - context.subscribers.publish(sessionId, attached.journal, activity) - ) - } catch (error) { - await agentSessionJournalCloseRetries.closeOrRetain(attached.journal) - throw error + if (acquiredOwner) { + // Before the drain: the buffered events are the new child's, never a stale row's. + await settleStaleStructuredAgentSessionState({ + journal: attached.journal, + sessionId, + fence, + acquisitionGeneration, + deathEvidence: priorDeathEvidence + }) } - // Site 10: a `set` over a live entry would orphan its handle — and a - // close that REJECTED did not release it. The replacement is therefore - // ABORTED rather than completed over a handle nothing can reach again: - // `previous` stays indexed, so teardown still owns it and can retry. - if (previous && previous.journal !== attached.journal) { - try { - await previous.journal.close() - } catch (error) { - await agentSessionJournalCloseRetries.closeOrRetain(attached.journal) - throw error + await bindAndDrain(eventSink, attached.journal, fence, (activity) => + context.subscribers.publish(sessionId, attached.journal, activity) + ) + attempt.candidate = { + sink: eventSink, + child: { + generation: acquisitionGeneration ?? current?.generation ?? null, + fence, + // A re-attach to a live child keeps what that child already proved. + phase: acquiredOwner ? providerChildPhase : (current?.phase ?? 'ready') } } - context.runtimeState.adoptEventSink(sessionId, eventSink) - attemptSinkAdopted = eventSink === attemptSink - context.sessions.set(sessionId, { - journal: attached.journal, - params, - fence, - hasProviderChild: true, - // A re-attach to a live child keeps what that child already proved. - providerChildPhase: acquiredOwner - ? providerChildPhase - : (previous?.providerChildPhase ?? 'ready'), - acquisitionGeneration: acquisitionGeneration ?? previous?.acquisitionGeneration ?? null - }) await recoverStructuredRewind( context.deps.store, sessionId, @@ -216,21 +209,59 @@ async function runAttach( context.now ) await recoverInterruptedCompaction(context.deps.store, sessionId, attached.journal, fence) - if (attached.recovery) { - context.subscribers.reset(sessionId, attached.journal, attached.recovery.reset, fence) - } else if (previousFence !== undefined && previousFence !== fence) { + if (fenceBefore !== null && fence !== fenceBefore) { context.subscribers.snapshot(sessionId, attached.journal, fence) } else { context.subscribers.publish(sessionId, attached.journal) } } - }).finally(() => { - if (!attemptSinkAdopted) { - attemptSink.close() - } }) - ) - return attached + const { candidate } = attempt + const conversation = context.sessions.get(sessionId) + if (attached.ok && candidate && conversation) { + context.runtimeState.adoptEventSink(sessionId, candidate.sink) + attempt.committed = candidate.sink === attemptSink + indexProviderChild(conversation, candidate.child) + context.publishStatus?.(sessionId) + } + return stampFailedCreateOwnerVerdict(context.deps.store, callerKey, params.envelope, attached) + } finally { + if (!attempt.committed) { + attemptSink.close() + } + } +} + +type AttachCandidate = { + child: StructuredAgentSessionProviderChild + sink: DeferredStructuredAgentSessionEventSink +} + +function endReleasedChild( + context: StructuredAgentSessionAttachContext, + sessionId: string, + cause: unknown, + verdict: StructuredAgentSessionStopVerdict +): void { + const session = context.sessions.get(sessionId) + const child = session?.child + if ( + !session || + !child || + !endProviderChild(session, { + generation: child.generation, + fence: child.fence, + cause: 'attach-failed', + reason: cause instanceof Error ? cause.message : String(cause), + duringStartup: child.phase === 'starting', + ...verdict + }) + ) { + return + } + context.runtimeState.currentEventSink(sessionId)?.close() + context.runtimeState.discardEventSink(sessionId) + context.publishStatus?.(sessionId) } /** Binds the sink to the journal and waits for the barrier the host publishes diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts index 97f429933d8..4e15b23d4d3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts @@ -13,6 +13,7 @@ import { journalDirectoryFor } from '../agent-session-journal/journal-paths' import type { ProviderHistoryWindow } from '../agent-session-journal/journal-submission-reconciler' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { openTestAttachConversation } from './structured-agent-session-attach-test-conversation' import { attachJournal, journalIdentityFor, @@ -85,6 +86,7 @@ async function attach(adapter: StructuredAgentSessionAdapter) { record: RECORD, params: PARAMS, journalRoot: root, + openConversation: openTestAttachConversation(root), adapter }) journals.track(attached.journal) @@ -147,4 +149,37 @@ describe('attachJournal restart reconciliation', () => { expect(attached.unconfirmedClientMessageIds).toEqual(['cm_1']) expect(attached.journal.submissions()[0]?.dispatchState).toBe('unknown') }) + + it('leaves a message the open conversation still has queued alone (W4′e)', async () => { + const journal = await journals.open({ + identity: IDENTITY, + journalDir: journalDirectoryFor(root, { + workspaceId: IDENTITY.workspaceId, + sessionId: IDENTITY.sessionId + }) + }) + await journal.appendSubmission({ + clientMessageId: 'queued', + payloadFingerprint: digestPayload('still queued'), + body: userMessage('still queued'), + fence: RECORD.lease.runtimeFence, + handoverRecorded: true + }) + // History that holds nothing: absence would prove a handed-over message undelivered. + const { adapter, dispatch } = adapterWith(async () => window()) + + const attached = await attachJournal({ + record: RECORD, + params: PARAMS, + journalRoot: root, + + adapter, + openConversation: async () => journal + }) + + expect(attached.journal).toBe(journal) + expect(journal.submissions()[0]).toMatchObject({ dispatchState: 'pending' }) + expect(journal.submissions()[0]?.handedOverAt).toBeUndefined() + expect(dispatch).not.toHaveBeenCalled() + }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-test-conversation.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-test-conversation.ts new file mode 100644 index 00000000000..1e2322cbc6d --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-test-conversation.ts @@ -0,0 +1,18 @@ +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { openStructuredAgentSessionConversationJournal } from './structured-agent-session-conversation-open' + +/** For a test that attaches without a host: the conversation's journal, through the one open a + * host would take, so the attach under test adopts it the way it adopts the host's. */ +export function openTestAttachConversation( + journalRoot: string, + adapter: Pick = {} +): (record: AgentSessionRecord) => Promise { + return async (record) => + ( + await openStructuredAgentSessionConversationJournal({ journalRoot, adapter }, record, { + acquisition: true + }) + ).session.journal +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach.ts index de5fa297daf..4cf6505d276 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach.ts @@ -35,15 +35,9 @@ import { } from '../../../shared/agent-session-mutation-envelope' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { agentSessionProviderHandleChainHead } from '../../../shared/agent-session-provider-handle' -import { agentSessionJournalCloseRetries } from '../agent-session-journal/journal-close-retry' -import { journalDirectoryFor } from '../agent-session-journal/journal-paths' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { reconcileJournalSubmissionsAgainstHistory } from '../agent-session-journal/journal-restart-reconciliation' import type { ProviderHistoryWindow } from '../agent-session-journal/journal-submission-reconciler' -import { - openAgentSessionJournalWithRecovery, - type AgentSessionJournalRecovery -} from './agent-session-journal-recovery' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { structuredAgentSessionRefusalMessage } from './structured-agent-session-refusal-message' @@ -165,16 +159,13 @@ export function journalIdentityFor( export type AttachedJournal = { journal: AgentSessionJournal - recovery: AgentSessionJournalRecovery | null - /** Submissions still `unknown` after this open: the crash boundary settled - * them there and provider history could not decide them either. */ + /** Submissions the crash boundary left `unknown` that provider history could not decide. */ unconfirmedClientMessageIds: string[] } /** - * Open the session's journal, recovering it when the stored one is unusable, - * settle every submission left in flight by a previous process, then let - * provider history decide the ones it can prove. + * The conversation's journal — opened, and its crash boundary settled, by the conversation's own + * open — with provider history deciding the submissions that boundary could only doubt. * * Why the reconciliation belongs HERE and nowhere else: this runs after the * record store handed this host the lease and before `onAttached` starts a @@ -182,54 +173,44 @@ export type AttachedJournal = { * is read, and the window stays valid until the resume consumes it. Every other * settlement site — a proven child exit — runs while the host * may still start another child, and a read there could be overtaken before it - * is acted on. Orca still never re-sends: this decides state only. + * is acted on. Orca still never re-sends: this decides state only. A queued + * submission is left alone: it was never handed over, so history cannot hold it. */ export async function attachJournal(input: { record: AgentSessionRecord params: AgentSessionAttachParams journalRoot: string adapter: StructuredAgentSessionAdapter + /** The host's open conversation, whose journal the attach adopts. */ + openConversation: (record: AgentSessionRecord) => Promise /** Provider history sampled before a new child is acquired. `null` means the * adapter had no usable history; omit to read lazily for direct callers. */ providerHistoryWindow?: ProviderHistoryWindow | null }): Promise { const identity = journalIdentityFor(input.record, input.params) const fence = input.record.lease.runtimeFence - const historyFilePath = input.adapter.historyFilePath - ? await input.adapter.historyFilePath({ identity }) - : null - const opened = await openAgentSessionJournalWithRecovery({ + const journal = await input.openConversation(input.record) + const settled = await reconcileAgainstProviderHistory({ + adapter: input.adapter, identity, - journalDir: journalDirectoryFor(input.journalRoot, { - workspaceId: identity.workspaceId, - sessionId: identity.sessionId - }), + journal, fence, - historyFilePath + accountHome: input.record.accountHome, + ...(Object.hasOwn(input, 'providerHistoryWindow') + ? { history: input.providerHistoryWindow } + : {}) }) - try { - // That await is a WRITE. A failure in it leaves the journal with no caller - // holding a reference to close it. - const unconfirmed = await opened.journal.markPendingSubmissionsUnknown(fence) - const settled = await reconcileAgainstProviderHistory({ - adapter: input.adapter, - identity, - journal: opened.journal, - fence, - accountHome: input.record.accountHome, - ...(Object.hasOwn(input, 'providerHistoryWindow') - ? { history: input.providerHistoryWindow } - : {}) - }) - return { - ...opened, - unconfirmedClientMessageIds: unconfirmed.filter((id) => !settled.includes(id)) - } - } catch (error) { - // A rejected close leaves the handle open, so the journal is retained for a - // later retry rather than dropped along with the only reference to it. - await agentSessionJournalCloseRetries.closeOrRetain(opened.journal) - throw error + return { + journal, + unconfirmedClientMessageIds: journal + .submissions() + .filter( + (entry) => + entry.dispatchState === 'unknown' && + entry.recovered === true && + !settled.includes(entry.clientMessageId) + ) + .map((entry) => entry.clientMessageId) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts index 5d922d72350..8594b6d58ba 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts @@ -12,6 +12,7 @@ import type { StructuredAgentSessionHostDeps, StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' export class StructuredAgentSessionBackgroundTaskChannel { constructor( @@ -48,7 +49,7 @@ export class StructuredAgentSessionBackgroundTaskChannel { return this.subscribers.open({ ...input, journal: session.journal, - fence: this.deps.store.getRecord(input.sessionId)?.lease.runtimeFence ?? 0, + fence: structuredAgentSessionConversationFence(this.deps.store, input.sessionId), ...(backgroundTasks !== undefined ? { backgroundTasks } : {}) }) } @@ -57,7 +58,11 @@ export class StructuredAgentSessionBackgroundTaskChannel { const session = this.sessions.get(sessionId) const state = publishedState !== undefined ? publishedState : this.state(sessionId) if (session && state !== undefined) { - this.subscribers.backgroundTasks(sessionId, state, session.fence) + this.subscribers.backgroundTasks( + sessionId, + state, + structuredAgentSessionConversationFence(this.deps.store, sessionId) + ) this.onPublished(sessionId) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts index a73e46b898e..537a2ac695f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts @@ -103,10 +103,11 @@ describe('Claude root-exit eviction', () => { { journal, params, - fence, - hasProviderChild: true, - providerChildPhase: 'ready', - acquisitionGeneration: acquisition.acquisitionGeneration ?? null + child: { + generation: acquisition.acquisitionGeneration ?? null, + fence, + phase: 'ready' + } } ] ]) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts index b9e8f8e506d..303ecbe92fc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts @@ -78,10 +78,7 @@ function hostSession(journal: AgentSessionJournal): StructuredAgentSessionHostSe return { journal, params: {} as StructuredAgentSessionHostSession['params'], - fence: 1, - hasProviderChild: false, - providerChildPhase: 'ready', - acquisitionGeneration: null + child: null } } @@ -178,40 +175,34 @@ describe('the registry', () => { }) describe('the attach orchestration', () => { - it('ABORTS the map replacement when the previous journal will not close', async () => { - const previousDir = join(root, 'previous') - const provisionalDir = join(root, 'provisional') - const previous = flakyClose( - await journals.open({ identity: IDENTITY, journalDir: previousDir }), - 1 - ) - const provisional = await journals.open({ - identity: IDENTITY, - journalDir: provisionalDir - }) - attachFlow.journal = provisional - const sessions = new Map([[SESSION, hostSession(previous)]]) + // The attach adopts the conversation's one open journal; it never opens a second handle, so + // there is no replacement to abort and no provisional journal to close. + it('keeps the journal it adopted indexed and open when an attach succeeds', async () => { + const directory = join(root, 'adopted') + const journal = await journals.open({ identity: IDENTITY, journalDir: directory }) + attachFlow.journal = journal + const sessions = new Map([[SESSION, hostSession(journal)]]) + await attachStructuredAgentSession(attachContext(sessions), 'caller-1', attachParams) + + expect(sessions.get(SESSION)?.journal).toBe(journal) await expect( - attachStructuredAgentSession(attachContext(sessions), 'caller-1', attachParams) - ).rejects.toThrow('close rejected') - - // The live entry is UNTOUCHED: overwriting it would have left its handle - // open with nothing able to reach it again. - expect(sessions.get(SESSION)?.journal).toBe(previous) - // And the provisional journal is owned by the registry, not orphaned. + journal.appendItem( + { provider: 'orca', clientMessageId: 'after-attach' }, + { + kind: 'status', + text: 'still writable' + } + ) + ).resolves.toBeDefined() expect(agentSessionJournalCloseRetries.pendingDirectories).toEqual([]) - await expectNothingHoldsTheDirectory(provisionalDir) }) - it('retains the provisional journal when its own close rejects on the barrier path', async () => { - const provisionalDir = join(root, 'provisional-barrier') - const provisional = flakyClose( - await journals.open({ identity: IDENTITY, journalDir: provisionalDir }), - 1 - ) - attachFlow.journal = provisional - const sessions = new Map() + it('leaves the conversation indexed and open when the sink barrier fails', async () => { + const directory = join(root, 'adopted-barrier') + const journal = await journals.open({ identity: IDENTITY, journalDir: directory }) + attachFlow.journal = journal + const sessions = new Map([[SESSION, hostSession(journal)]]) const context = attachContext(sessions) const failing = { sink: {}, @@ -229,9 +220,19 @@ describe('the attach orchestration', () => { 'sink barrier failed' ) - expect(sessions.size).toBe(0) - // Retained rather than dropped, so teardown can still release the handle. - expect(agentSessionJournalCloseRetries.pendingDirectories).toEqual([provisionalDir]) + // A failed attach does not end the conversation: its queued messages and the failure row + // are written into this same journal. + expect(sessions.get(SESSION)?.journal).toBe(journal) + await expect( + journal.appendItem( + { provider: 'orca', clientMessageId: 'after-failure' }, + { + kind: 'status', + text: 'still writable' + } + ) + ).resolves.toBeDefined() + expect(agentSessionJournalCloseRetries.pendingDirectories).toEqual([]) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts new file mode 100644 index 00000000000..07d1fb6b5ff --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -0,0 +1,159 @@ +// The one way a conversation's journal becomes open on this host: for a send, for a reader, and +// for an attach that finds none open. +// +// It opens with recovery, so an unusable journal is rebuilt rather than refused, and it marks what +// an earlier host process handed over and left unanswered as in doubt, and settles what it left +// running — the crash boundary. That +// needs no lease: provider history decides such a row later, under a won lease, in the attach. A +// row an earlier process accepted and never handed over is the delivery loop's, which the open +// wakes. Nothing here starts a provider child. + +import type { AgentJournalResetReason } from '../../../shared/agent-session-journal-types' +import { journalDirectoryFor } from '../agent-session-journal/journal-paths' +import { openAgentSessionJournalWithRecovery } from './agent-session-journal-recovery' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { + attachFingerprintFields, + journalIdentityFor, + type AgentSessionAttachParams +} from './structured-agent-session-attach' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' +import type { + StructuredAgentSessionHostDeps, + StructuredAgentSessionHostSession +} from './structured-agent-session-host-types' + +export type OpenedStructuredAgentSessionConversation = { + session: StructuredAgentSessionHostSession + /** Set when the journal was rebuilt on the way; readers reload from a snapshot. */ + reset: AgentJournalResetReason | null +} + +export type StructuredAgentSessionConversationOpenDeps = { + store: Pick + adapter: Pick + journalRoot: string + onEventSinkError?: StructuredAgentSessionHostDeps['onEventSinkError'] +} + +/** An acquisition's own open: its reserve cleared the record's death evidence, so it settles + * what the gone generation left running itself, from what it read before. */ +export type StructuredAgentSessionConversationOpenOptions = { acquisition?: boolean } + +export type StructuredAgentSessionConversationOpenContext = { + deps: StructuredAgentSessionConversationOpenDeps + sessions: Map + /** Indexes a conversation that just became open; the host publishes it and wakes delivery. */ + adoptOpened: ( + sessionId: string, + opened: OpenedStructuredAgentSessionConversation + ) => Promise +} + +/** The open conversation, or null when this host has no record of it. For a caller inside the + * session's serialize, which is what makes "not open yet" exact. */ +export async function openStructuredAgentSessionConversation( + context: StructuredAgentSessionConversationOpenContext, + sessionId: string, + options: StructuredAgentSessionConversationOpenOptions = {} +): Promise { + const open = context.sessions.get(sessionId) + if (open) { + return open + } + const record = context.deps.store.getRecord(sessionId) + if (!record) { + return null + } + const opened = await openStructuredAgentSessionConversationJournal(context.deps, record, options) + await context.adoptOpened(sessionId, opened) + return opened.session +} + +/** The open itself, indexed by nobody yet: the caller adopts the result. */ +export async function openStructuredAgentSessionConversationJournal( + deps: Omit, + record: AgentSessionRecord, + options: StructuredAgentSessionConversationOpenOptions = {} +): Promise { + const { sessionId } = record + const fence = record.lease.runtimeFence + const params = attachParamsForRecord(record, { + clientOperationId: `read-restore:${sessionId}`, + expectedRuntimeFence: fence + }) + const identity = journalIdentityFor(record, params) + const opened = await openAgentSessionJournalWithRecovery({ + identity, + journalDir: journalDirectoryFor(deps.journalRoot, { + workspaceId: record.location.workspaceId, + sessionId + }), + fence, + historyFilePath: (await deps.adapter.historyFilePath?.({ identity })) ?? null + }) + try { + // A queued row found here is a leftover the delivery loop's first step rejects; a handed-over + // one is only doubt, which provider history decides under a won lease. + await opened.journal.markPendingSubmissionsUnknown(fence) + } catch (error) { + deps.onEventSinkError?.({ sessionId, error }) + } + try { + // No child in this process writes to a journal nobody had open, so whatever it shows running + // belongs to a generation that is gone, whatever the lease still claims. Settled before any + // reader or child sees it. + if (!options.acquisition) { + await settleStaleStructuredAgentSessionState({ + journal: opened.journal, + sessionId, + fence, + acquisitionGeneration: null, + deathEvidence: record.lease.deathEvidence ?? null + }) + } + } catch (error) { + // Best effort: the next acquire re-derives it. + deps.onEventSinkError?.({ sessionId, error }) + } + return { + session: { journal: opened.journal, params, child: null }, + reset: opened.recovery?.reset ?? null + } +} + +export function attachParamsForRecord( + record: AgentSessionRecord, + input: { + clientOperationId: string + expectedRuntimeFence: number + } +): AgentSessionAttachParams { + const params: AgentSessionAttachParams = { + envelope: { + sessionId: record.sessionId, + clientOperationId: input.clientOperationId, + expectedRuntimeFence: input.expectedRuntimeFence, + payloadFingerprint: '' + }, + location: record.location, + provider: record.provider, + agent: record.provider, + accountHome: record.accountHome, + runtimeKind: 'native' + } + return { + ...params, + envelope: { + ...params.envelope, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.attach', + sessionId: record.sessionId, + fields: attachFingerprintFields(params) + }) + } + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts index 202ffe50ae4..7014e394728 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts @@ -37,10 +37,7 @@ function session(journal: AgentSessionJournal) { return { journal, params: hostTestAttachParams(null), - fence: 0, - hasProviderChild: false, - providerChildPhase: 'ready' as const, - acquisitionGeneration: null + child: null } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts index d553fbaecf3..8ad51871c24 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts @@ -190,6 +190,7 @@ describe('dead structured-session generation settlement', () => { | 'submissions' | 'markPendingSubmissionsUnknown' | 'rejectPendingSubmissions' + | 'rejectQueuedSubmissions' | 'appendLifecycleBatch' > = { snapshot: () => ({ @@ -203,6 +204,9 @@ describe('dead structured-session generation settlement', () => { rejectPendingSubmissions: async () => { throw new Error('journal_closed') }, + rejectQueuedSubmissions: async () => { + throw new Error('journal_closed') + }, appendLifecycleBatch: async () => { throw new Error('journal_closed') } @@ -273,7 +277,7 @@ describe('dead structured-session generation settlement', () => { pendingSubmissionReason: 'provider_closed_before_acknowledgement', verdict: { state: 'interrupted', completedAt: 1_000 }, unexpectedExitReason: 'claude stream-json exited (code 1): not signed in', - exitedDuringStartup: true + exitedDuringStartup: { generation: 'generation-1' } }) const reason = diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index 05f7b4a7fe8..9d20d93a82f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -11,6 +11,7 @@ import { cancelledJournalPromptBody } from '../agent-session-journal/journal-prompt-body-bounds' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { structuredAgentSessionStartFailureRow } from './structured-agent-session-start-failure-row' import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import { runningTurnLifecycleRevisions, @@ -57,6 +58,14 @@ export function providerStartupFailureOutcome(reason?: string): string { : 'The provider stopped before it finished starting.' } +/** Why a message accepted but not yet handed over was rejected when its child exited. */ +export function providerExitBeforeDeliveryRejection(reason?: string): string { + const detail = exitReasonDetail(reason) + return detail + ? `The provider stopped before this message was sent: ${detail}.` + : 'The provider stopped before this message was sent.' +} + /** Why a send a child that never started left unwritten was rejected. The child's own diagnostic is * the cause the user can act on, so it is the reason, in the words the chat row uses. */ export function providerStartupFailureRejection(cause?: unknown): string { @@ -74,7 +83,7 @@ function exitReasonDetail(reason: string | undefined): string | undefined { type DeadGenerationSubmission = Pick< ReturnType[number], - 'clientMessageId' | 'dispatchState' | 'recovered' + 'clientMessageId' | 'dispatchState' | 'recovered' | 'handoverRecorded' | 'handedOverAt' > export type DeadGenerationJournal = { @@ -144,8 +153,9 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { showUnexpectedExitOutcome?: boolean /** Why the provider stopped, when the host has it. Rendered with the outcome copy. */ unexpectedExitReason?: string - /** The provider never finished starting; the outcome says so instead of naming a response. */ - exitedDuringStartup?: boolean + /** The provider never finished starting: the start that failed, keyed by the child's + * generation. Its row is the one the delivery loop writes for the same start. */ + exitedDuringStartup?: { generation: string | null } onError?: (sessionId: string, error: unknown) => void }): Promise { try { @@ -154,9 +164,10 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { if (!showUnexpectedExitOutcome && !hasUnfinishedWork) { return true } - // A child that never proved its start accepted nothing — input is written only after it - // initializes — so every send it left unanswered is provably unwritten and is rejected with the - // child's own diagnostic. A proven child's unanswered sends stay in doubt. + // A queued message is the delivery loop's to settle: it was never handed to this child. A + // child that never proved its start accepted nothing either — input is written only after it + // initializes — so every send it was handed is rejected with the child's own diagnostic. A + // proven child's handed-over sends stay in doubt. await (input.exitedDuringStartup ? input.journal.rejectPendingSubmissions( input.fence, @@ -165,17 +176,21 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { : input.journal.markPendingSubmissionsUnknown(input.fence, input.pendingSubmissionReason)) const items = input.journal.snapshot().items const mutations: JournalLifecycleMutationInput[] = [] - if (showUnexpectedExitOutcome) { + if (showUnexpectedExitOutcome && input.exitedDuringStartup) { + // Until views stop starting children, a start can die with nothing queued for the loop. + mutations.push( + structuredAgentSessionStartFailureRow( + input.exitedDuringStartup.generation ?? input.settlementId, + providerStartupFailureOutcome(input.unexpectedExitReason) + ) + ) + } else if (showUnexpectedExitOutcome) { mutations.push({ kind: 'item', identity: { provider: 'orca', clientMessageId: input.settlementId }, body: { kind: 'status', - text: boundJournalStatusText( - input.exitedDuringStartup - ? providerStartupFailureOutcome(input.unexpectedExitReason) - : unexpectedProviderExitOutcome(input.unexpectedExitReason) - ) + text: boundJournalStatusText(unexpectedProviderExitOutcome(input.unexpectedExitReason)) } }) } @@ -298,7 +313,9 @@ function hasUnsettledSubmission(journal: DeadGenerationJournal): boolean { return submissions ? submissions.some( (submission) => - submission.dispatchState === 'pending' || + // A queued message is not work in progress: nothing has it yet. + (submission.dispatchState === 'pending' && + !(submission.handoverRecorded && submission.handedOverAt === undefined)) || (submission.dispatchState === 'unknown' && submission.recovered !== true) ) : (journal.pendingSubmissions?.().length ?? 0) > 0 diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts new file mode 100644 index 00000000000..e12acb3a12e --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts @@ -0,0 +1,248 @@ +// The one thing that starts a provider child for a send, the one thing that hands a message to +// it, and the one thing that settles a queued message because of a start, a child or a leftover. +// +// A send is accepted on its own serialized step and returns; this loop does the rest. It exists +// for a session exactly while a message is queued there — accepted, not yet handed over — and +// every step re-reads the journal and the conversation's child record to decide, so there is no +// loop state to disagree with them. Each step is its own serialized task. That is what lets a Stop +// that arrives while a start holds the queue withdraw the queued messages before the handover that +// would have written them. Stop and the conversation's close are the only other writers of a +// queued message: a child's exit only ends the child, and this loop reads why. + +import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' +import { DISPATCH_REJECTED_HOST_RESTARTED } from '../../../shared/structured-agent-session-dispatch-rejection' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { + providerExitBeforeDeliveryRejection, + providerStartupFailureOutcome +} from './structured-agent-session-dead-generation-settlement' +import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-hold-resume' +import type { + StructuredAgentSessionEndedChild, + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChildIdentity +} from './structured-agent-session-host-types' +import { + oldestQueuedSubmission, + recordStructuredAgentSessionStartFailure +} from './structured-agent-session-start-failure-row' +import { failedProviderChildStart } from './structured-agent-session-provider-child' +import { handOverSubmission } from './structured-agent-session-turns' + +export type StructuredAgentSessionDeliveryLoopDeps = { + sessions: ReadonlyMap + adapter: StructuredAgentSessionAdapter + serialize: (sessionId: string, task: () => Promise) => Promise + /** A start step, tracked from enqueue so quit waits for the child it may produce. */ + trackStart: (start: Promise) => Promise + /** Gives the session a provider child if it has none; for a caller inside `serialize`. */ + ensureProviderChild: (sessionId: string) => Promise + /** The fence the conversation's own writes carry; see `structuredAgentSessionConversationFence`. */ + conversationFence: (sessionId: string) => number + /** What the chat says when the session could not be made ready. */ + startFailureText: (sessionId: string, cause: AgentSessionWireRefusal) => string + onError: (sessionId: string, error: unknown) => void +} + +type Step = 'continue' | 'stop' + +type Prepared = + | 'stop' + | { ok: false; refusal: AgentSessionWireRefusal } + | { ok: true; awaited: StructuredAgentSessionProviderChildIdentity | null } + +type StartFailure = { startKey: string | null; text: string } + +export class StructuredAgentSessionDeliveryLoop { + private readonly running = new Set() + private disposed = false + + constructor(private readonly deps: StructuredAgentSessionDeliveryLoopDeps) {} + + isRunning(sessionId: string): boolean { + return this.running.has(sessionId) + } + + /** Quit: no step after this one starts a child or hands a message over. */ + dispose(): void { + this.disposed = true + } + + /** From inside the session's serialize, after a message was accepted or the conversation + * opened. A loop already running re-reads the journal on its next step. */ + wake(sessionId: string): void { + if (this.disposed || this.running.has(sessionId)) { + return + } + this.running.add(sessionId) + void this.run(sessionId) + } + + private async run(sessionId: string): Promise { + try { + for (;;) { + const prepared = await this.deps.trackStart( + this.deps.serialize(sessionId, () => this.prepare(sessionId)) + ) + if (prepared === 'stop') { + return + } + if (!prepared.ok) { + const text = this.deps.startFailureText(sessionId, prepared.refusal) + await this.deps.serialize(sessionId, () => this.fail(sessionId, { startKey: null, text })) + return + } + // A child published before it proved its start takes no input yet; waited for outside + // the queue so a Stop can reach it meanwhile. + const failure = await this.deps.adapter.awaitStarted?.(sessionId) + const handed = await this.deps.serialize(sessionId, () => + this.handOver(sessionId, prepared.awaited, failure || null) + ) + if (handed === 'stop') { + return + } + } + } catch (error) { + this.deps.onError(sessionId, error) + const text = this.deps.startFailureText(sessionId, { + code: 'agent_session_owner_restart_failed', + message: error instanceof Error ? error.message : String(error) + }) + await this.deps + .serialize(sessionId, () => this.fail(sessionId, { startKey: null, text })) + .catch((failure: unknown) => { + // Rows left queued are rejected by the next open, or by the next loop an accept wakes. + this.running.delete(sessionId) + this.deps.onError(sessionId, failure) + }) + } + } + + /** Settles what an earlier host process left queued, then makes the session ready. */ + private async prepare(sessionId: string): Promise { + const session = this.deps.sessions.get(sessionId) + if (!session || this.disposed) { + return this.stop(sessionId) + } + await session.journal.rejectQueuedSubmissions( + this.deps.conversationFence(sessionId), + DISPATCH_REJECTED_HOST_RESTARTED, + // A handle closes only with nothing queued, so one an earlier handle wrote is a leftover. + (submission) => session.journal.wroteBeforeOpen(submission.acceptedSequence) + ) + const oldest = oldestQueuedSubmission(session) + if (!oldest) { + return this.stop(sessionId) + } + const failedStart = startThatFailedWhileQueued(session, oldest) + if (failedStart) { + return this.fail(sessionId, failedStart) + } + const ready = await this.deps.ensureProviderChild(sessionId) + if (!ready.ok) { + return ready + } + const child = this.deps.sessions.get(sessionId)?.child + // The child this run waits on; handover checks it is still the one there. + return { + ok: true, + awaited: child ? { generation: child.generation, fence: child.fence } : null + } + } + + private async handOver( + sessionId: string, + awaited: StructuredAgentSessionProviderChildIdentity | null, + startFailure: string | null + ): Promise { + const session = this.deps.sessions.get(sessionId) + if (!session || this.disposed) { + return this.stop(sessionId) + } + // Re-derived here, not carried from the start: the child may have ended, or another may have + // taken its place, since. + const { child } = session + const awaitedChild = + child && awaited && child.generation === awaited.generation && child.fence === awaited.fence + ? child + : null + // The host's `starting` trails the adapter's `started` by one serialized step, so for the child + // waited on, the adapter's own answer decides whether its start landed. + if (!awaitedChild || (awaitedChild.phase === 'starting' && startFailure !== null)) { + // The child waited on is gone, replaced by another, or settled its start without proving it. + const ended = awaitedChild ? undefined : session.lastEndedChild + // A user's Stop is not a failure: the next step starts, or waits on, a child for what is + // queued. A host stop is: its cause is why the start did not land. + if (ended?.cause === 'user-stop') { + return 'continue' + } + return this.fail(sessionId, { + startKey: awaited?.generation ?? null, + text: ended ? endedChildRejection(ended) : (startFailure ?? providerStartupFailureOutcome()) + }) + } + const next = oldestQueuedSubmission(session) + if (!next) { + return this.stop(sessionId) + } + await handOverSubmission( + { + sessionId, + journal: session.journal, + fence: awaitedChild.fence, + adapter: this.deps.adapter, + providerChildPhase: () => this.deps.sessions.get(sessionId)?.child?.phase + }, + next + ) + return 'continue' + } + + private async fail(sessionId: string, failure: StartFailure): Promise<'stop'> { + const session = this.deps.sessions.get(sessionId) + if (session) { + await recordStructuredAgentSessionStartFailure( + { journal: session.journal, fence: this.deps.conversationFence(sessionId) }, + failure + ) + } + return this.stop(sessionId) + } + + /** Inside the serialized step that found nothing to do, so an accept after it wakes anew. */ + private stop(sessionId: string): 'stop' { + this.running.delete(sessionId) + return 'stop' + } +} + +/** A start that died while this message waited on it — a view's, say — is the message's failed + * start: settled with it, under its key, rather than started again into the same failure. */ +function startThatFailedWhileQueued( + session: StructuredAgentSessionHostSession, + oldest: NonNullable> +): StartFailure | null { + const ended = failedProviderChildStart(session) + if ( + !ended || + oldest.acceptedSequence === undefined || + ended.endedAt.epoch !== session.journal.cursor().epoch || + ended.endedAt.sequence < oldest.acceptedSequence + ) { + return null + } + return { startKey: ended.generation, text: endedChildRejection(ended) } +} + +const HOST_STOPPED_BEFORE_DELIVERY = 'Orca stopped the agent before this message was sent.' + +/** Why a queued message the child never took is rejected, in the words the chat row uses. */ +function endedChildRejection(ended: StructuredAgentSessionEndedChild): string { + if (ended.cause === 'host-stop') { + return ended.reason ?? HOST_STOPPED_BEFORE_DELIVERY + } + const reason = ended.reason ?? undefined + return ended.duringStartup + ? providerStartupFailureOutcome(reason) + : providerExitBeforeDeliveryRejection(reason) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts index 19989e10711..4905cc19257 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts @@ -42,14 +42,13 @@ export class StructuredAgentSessionEventRecovery { this.sinkFailures.add(sessionId) void this.context .serialize(sessionId, async () => { - const session = this.context.sessions.get(sessionId) + const child = this.context.sessions.get(sessionId)?.child const stop = this.context.deps.adapter.forceCloseSession ?? this.context.deps.adapter.closeSession - if (!session?.hasProviderChild || !stop) { + if (!child || !stop) { return null } - const fence = session.fence - const acquisitionGeneration = session.acquisitionGeneration + const { fence, generation: acquisitionGeneration } = child const stopped = await stopAgentSessionProviderRoot(() => stop(sessionId)) if (!stopped || !acquisitionGeneration) { return null diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts index cb0cb2fb233..8fc810de2a4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts @@ -30,8 +30,8 @@ function context(): StructuredAgentSessionEvictionContext & { order: string[] } return true }) } as unknown as StructuredAgentSessionEvictionContext['adapter'], - forget: vi.fn(async () => { - order.push('forget') + acknowledgeRelease: vi.fn(() => { + order.push('acknowledgeRelease') }), discardSink: vi.fn(() => order.push('discardSink')), settleWork: vi.fn(async () => { @@ -51,7 +51,7 @@ function runtimeState(): StructuredAgentSessionHostRuntimeState { } describe('structured agent session eviction', () => { - it('stops the child before it lets the sink go, then forgets the session', async () => { + it('stops the child before it lets the sink go, then acknowledges the release', async () => { const ctx = context() await evictStructuredAgentSession(ctx) expect(ctx.order).toEqual([ @@ -62,7 +62,7 @@ describe('structured agent session eviction', () => { 'close', 'discardSink', 'releaseLease', - 'forget' + 'acknowledgeRelease' ]) }) @@ -90,7 +90,7 @@ describe('structured agent session eviction', () => { 'close-sink', 'discard-sink', 'release-lease', - 'forget-session' + 'acknowledge-release' ]) }) @@ -114,7 +114,7 @@ describe('structured agent session eviction', () => { } }) - it('aborts after a failed drain barrier without unbinding or forgetting the session', async () => { + it('aborts after a failed drain barrier without unbinding or acknowledging the release', async () => { const ctx = context() ctx.eventSink.drained = vi.fn(async () => { ctx.order.push('drained') @@ -128,7 +128,7 @@ describe('structured agent session eviction', () => { expect(ctx.eventSink.close).not.toHaveBeenCalled() expect(ctx.discardSink).not.toHaveBeenCalled() expect(ctx.releaseLease).not.toHaveBeenCalled() - expect(ctx.forget).not.toHaveBeenCalled() + expect(ctx.acknowledgeRelease).not.toHaveBeenCalled() expect(ctx.order).toEqual(['closeSession', 'drained']) }) }) @@ -154,9 +154,9 @@ describe('rows the provider emits while closing', () => { return true } } as never, - forget: async () => {}, discardSink: () => state.discardEventSink(sessionId), - releaseLease: async () => {} + releaseLease: async () => {}, + acknowledgeRelease: () => {} }) expect(published).toEqual(['final-flush']) @@ -174,9 +174,13 @@ describe('a child that will not stop', () => { ctx.adapter.closeSession = vi.fn(async () => { throw error }) + const stopped = vi.fn() + ctx.onProviderChildStopped = stopped await evictStructuredAgentSession(ctx) + // The host ends its child on the one reading of the verdict, not a second one of its own. + expect(stopped).toHaveBeenCalledWith({ rootGone: true }) expect(ctx.order).toEqual([ 'drained', 'settleWork', @@ -184,18 +188,18 @@ describe('a child that will not stop', () => { 'close', 'discardSink', 'releaseLease', - 'forget' + 'acknowledgeRelease' ]) }) - it('aborts without forgetting the session, so the next close is a real retry', async () => { + it('aborts without acknowledging the release, so the next stop is a real retry', async () => { const ctx = context() ctx.adapter.closeSession = vi.fn(async () => false) await expect(evictStructuredAgentSession(ctx)).rejects.toMatchObject({ step: 'stop-provider-child' }) - expect(ctx.forget).not.toHaveBeenCalled() + expect(ctx.acknowledgeRelease).not.toHaveBeenCalled() expect(ctx.discardSink).not.toHaveBeenCalled() expect(ctx.order).toEqual([]) }) @@ -210,7 +214,7 @@ describe('a child that will not stop', () => { StructuredAgentSessionEvictionError ) expect(ctx.eventSink.close).not.toHaveBeenCalled() - expect(ctx.forget).not.toHaveBeenCalled() + expect(ctx.acknowledgeRelease).not.toHaveBeenCalled() }) }) @@ -225,9 +229,9 @@ describe('eviction against the real sink cache', () => { sessionId, eventSink: state.eventSinkFor(sessionId), adapter: { closeSession: async () => true } as never, - forget: async () => {}, discardSink: () => state.discardEventSink(sessionId), - releaseLease: async () => {} + releaseLease: async () => {}, + acknowledgeRelease: () => {} }) const published: string[] = [] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts index e5a1d942818..2c03143ff41 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts @@ -1,4 +1,4 @@ -// Releasing one structured session's resources. +// Stopping one structured session's provider child and handing its lease back. // // Teardown is a DATA list, not a method body, for the reason this file exists at all: the host // tracked which sessions were live in a map, and tore them down at three unrelated call sites @@ -22,6 +22,7 @@ import { type StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import type { DeferredStructuredAgentSessionEventSink } from './structured-agent-session-event-sink' +import type { StructuredAgentSessionStopVerdict } from './structured-agent-session-host-types' import { withTimeout } from '../../../shared/promise-timeout-fallback' export type StructuredAgentSessionEvictionContext = { @@ -29,17 +30,17 @@ export type StructuredAgentSessionEvictionContext = { hasProviderChild?: boolean eventSink: DeferredStructuredAgentSessionEventSink adapter: StructuredAgentSessionAdapter - /** Closes the session's journal handle and drops the map entry. Async and - * awaited: `close()` is ordered behind queued writes, and a delete that - * returns while the close is still queued leaves nothing to retry. */ - forget: () => Promise + /** Tells the adapter the released lease is done with, so it drops this child's route and index. + * The conversation stays: stopping the agent never closes its journal. */ + acknowledgeRelease: () => Promise | void /** Drops the cached sink so a later attach mints a fresh one. */ discardSink: () => void /** Fires right before the stop, while the child's turn and background roster are still live. A * throw is logged, never allowed to abort the stop. */ beforeProviderChildStop?: () => void - /** Fires once the adapter has PROVEN the child gone, so host bookkeeping stops claiming one. */ - onProviderChildStopped?: () => void + /** Fires with the stop's verdict once `stopAgentSessionProviderRoot` read the root gone, so host + * bookkeeping stops claiming a child. */ + onProviderChildStopped?: (verdict: StructuredAgentSessionStopVerdict) => void /** Whether this host still owes the child's wind-down. Distinct from `hasProviderChild`, which a * proven exit retires mid-run: the two disagree for exactly the steps a retry has to repeat. */ owesProviderChildWindDown?: boolean @@ -84,13 +85,13 @@ export const STRUCTURED_AGENT_SESSION_EVICTION_STEPS: readonly StructuredAgentSe } // An adapter with no close has nothing to stop; anything else must PROVE the exit. const stop = context.adapter.disposeSession ?? context.adapter.closeSession - if ( - stop && - !(await stopAgentSessionProviderRoot(() => stop.call(context.adapter, context.sessionId))) - ) { + const rootGone = stop + ? await stopAgentSessionProviderRoot(() => stop.call(context.adapter, context.sessionId)) + : true + if (!rootGone) { throw new Error('provider child exit was not proven') } - context.onProviderChildStopped?.() + context.onProviderChildStopped?.({ rootGone }) } }, { @@ -115,11 +116,11 @@ export const STRUCTURED_AGENT_SESSION_EVICTION_STEPS: readonly StructuredAgentSe // these two; eviction has to as well. { name: 'discard-sink', run: (context) => context.discardSink() }, // Why here and not last: the durable lease still names a process this host just stopped, and a - // record left claiming a live owner is one nothing can resume — the next surface to open the - // chat would find a session it may not acquire. Placed BEFORE forget so a release that cannot - // be written aborts while the session is still indexed, which is what makes the retry real. + // record left claiming a live owner is one nothing can resume — the next send would find a + // session it may not acquire. Placed BEFORE the acknowledgement so a release that cannot be + // written aborts while the adapter still routes the session, which is what makes the retry real. { name: 'release-lease', run: (context) => context.releaseLease() }, - { name: 'forget-session', run: (context) => context.forget() } + { name: 'acknowledge-release', run: (context) => context.acknowledgeRelease() } ] export class StructuredAgentSessionEvictionError extends Error { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts index 20531bd6465..1ffc08a0249 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts @@ -21,11 +21,10 @@ import type { StructuredAgentSessionAttachContext } from './structured-agent-ses import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { StructuredAgentSessionStatusFeed } from './structured-agent-session-status-feed' -// Everything before the journal is out of scope here; what matters is that the orchestration's -// own `onAttachFailed` runs, which is the real one. +// Everything before the journal is out of scope here; what matters is what the orchestration does +// when the attach throws after acquisition. vi.mock('./structured-agent-session-attach-flow', () => ({ - performAttach: async (input: { onAttachFailed?: () => Promise }) => { - await input.onAttachFailed?.() + performAttach: async () => { throw new Error('attach failed after acquisition') } })) @@ -136,10 +135,7 @@ async function workingSession(): Promise<{ accountHome: ownerRecord().accountHome, runtimeKind: 'native' }, - fence: 1, - hasProviderChild: true, - providerChildPhase: 'ready', - acquisitionGeneration: null + child: { generation: null, fence: 1, phase: 'ready' } } ] ]) @@ -230,15 +226,19 @@ describe('a session that leaves the host without an explicit close', () => { expect(server.getStatusSnapshot()).toEqual([expect.objectContaining({ prompt: 'other host' })]) }) - it('leaves the agent-status store with it when an attach fails', async () => { + // A failed attach no longer drops the session: the conversation stays open for the failure to be + // written into, so its row stays with it and the later close forgets both together. + it('keeps the session and its status row together when an attach fails', async () => { const { server, feed, sessions } = await workingSession() + const drop = vi.spyOn(server, 'dropStructuredStatus') await expect( attachStructuredAgentSession(attachContext(sessions, feed), 'caller-1', attachParams) ).rejects.toThrow('attach failed after acquisition') - expect(sessions.has(SESSION)).toBe(false) - expect(server.getStatusSnapshot()).toEqual([]) + expect(sessions.has(SESSION)).toBe(true) + expect(drop).not.toHaveBeenCalled() + expect(server.getStatusSnapshot()).toHaveLength(1) }) // The feed's own cache deliberately retains the projection for reload history; only the store diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts index 974a69b1c3c..b1004f7e396 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts @@ -29,6 +29,7 @@ function resumeHarness() { resume, serialize: keyedSerialize(), hasProviderChild: () => child, + lastStartFailed: () => false, hasOwedWork: () => turnActive, evict, graceMs: GRACE_MS @@ -212,6 +213,7 @@ describe('a surface leaving while its structured session resumes', () => { }, serialize: keyedSerialize(), hasProviderChild: () => child, + lastStartFailed: () => false, hasOwedWork: () => false, evict: async () => {}, graceMs: GRACE_MS @@ -245,6 +247,7 @@ describe('a surface leaving while its structured session resumes', () => { resume, serialize: keyedSerialize(), hasProviderChild: () => child, + lastStartFailed: () => false, hasOwedWork: () => false, evict: async () => {}, graceMs: GRACE_MS diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts index 68027f81887..822f36df5bf 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts @@ -184,6 +184,7 @@ describe('holds', () => { resume, serialize: keyedSerialize(), hasProviderChild: () => child, + lastStartFailed: () => false, hasOwedWork: () => false, evict: async () => {}, graceMs: 1 @@ -214,6 +215,7 @@ describe('holds', () => { resume, serialize, hasProviderChild: () => child, + lastStartFailed: () => false, hasOwedWork: () => false, evict: async () => {}, graceMs: 1 @@ -244,6 +246,7 @@ describe('holds', () => { }, serialize, hasProviderChild: () => child, + lastStartFailed: () => false, hasOwedWork: () => false, evict: async () => {}, graceMs: 60_000 @@ -262,6 +265,7 @@ describe('holds', () => { resume: async () => ({ ok: true as const }), serialize: keyedSerialize(), hasProviderChild: () => false, + lastStartFailed: () => false, hasOwedWork: () => false, evict, graceMs: 1 @@ -281,6 +285,7 @@ describe('holds', () => { resume: async () => ({ ok: true as const }), serialize: keyedSerialize(), hasProviderChild: () => false, + lastStartFailed: () => false, hasOwedWork: () => false, evict: async () => {}, graceMs: 1 @@ -296,7 +301,7 @@ describe('holds', () => { describe('the teardown deadline', () => { it('leaves the child loaded instead of forcing it, and keeps the session indexed', async () => { - const forget = vi.fn() + const acknowledgeRelease = vi.fn() const releaseLease = vi.fn(async () => {}) await expect( @@ -309,7 +314,7 @@ describe('the teardown deadline', () => { close: vi.fn() } as never, adapter: { closeSession: () => new Promise(() => {}) } as never, - forget, + acknowledgeRelease, discardSink: vi.fn(), releaseLease }, @@ -317,7 +322,7 @@ describe('the teardown deadline', () => { ) ).rejects.toMatchObject({ step: 'stop-provider-child' }) - expect(forget).not.toHaveBeenCalled() + expect(acknowledgeRelease).not.toHaveBeenCalled() expect(releaseLease).not.toHaveBeenCalled() }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts index 0acbccb6b55..89f28b890ed 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts @@ -6,7 +6,7 @@ // // A surface takes a hold when it binds and drops it when it goes away. The first hold on a session // with no child resumes it — that, and not the shape of a lease on disk, is what makes a provider -// process exist. The last hold leaving starts the idle release clock. Transport close is the BACKSTOP, +// process exist — unless that session's last start died starting: only a send retries one. The last hold leaving starts the idle release clock. Transport close is the BACKSTOP, // not the mechanism: a client that vanishes mid-flight never sends its release, so the caller // registers one against the connection and the holder set absorbs the duplicate. // @@ -33,6 +33,8 @@ export type StructuredAgentSessionHoldsDeps = { serialize: (sessionId: string, task: () => Promise) => Promise /** Whether evicting this session would actually free anything. */ hasProviderChild: (sessionId: string) => boolean + /** The last start died starting; a surface does not retry it, the next send does. */ + lastStartFailed: (sessionId: string) => boolean hasOwedWork: (sessionId: string) => boolean evict: (sessionId: string) => Promise onError?: (input: { sessionId: string; error: unknown }) => void @@ -77,7 +79,11 @@ export class StructuredAgentSessionHolds { } let resumed: StructuredAgentSessionResumeOutcome try { - resumed = await this.deps.serialize(sessionId, () => this.ensureProviderChild(sessionId)) + resumed = await this.deps.serialize(sessionId, () => + this.deps.lastStartFailed(sessionId) + ? Promise.resolve({ ok: true as const }) + : this.ensureProviderChild(sessionId) + ) } catch (error) { this.releaseFailedHold(sessionId, holderId, alreadyHeld, incarnation) throw error diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts new file mode 100644 index 00000000000..dab9ea639c6 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts @@ -0,0 +1,101 @@ +// The host's conversations: how one becomes open, and the delivery loop that hands its accepted +// messages to a provider child. Bundled because they share one invariant — a conversation open +// with a message queued has a delivery loop — and the open is where a loop for leftovers wakes. + +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import type { AgentJournalResetReason } from '../../../shared/agent-session-journal-types' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + openStructuredAgentSessionConversation, + type OpenedStructuredAgentSessionConversation, + type StructuredAgentSessionConversationOpenOptions +} from './structured-agent-session-conversation-open' +import { StructuredAgentSessionDeliveryLoop } from './structured-agent-session-delivery-loop' +import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-hold-resume' +import type { + StructuredAgentSessionHostDeps, + StructuredAgentSessionHostSession +} from './structured-agent-session-host-types' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' +import { structuredAgentSessionStartFailureText } from './structured-agent-session-send-preparation' +import { settleInterruptedCompaction } from './structured-compaction-recovery' +import { recoverStructuredRewind } from './structured-rewind-recovery' + +export type StructuredAgentSessionConversationDelivery = { + loop: StructuredAgentSessionDeliveryLoop + /** For a caller inside the session's serialize. */ + open: ( + sessionId: string, + options?: StructuredAgentSessionConversationOpenOptions + ) => Promise + /** Indexes a conversation some other open produced, as `open` would have. */ + adoptOpened: ( + sessionId: string, + opened: OpenedStructuredAgentSessionConversation + ) => Promise +} + +export function createStructuredAgentSessionConversationDelivery(input: { + deps: StructuredAgentSessionHostDeps + sessions: Map + serialize: (sessionId: string, task: () => Promise) => Promise + trackStart: (start: Promise) => Promise + ensureProviderChild: (sessionId: string) => Promise + reset: (sessionId: string, journal: AgentSessionJournal, reset: AgentJournalResetReason) => void + publishRestored: (sessionId: string) => void +}): StructuredAgentSessionConversationDelivery { + const { deps, sessions } = input + const loop = new StructuredAgentSessionDeliveryLoop({ + sessions, + adapter: deps.adapter, + serialize: input.serialize, + trackStart: input.trackStart, + ensureProviderChild: input.ensureProviderChild, + conversationFence: (sessionId) => + structuredAgentSessionConversationFence(deps.store, sessionId), + startFailureText: (sessionId, cause) => + structuredAgentSessionStartFailureText(deps.store.getRecord(sessionId), cause), + onError: (sessionId, error) => deps.onEventSinkError?.({ sessionId, error }) + }) + const adoptOpened = async ( + sessionId: string, + opened: OpenedStructuredAgentSessionConversation + ): Promise => { + const { session, reset } = opened + sessions.set(sessionId, session) + if (reset) { + input.reset(sessionId, session.journal, reset) + } + input.publishRestored(sessionId) + await settleInterruptedCommands(deps, sessionId, session) + if (session.journal.submissions().some(isQueuedAgentJournalSubmission)) { + loop.wake(sessionId) + } + } + return { + loop, + adoptOpened, + open: (sessionId, options) => + openStructuredAgentSessionConversation({ deps, sessions, adoptOpened }, sessionId, options) + } +} + +/** + * A compaction or rewind found prepared when the conversation opens was started under a child + * this process no longer has — the open runs only when none is indexed — so nothing will finish + * it, and left alone it refuses every send until a view attaches. Settled here instead of by a + * start inside acceptance. A Codex rewind only its provider can prove stays for the attach. + */ +async function settleInterruptedCommands( + deps: StructuredAgentSessionHostDeps, + sessionId: string, + session: StructuredAgentSessionHostSession +): Promise { + const fence = structuredAgentSessionConversationFence(deps.store, sessionId) + try { + await settleInterruptedCompaction(deps.store, sessionId, session.journal, fence) + await recoverStructuredRewind(deps.store, sessionId, session.journal, fence) + } catch (error) { + deps.onEventSinkError?.({ sessionId, error }) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index 0eed19031c7..49d8b24e644 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -7,6 +7,8 @@ import { agentChildWorkLiveness } from '../../../shared/agent-status-child-work-liveness' import { activeStructuredAgentSessionTurnId } from '../../../shared/structured-agent-session-projection' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import { DISPATCH_REJECTED_PROVIDER_CLOSED } from '../../../shared/structured-agent-session-dispatch-rejection' import { evictStructuredAgentSession, STRUCTURED_AGENT_SESSION_EVICTION_STEPS, @@ -16,9 +18,16 @@ import { withStructuredAgentSessionEvictionDeadline } from './structured-agent-s import { StructuredAgentSessionHolds } from './structured-agent-session-holds' import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' import type { + StructuredAgentSessionChildEndCause, StructuredAgentSessionHostDeps, - StructuredAgentSessionHostSession + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChildIdentity } from './structured-agent-session-host-types' +import { + endProviderChild, + failedProviderChildStart, + structuredAgentSessionConversationFence +} from './structured-agent-session-provider-child' import { releaseStoredStructuredAgentSessionOwner } from './structured-agent-session-lease-release' import { resumeHeldStructuredAgentSession } from './structured-agent-session-hold-resume' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' @@ -31,6 +40,8 @@ export type StructuredAgentSessionLifetimeContext = { now: () => number /** Drops the session's row from the agent-status store; see `forgetStructuredAgentSession`. */ forgetStatus: (sessionId: string) => void + /** Re-projects the session's status after its agent stopped and the chat stays. */ + publishStatus?: (sessionId: string) => void /** Quit-only snapshot taken immediately before the provider child is stopped. */ restartWitness?: { beforeStop: (sessionId: string) => void @@ -38,13 +49,39 @@ export type StructuredAgentSessionLifetimeContext = { } } +type ConversationCloseDeps = Pick & { + store: Pick +} + +/** A conversation's handle closes with nothing queued: what is still queued when the chat closes, + * or the app quits, will not be handed over. Best effort: the next open's delivery loop rejects a + * leftover itself. */ +export async function abandonQueuedStructuredAgentSessionMessages( + deps: ConversationCloseDeps, + sessionId: string, + journal: StructuredAgentSessionHostSession['journal'] +): Promise { + await journal + .rejectQueuedSubmissions( + structuredAgentSessionConversationFence(deps.store, sessionId), + DISPATCH_REJECTED_PROVIDER_CLOSED + ) + .catch((error: unknown) => deps.onEventSinkError?.({ sessionId, error })) +} + /** Dropping a session and dropping its status row are ONE operation: the store keeps the row until * told, so a caller that only deletes strands a live-looking row no reader can ever decay. */ export async function forgetStructuredAgentSession( - context: StructuredAgentSessionLifetimeContext, + context: Pick & { + deps: ConversationCloseDeps + }, sessionId: string ): Promise { - await context.sessions.get(sessionId)?.journal.close() + const session = context.sessions.get(sessionId) + if (session) { + await abandonQueuedStructuredAgentSessionMessages(context.deps, sessionId, session.journal) + } + await session?.journal.close() context.sessions.delete(sessionId) context.forgetStatus(sessionId) } @@ -53,58 +90,79 @@ function hasProviderChild( context: StructuredAgentSessionLifetimeContext, sessionId: string ): boolean { - return context.sessions.get(sessionId)?.hasProviderChild === true + return (context.sessions.get(sessionId)?.child ?? null) !== null } /** The wind-down this host owes for the session's child. A live child always owes one, whatever a - * previous childless eviction recorded: a remembered `false` must never outrank the child in front - * of it. */ -function owesProviderChildWindDown(session: StructuredAgentSessionHostSession): boolean { - return session.hasProviderChild || session.owesProviderChildWindDown === true + * previous childless eviction recorded: a remembered tombstone must never outrank the child in + * front of it. */ +function owedProviderChildWindDown( + session: StructuredAgentSessionHostSession +): StructuredAgentSessionProviderChildIdentity | undefined { + return session.child + ? { generation: session.child.generation, fence: session.child.fence } + : session.owesProviderChildWindDown } -/** Runs the eviction steps under a deadline. A step that fails — or runs out of time — aborts the - * rest, which leaves the session indexed and the child loaded so the next close is a real retry. */ -export async function evictHeldStructuredAgentSession( +/** + * The agent goes to rest; the conversation stays. Runs the eviction steps under a deadline. A step + * that fails — or runs out of time — aborts the rest and leaves the wind-down owed, so the next + * stop is a real retry. `ending` is how the child's end is told: a user's Stop, the host stopping it + * for a cause (with its text), or an eviction whose close forgets the conversation next. + */ +export async function stopStructuredAgentSessionAgentUnderSerialize( context: StructuredAgentSessionLifetimeContext, - sessionId: string + sessionId: string, + ending: { + cause: Extract + reason?: string + } = { cause: 'user-stop' } ): Promise { const session = context.sessions.get(sessionId) if (!session) { return } - // The obligation OUTLIVES the child. `hasProviderChild` is retired the instant the adapter - // proves the exit, so a step that aborts after that point would otherwise leave the retry - // reading "no child here" and skipping the settlement and the lease release it still owes. - const owesWindDown = owesProviderChildWindDown(session) - session.owesProviderChildWindDown = owesWindDown + // The obligation OUTLIVES the child. `child` is ended the instant the adapter proves the exit, + // so a step that aborts after that point would otherwise leave the retry reading "no child + // here" and skipping the settlement and the lease release it still owes. + const owed = owedProviderChildWindDown(session) + session.owesProviderChildWindDown = owed + const stopping = session.child let settlementError: unknown const eviction: StructuredAgentSessionEvictionContext = { sessionId, // The retry must not re-stop a child the adapter already proved gone, so this stays honest. - hasProviderChild: session.hasProviderChild, - owesProviderChildWindDown: owesWindDown, + hasProviderChild: stopping !== null, + owesProviderChildWindDown: owed !== undefined, eventSink: context.runtimeState.eventSinkFor(sessionId), adapter: context.deps.adapter, ...(context.restartWitness ? { beforeProviderChildStop: () => context.restartWitness?.beforeStop(sessionId) } : {}), - // Host state must not disagree with the adapter for the seven steps in between. - onProviderChildStopped: () => { - session.hasProviderChild = false + // Host state must not disagree with the adapter for the steps in between. + onProviderChildStopped: (verdict) => { + if (stopping) { + endProviderChild(session, { + generation: stopping.generation, + fence: stopping.fence, + cause: ending.cause, + reason: ending.reason ?? null, + duringStartup: stopping.phase === 'starting', + ...verdict + }) + } context.restartWitness?.stopped(sessionId) }, - forget: async () => { - await forgetStructuredAgentSession(context, sessionId) - context.deps.adapter.acknowledgeSessionRelease?.(sessionId) - }, + acknowledgeRelease: () => context.deps.adapter.acknowledgeSessionRelease?.(sessionId), discardSink: () => context.runtimeState.discardEventSink(sessionId), settleWork: async () => { + const fence = + owed?.fence ?? structuredAgentSessionConversationFence(context.deps.store, sessionId) const settled = await settleStructuredAgentSessionDeadGeneration({ journal: session.journal, sessionId, - fence: session.fence, - settlementId: `expected-close:${sessionId}:${session.fence}:${session.acquisitionGeneration ?? 'unknown'}`, + fence, + settlementId: `expected-close:${sessionId}:${fence}:${owed?.generation ?? 'unknown'}`, pendingSubmissionReason: 'provider_closed_before_acknowledgement', verdict: { state: 'interrupted', completedAt: context.now() }, showUnexpectedExitOutcome: false, @@ -119,15 +177,22 @@ export async function evictHeldStructuredAgentSession( } }, releaseLease: async () => { - await releaseStoredStructuredAgentSessionOwner({ - store: context.deps.store, - sessionId, - hasProviderChild: owesWindDown, - expectedFence: session.fence, - now: context.now() - }) - session.owesProviderChildWindDown = false - context.forgetStatus(sessionId) + if (owed) { + await releaseStoredStructuredAgentSessionOwner({ + store: context.deps.store, + sessionId, + hasProviderChild: true, + expectedFence: owed.fence, + now: context.now() + }) + } + session.owesProviderChildWindDown = undefined + if (ending.cause === 'evict') { + context.forgetStatus(sessionId) + return + } + // The conversation stays: its readers keep their own fence, and only the status moves. + context.publishStatus?.(sessionId) } } await evictStructuredAgentSession( @@ -136,6 +201,19 @@ export async function evictHeldStructuredAgentSession( ) } +/** Ends the conversation's resources, not the conversation: its child stops, and then its handle + * closes and it leaves the map. A stop that fails throws first, leaving it indexed for a retry. */ +export async function evictHeldStructuredAgentSession( + context: StructuredAgentSessionLifetimeContext, + sessionId: string +): Promise { + if (!context.sessions.has(sessionId)) { + return + } + await stopStructuredAgentSessionAgentUnderSerialize(context, sessionId, { cause: 'evict' }) + await forgetStructuredAgentSession(context, sessionId) +} + /** Stops every provider child owned by this host while keeping failed evictions reachable. A * session whose child is already stopped but whose wind-down aborted is still in scope — that is * the retry. */ @@ -146,7 +224,7 @@ export async function evictOwnedStructuredAgentSessions( retainOnFailure: Set ): Promise { const ownedSessionIds = [...context.sessions] - .filter(([, session]) => owesProviderChildWindDown(session)) + .filter(([, session]) => owedProviderChildWindDown(session) !== undefined) .map(([sessionId]) => sessionId) // Retained up front and cleared only once an eviction settles: the quit phase is bounded, and a // timeout leaves these still running. Closing their journals underneath them is the one outcome @@ -176,7 +254,8 @@ export async function evictOwnedStructuredAgentSessions( * resume and a send's ensure-owner step are the same serialized attach with a different asker. */ export function createStructuredAgentSessionHolds( attachContext: () => StructuredAgentSessionAttachContext, - close: (sessionId: string) => Promise + close: (sessionId: string) => Promise, + deliveryActive: (sessionId: string) => boolean ): StructuredAgentSessionHolds { const context = attachContext() return new StructuredAgentSessionHolds({ @@ -197,15 +276,22 @@ export function createStructuredAgentSessionHolds( }, evict: close, hasProviderChild: (sessionId) => hasProviderChild(context, sessionId), - // A send pending while the child is still starting is held for that start; evicting would - // refuse it. Any other pending send may wait on an echo that never comes, so eviction retires it. - // Subagents, commands and monitors outlive the lead's turn inside the child, so the live roster - // the sidebar shows as working is owed too; stopping the child would end them silently. + lastStartFailed: (sessionId) => { + const session = context.sessions.get(sessionId) + return session !== undefined && failedProviderChildStart(session) !== null + }, + // A message accepted and not yet handed over is owed to this child, and so is one pending while + // the child still starts. Any other pending send may wait on an echo that never comes, so + // eviction retires it. Subagents, commands and monitors outlive the lead's turn inside the + // child, so the live roster the sidebar shows as working is owed too; stopping the child would + // end them silently. hasOwedWork: (sessionId) => { const session = context.sessions.get(sessionId) return session ? activeStructuredAgentSessionTurnId(session.journal.snapshot().items) !== null || - (session.providerChildPhase === 'starting' && + deliveryActive(sessionId) || + session.journal.submissions().some(isQueuedAgentJournalSubmission) || + (session.child?.phase === 'starting' && session.journal.pendingSubmissions().length > 0) || agentChildWorkLiveness(context.deps.adapter.backgroundTaskState?.(sessionId)?.tasks) !== null diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts index c1ae42e237c..9decd5a488a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts @@ -3,8 +3,8 @@ // // They share one shape — admit the envelope against the lease, run a plan, publish the journal — so // they share one path here rather than five copies in the host. The host keeps attach, holds and -// teardown. A send is the one mutation that may need those first: it makes sure the session has -// an owner as a step of its own serialized admission, see `structured-agent-session-send-preparation`. +// teardown. A send and a Stop are conversation writes: they open the conversation and are admitted +// without the writer lease; the session's delivery loop starts the provider child a send needs. import type { AgentJournalItemIdentity, @@ -21,15 +21,16 @@ import type { AgentSessionThreadGoalChange, AgentSessionThreadGoalResult } from '../../../shared/agent-session-wire' -import type { StructuredAgentSessionHolds } from './structured-agent-session-holds' +import { DISPATCH_REJECTED_CANCELLED } from '../../../shared/structured-agent-session-dispatch-rejection' import type { AgentSessionPromptRequest } from './structured-agent-session-turns-prompt' import { threadGoalPlan } from './structured-agent-session-thread-goal' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' import { admitAndRunAgentSessionMutation, type AgentSessionMutationRequest } from './structured-agent-session-mutation-admission' import { - prepareStructuredAgentSessionSend, + openConversationForWrite, structuredAgentSessionSendBlock } from './structured-agent-session-send-preparation' import { @@ -52,11 +53,12 @@ export type StructuredAgentSessionMutationContext = { flushStreamedEvents: (sessionId: string) => Promise requireSession: (sessionId: string) => StructuredAgentSessionHostSession serialize: (sessionId: string, task: () => Promise) => Promise - /** A send that finds the owner gone brings it back through here, inside its own serialize. */ - holds: Pick - /** Makes a closed session's journal readable again, inside the caller's serialize, for a send - * the ledger answers without an owner. */ - restoreReadable: (sessionId: string) => Promise + /** The session's conversation, opened when closed; inside the caller's serialize. */ + openConversation: (sessionId: string) => Promise + /** A message was accepted: the session's delivery loop hands it over. */ + wakeDelivery: (sessionId: string) => void + /** Stops the session's provider child, keeping its conversation; inside the caller's serialize. */ + stopAgent: (sessionId: string) => Promise now: () => number } @@ -78,7 +80,7 @@ function mutate( prepareSession, publish: (journal) => context.publish(envelope.sessionId, journal), flushStreamedEvents: context.flushStreamedEvents, - providerChildPhase: () => context.sessions.get(envelope.sessionId)?.providerChildPhase, + providerChildPhase: () => context.sessions.get(envelope.sessionId)?.child?.phase, now: () => context.now() }) ) @@ -101,12 +103,19 @@ export function sendStructuredAgentSessionTurn( params.envelope, { ...plan, - run: (ctx) => { + run: async (ctx) => { const blocked = structuredAgentSessionSendBlock(context.deps.store.getRecord(ctx.sessionId)) - return blocked ? Promise.resolve(blocked) : plan.run(ctx) + if (blocked) { + return blocked + } + const accepted = await plan.run(ctx) + if (accepted.ok) { + context.wakeDelivery(ctx.sessionId) + } + return accepted } }, - (ledger, record) => prepareStructuredAgentSessionSend(context, params.envelope, ledger, record) + () => openConversationForWrite(context.openConversation, params.envelope) ) } @@ -131,7 +140,35 @@ export function cancelStructuredAgentSessionTurn( context.serialize(`compact-cancel:${sessionId}`, task) } : context - return mutate(cancellationContext, caller, params.envelope, cancelPlan(params)) + const plan = cancelPlan(params) + if (params.scope || params.prompt) { + return mutate(cancellationContext, caller, params.envelope, plan) + } + return mutate( + cancellationContext, + caller, + params.envelope, + { + ...plan, + run: async (ctx) => { + // Stop withdraws every queued message first, whatever the start or the child is doing. + const withdrawn = await ctx.journal.rejectQueuedSubmissions( + ctx.fence, + DISPATCH_REJECTED_CANCELLED + ) + const child = context.sessions.get(ctx.sessionId)?.child + if (child?.phase === 'starting') { + // A start that may never land is the one thing here Stop has to end; the chat stays. + await context.stopAgent(ctx.sessionId) + return { ok: true, value: { turnId: params.turnId, cancelled: true } } + } + return child + ? plan.run(ctx) + : { ok: true, value: { turnId: params.turnId, cancelled: withdrawn.length > 0 } } + } + }, + () => openConversationForWrite(context.openConversation, params.envelope) + ) } export function respondToStructuredAgentSessionPrompt( @@ -169,7 +206,12 @@ export function readStructuredAgentSessionOptions( if (!context.deps.adapter.readOptions) { throw new Error('structured_agent_session_options_unsupported') } - const options = await context.deps.adapter.readOptions({ sessionId, fence: session.fence }) + const options = await context.deps.adapter.readOptions({ + sessionId, + fence: + session.child?.fence ?? + structuredAgentSessionConversationFence(context.deps.store, sessionId) + }) return { ...options, rewind: @@ -203,6 +245,7 @@ export async function settleStructuredAgentSessionLateDispatch( if (!session) { return } + const fence = structuredAgentSessionConversationFence(context.deps.store, input.sessionId) // The journal queue drains before close; the host queue would defer this past teardown. await session.journal.resolveDispatch( 'providerIdentity' in input @@ -210,13 +253,13 @@ export async function settleStructuredAgentSessionLateDispatch( clientMessageId: input.clientMessageId, state: 'accepted', providerIdentity: input.providerIdentity, - fence: session.fence + fence } : { clientMessageId: input.clientMessageId, state: 'rejected', reason: input.reason, - fence: session.fence + fence } ) } @@ -234,7 +277,9 @@ export async function settleStructuredAgentSessionLateDispatch( * it never makes a send re-deliverable, because the provider may well have run it. */ export async function releaseStructuredAgentSessionUnansweredDispatches( - context: Pick, + context: Pick & { + deps: { store: Pick } + }, input: { sessionId: string; reason: string } ): Promise { const session = context.sessions.get(input.sessionId) @@ -253,7 +298,7 @@ export async function releaseStructuredAgentSessionUnansweredDispatches( state: 'unknown', // The earlier reason names a sharper fact than this one does. reason: entry.reason ?? input.reason, - fence: session.fence, + fence: structuredAgentSessionConversationFence(context.deps.store, input.sessionId), recovered: true }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts index 6b1693c3dca..2f1b0f3c287 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts @@ -9,6 +9,7 @@ import type { AgentSessionResumeTrigger } from '../../../shared/agent-session-resume-marker' import type { StructuredAgentSessionRestartResume } from './structured-agent-session-restart-resume-host' import { + abandonQueuedStructuredAgentSessionMessages, evictOwnedStructuredAgentSessions, type StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' @@ -93,6 +94,8 @@ export async function tearDownStructuredAgentSessionHost(input: { sessions: Map retainSessionIds?: ReadonlySet acknowledgeSessionRelease?: (sessionId: string) => void + /** Quit closes every conversation, so it settles what they still queue as a close does. */ + abandonQueued?: (sessionId: string, session: StructuredAgentSessionHostSession) => Promise }): Promise { const failures: unknown[] = [] for (const phase of input.phases) { @@ -107,7 +110,12 @@ export async function tearDownStructuredAgentSessionHost(input: { ([sessionId]) => !input.retainSessionIds?.has(sessionId) ) // `allSettled`, so one rejected close cannot skip the others. - const closed = await Promise.allSettled(entries.map(([, session]) => session.journal.close())) + const closed = await Promise.allSettled( + entries.map(async ([sessionId, session]) => { + await input.abandonQueued?.(sessionId, session) + await session.journal.close() + }) + ) closed.forEach((result, index) => { const sessionId = entries[index]?.[0] if (result.status === 'fulfilled') { @@ -160,6 +168,8 @@ export async function flushStructuredAgentSessionHost( sessions: context.sessions, retainSessionIds, acknowledgeSessionRelease: (sessionId) => - context.deps.adapter.acknowledgeSessionRelease?.(sessionId) + context.deps.adapter.acknowledgeSessionRelease?.(sessionId), + abandonQueued: (sessionId, session) => + abandonQueuedStructuredAgentSessionMessages(context.deps, sessionId, session.journal) }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index fd2c911c74e..44611300c7c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -1,4 +1,5 @@ import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' @@ -26,25 +27,60 @@ export type StructuredAgentSessionReveal = { readable: boolean } +/** Which provider child: the adapter acquisition and the lease fence it writes at. */ +export type StructuredAgentSessionProviderChildIdentity = { + readonly generation: string | null + readonly fence: number +} + +/** The provider process behind a conversation. Written only in + * `structured-agent-session-provider-child`. */ +export type StructuredAgentSessionProviderChild = StructuredAgentSessionProviderChildIdentity & { + /** A publish-first acquire is `starting` until the adapter's `started` event; only then are its + * reported options fact. */ + phase: StructuredAgentSessionProviderChildPhase +} + +/** What ending a child established about its provider root. A stop's comes only from + * `stopAgentSessionProviderRoot`; an observed exit's root is gone by definition. */ +export type StructuredAgentSessionStopVerdict = { rootGone: boolean } + +export type StructuredAgentSessionChildEndCause = + | 'user-stop' + | 'host-stop' + | 'exit' + | 'attach-failed' + | 'evict' + +/** How the conversation's last child ended. In memory only: the delivery loop reads it to tell a + * Stop from a failure. */ +export type StructuredAgentSessionEndedChild = StructuredAgentSessionProviderChildIdentity & + StructuredAgentSessionStopVerdict & { + /** `user-stop` is a Stop the user asked for; `host-stop` is the host stopping the child for a + * cause of its own, which fails the start the delivery loop was waiting on. */ + cause: StructuredAgentSessionChildEndCause + /** Descriptive text only — the provider's diagnostic, or the host's cause. Decides nothing. */ + reason: string | null + duringStartup: boolean + /** Where the conversation's journal stood when the child ended, to order the end against a + * message's acceptance. */ + endedAt: AgentJournalCursor + } + +/** The conversation: its journal, params and readers outlive any child that serves it. */ export type StructuredAgentSessionHostSession = { /** Readonly: a new handle enters only through the session map's `set`, which binds its delivery. */ readonly journal: AgentSessionJournal params: AgentSessionAttachParams - fence: number - /** Whether THIS host generation is running the provider process behind the session. A journal - * restored for reading has none — so it may not be evicted to free a child, nor have its lease - * released as an observed exit. */ - hasProviderChild: boolean - /** Whether the child behind `hasProviderChild` has proven its start. A publish-first acquire - * is `starting` until the adapter's `started` event; only then are its reported options fact. */ - providerChildPhase: StructuredAgentSessionProviderChildPhase + /** The child THIS host generation runs for the conversation. A conversation opened for reading + * has none — so it may not be evicted to free a child, nor have its lease released as an + * observed exit. */ + child: StructuredAgentSessionProviderChild | null /** The wind-down this host still owes for a child it started: settling that generation's work - * and handing the lease back. A separate fact from `hasProviderChild`, which goes false the - * moment the adapter proves the exit — an eviction that aborts after that point must still be - * able to finish the wind-down on the next close. */ - owesProviderChildWindDown?: boolean - /** Exact adapter acquisition behind `hasProviderChild`; retained after exit to fence recovery. */ - acquisitionGeneration: string | null + * and handing the lease back. Outlives `child`, which ends the moment the adapter proves the + * exit — an eviction that aborts after that point must still finish it on the next close. */ + owesProviderChildWindDown?: StructuredAgentSessionProviderChildIdentity + lastEndedChild?: StructuredAgentSessionEndedChild } export type StructuredAgentSessionHostDeps = { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts index ad2ac96da7a..3d0f5c9d427 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts @@ -788,9 +788,19 @@ describe('subscribe', () => { body }) - expect(result).toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } }) - expect(dispatch).toHaveBeenCalledTimes(1) - expect(events.some((event) => event.type === 'batch')).toBe(true) + expect(result).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + // The failed transport does not stop the delivery loop either: the handover still lands and + // reaches the live subscriber. + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) + await vi.waitFor(() => + expect( + events.some( + (event) => + event.type === 'batch' && + event.batch.submissions?.some((entry) => entry.dispatchState === 'accepted') + ) + ).toBe(true) + ) }) it('resets a subscriber whose epoch is gone', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 40bbc38238d..6462fbe5ad7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -20,6 +20,7 @@ import { attachStructuredAgentSession } from './structured-agent-session-attach- import { createStructuredAgentSessionHolds, evictHeldStructuredAgentSession, + stopStructuredAgentSessionAgentUnderSerialize, type StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' import type { @@ -50,6 +51,8 @@ import { type StructuredAgentSessionRestartResume } from './structured-agent-session-restart-resume-host' import { structuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-wiring' +import { createStructuredAgentSessionConversationDelivery } from './structured-agent-session-host-delivery' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' export type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' export class StructuredAgentSessionHost { @@ -75,6 +78,9 @@ export class StructuredAgentSessionHost { ) => Promise private readonly restore: ReturnType private readonly holds: StructuredAgentSessionHolds + private readonly conversationDelivery: ReturnType< + typeof createStructuredAgentSessionConversationDelivery + > private readonly eventRecovery: StructuredAgentSessionEventRecovery private readonly backgroundTasks: StructuredAgentSessionBackgroundTaskChannel /** Public because the RPC surface addresses it directly; see the restart-resume collaborator. */ @@ -97,9 +103,26 @@ export class StructuredAgentSessionHost { ...(deps.probeOwners ? { probeMany: deps.probeOwners } : {}), now: () => this.now() }) + this.conversationDelivery = createStructuredAgentSessionConversationDelivery({ + deps, + sessions: this.sessions, + serialize: (sessionId, task) => this.serialize(sessionId, task), + // Quit drains a delivery start before it evicts, so the child it produces is stopped. + trackStart: (start) => this.tasks.trackAttach(start), + ensureProviderChild: (sessionId) => this.holds.ensureProviderChild(sessionId), + reset: (sessionId, journal, reset) => + this.subscribers.reset( + sessionId, + journal, + reset, + structuredAgentSessionConversationFence(deps.store, sessionId) + ), + publishRestored: this.clientDelivery.publishRestored + }) this.holds = createStructuredAgentSessionHolds( () => this.attachContext(), - (sessionId) => this.close(sessionId) + (sessionId) => this.close(sessionId), + (sessionId) => this.conversationDelivery.loop.isRunning(sessionId) ) this.restore = createStructuredAgentSessionHostRestore(deps, { reconcile: this.reconcileLeases, @@ -108,10 +131,7 @@ export class StructuredAgentSessionHost { hasSession: this.hasSession, // Site 10: cannot overwrite a live entry — the restorer returns early on // `hasSession` inside the same serialized step as this `set`. - onReadable: (sessionId, restored) => { - this.sessions.set(sessionId, restored) - this.clientDelivery.publishRestored(sessionId) - } + onReadable: this.conversationDelivery.adoptOpened }) this.eventRecovery = new StructuredAgentSessionEventRecovery({ deps, @@ -119,7 +139,11 @@ export class StructuredAgentSessionHost { sessions: this.sessions, flushLifecycle: (sessionId) => this.runtimeState.lifecycleBarrier(sessionId), publishFence: (sessionId, session) => - this.subscribers.snapshot(sessionId, session.journal, session.fence), + this.subscribers.snapshot( + sessionId, + session.journal, + structuredAgentSessionConversationFence(deps.store, sessionId) + ), publishStatus: this.clientDelivery.publishStatusAndSettlement, hasResumeCapableHolder: (sessionId) => this.holds.hasResumeCapableHolder(sessionId), restartReleaseGrace: (sessionId) => this.holds.renew(sessionId), @@ -143,7 +167,8 @@ export class StructuredAgentSessionHost { isHeld = (sessionId: string): boolean => this.holds.isHeld(sessionId) /** A surface bound to this session and wants it live. The FIRST hold on a session with no - * provider child is what resumes one; a retained hold (a subscription) only keeps it. */ + * provider child is what resumes one, unless its last start failed; a retained hold (a + * subscription) only keeps it. */ hold = ( sessionId: string, holderId: string, @@ -162,7 +187,8 @@ export class StructuredAgentSessionHost { runtimeState: this.runtimeState, sessions: this.sessions, now: () => this.now(), - forgetStatus: this.clientDelivery.forgetStatus + forgetStatus: this.clientDelivery.forgetStatus, + publishStatus: this.clientDelivery.publishStatus } } @@ -174,18 +200,21 @@ export class StructuredAgentSessionHost { tasks: this.tasks, reconcileLeases: (sessionId) => this.reconcileLeases(sessionId), serialize: (sessionId, task) => this.serialize(sessionId, task), - publishStatus: this.clientDelivery.publishStatus + publishStatus: this.clientDelivery.publishStatus, + openConversation: this.conversationDelivery.open } } /** Releases a session's resources without ending the conversation: the record and journal stay * on disk, so the same session can be attached again. */ close(sessionId: string): Promise { - return this.serialize(sessionId, async () => { - await evictHeldStructuredAgentSession(this.lifetimeContext(), sessionId) - this.clientDelivery.closeSession(sessionId) - // The holders now look at a session that is gone; a failed eviction throws above, keeping them. - this.holds.forget(sessionId) - }) + return this.serialize(sessionId, () => this.closeUnderSerialize(sessionId)) + } + + private async closeUnderSerialize(sessionId: string): Promise { + await evictHeldStructuredAgentSession(this.lifetimeContext(), sessionId) + this.clientDelivery.closeSession(sessionId) + // The holders now look at a session that is gone; a failed eviction throws above, keeping them. + this.holds.forget(sessionId) } supportsCreate = (location: AgentSessionExecutionLocation, agent: string): boolean => @@ -227,6 +256,7 @@ export class StructuredAgentSessionHost { // Trigger inlined rather than imported: `AgentSessionResumeTrigger` in shared is the canonical // type, and this file has no line budget left for the import. async flushAllStreamedEvents(options?: { trigger?: 'quit' | 'update' }): Promise { + this.conversationDelivery.loop.dispose() await flushStructuredAgentSessionHost({ ...this.lifetimeContext(), holds: this.holds, @@ -245,8 +275,12 @@ export class StructuredAgentSessionHost { flushStreamedEvents: this.flushStreamedEvents, requireSession: (sessionId) => this.requireSession(sessionId), serialize: (sessionId, task) => this.serialize(sessionId, task), - holds: this.holds, - restoreReadable: (sessionId) => this.restore.restoreReadableUnderSerialize(sessionId), + openConversation: this.conversationDelivery.open, + wakeDelivery: (sessionId) => this.conversationDelivery.loop.wake(sessionId), + stopAgent: (sessionId) => + stopStructuredAgentSessionAgentUnderSerialize(this.lifetimeContext(), sessionId, { + cause: 'user-stop' + }), now: () => this.now() } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts index ef08e1c7e93..942c9ef6152 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts @@ -15,11 +15,7 @@ import { journalDatabaseFile } from '../agent-session-journal/journal-paths' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' import { openAgentSessionJournalWithRecovery } from './agent-session-journal-recovery' -import { - evictStructuredAgentSession, - STRUCTURED_AGENT_SESSION_EVICTION_STEPS, - type StructuredAgentSessionEvictionContext -} from './structured-agent-session-eviction' +import { forgetStructuredAgentSession } from './structured-agent-session-host-lifetime' import { tearDownStructuredAgentSessionHost } from './structured-agent-session-host-teardown' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' @@ -73,30 +69,12 @@ function hostSession(journal: AgentSessionJournal): StructuredAgentSessionHostSe return { journal, params: {} as StructuredAgentSessionHostSession['params'], - fence: 1, - hasProviderChild: false, - providerChildPhase: 'ready', - acquisitionGeneration: null + child: null } } -function evictionContext( - overrides: Partial -): StructuredAgentSessionEvictionContext { - return { - sessionId: SESSION, - hasProviderChild: false, - eventSink: { - drained: async () => ({ ok: true }) as const, - unbind: () => undefined, - close: () => undefined - } as unknown as StructuredAgentSessionEvictionContext['eventSink'], - adapter: {} as StructuredAgentSessionEvictionContext['adapter'], - forget: async () => undefined, - discardSink: () => undefined, - releaseLease: async () => undefined, - ...overrides - } +function forgetContext(sessions: Map) { + return { deps: { store: { getRecord: () => null } }, sessions, forgetStatus: () => undefined } } beforeEach(async () => { @@ -140,46 +118,37 @@ describe('site 6: recovery rehydration', () => { }) }) -describe('sites 9 and 10: the delete and overwrite callbacks', () => { +describe('sites 9 and 10: closing a conversation handle', () => { it('awaits the journal close before dropping the map entry', async () => { const journal = await journals.open({ identity: IDENTITY, journalDir }) const sessions = new Map([[SESSION, hostSession(journal)]]) const order: string[] = [] + const close = journal.close.bind(journal) + journal.close = async () => { + order.push('close-started') + await close() + order.push(sessions.has(SESSION) ? 'closed' : 'dropped-before-close') + } - await evictStructuredAgentSession( - evictionContext({ - forget: async () => { - order.push('close-started') - await sessions.get(SESSION)?.journal.close() - order.push('closed') - sessions.delete(SESSION) - order.push('forgotten') - } - }), - STRUCTURED_AGENT_SESSION_EVICTION_STEPS - ) + await forgetStructuredAgentSession(forgetContext(sessions), SESSION) - expect(order).toEqual(['close-started', 'closed', 'forgotten']) + expect(order).toEqual(['close-started', 'closed']) expect(sessions.size).toBe(0) await expectNothingHoldsTheDirectory(journalDir) }) - it('aborts the eviction with the session still indexed when the close rejects', async () => { + it('keeps the session indexed when the close rejects', async () => { const journal = await journals.open({ identity: IDENTITY, journalDir }) const sessions = new Map([[SESSION, hostSession(journal)]]) + const close = journal.close.bind(journal) + journal.close = () => Promise.reject(new Error('close rejected')) - await expect( - evictStructuredAgentSession( - evictionContext({ - forget: async () => { - await Promise.reject(new Error('close rejected')) - } - }), - STRUCTURED_AGENT_SESSION_EVICTION_STEPS - ) - ).rejects.toMatchObject({ step: 'forget-session' }) + await expect(forgetStructuredAgentSession(forgetContext(sessions), SESSION)).rejects.toThrow( + 'close rejected' + ) // Still indexed, so the next close is a real retry. expect(sessions.has(SESSION)).toBe(true) + journal.close = close }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts index 1874f7ee1c6..6bda0b66189 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts @@ -71,6 +71,18 @@ function journal(): AgentSessionJournal { ).sessions.get(SESSION)!.journal } +/** A send is accepted first; this waits for the delivery loop to hand it to the provider. */ +async function handedOver(clientMessageId: string): Promise { + await vi.waitFor(() => { + expect( + journal() + .submissions() + .find((entry) => entry.clientMessageId === clientMessageId)?.handedOverAt + ).toBeDefined() + expect(dispatch).toHaveBeenCalled() + }) +} + beforeEach(async () => { root = await mkdtemp(join(tmpdir(), 'orca-wire-late-settle-')) resetHostTestOperationIds() @@ -152,10 +164,13 @@ describe('settling a send the provider proves it received after the ack window', finishDispatch({ state: 'unknown', reason: 'ack timeout' }) unsubscribe() } - await expect(pending).resolves.toMatchObject({ - ok: true, - value: { submission: { dispatchState: 'accepted' } } - }) + await expect(pending).resolves.toMatchObject({ ok: true }) + // The late `unknown` from the handover does not reopen the proven acceptance. + await vi.waitFor(() => + expect(submissions()).toMatchObject([ + { clientMessageId: params.envelope.clientOperationId, dispatchState: 'accepted' } + ]) + ) await expect(host.send(CALLER, { ...params, retryUnknown: true })).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } @@ -167,6 +182,8 @@ describe('settling a send the provider proves it received after the ack window', dispatch.mockResolvedValueOnce({ state: 'unknown', reason: 'ack timeout' }) const params = sendParams('received just before shutdown') await host.send(CALLER, params) + await handedOver(params.envelope.clientOperationId) + await vi.waitFor(() => expect(submissions()).toMatchObject([{ dispatchState: 'unknown' }])) let settlement: Promise | undefined closeSession.mockImplementationOnce(async () => { settlement = host.settleLateDispatch({ @@ -188,8 +205,9 @@ describe('settling a send the provider proves it received after the ack window', it('moves a durable unknown to accepted so nothing offers to send it again', async () => { dispatch.mockRejectedValueOnce(new Error('socket closed')) const params = sendParams('sent while a turn was running') - const first = await host.send(CALLER, params) - expect(first).toMatchObject({ ok: true, value: { submission: { dispatchState: 'unknown' } } }) + await host.send(CALLER, params) + await handedOver(params.envelope.clientOperationId) + await vi.waitFor(() => expect(submissions()).toMatchObject([{ dispatchState: 'unknown' }])) await host.settleLateDispatch({ sessionId: SESSION, @@ -209,6 +227,7 @@ describe('settling a send the provider proves it received after the ack window', dispatch.mockResolvedValueOnce({ state: 'admitted' }) const params = sendParams('queued behind the active turn') await host.send(CALLER, params) + await handedOver(params.envelope.clientOperationId) await host.settleLateDispatch({ sessionId: SESSION, @@ -230,6 +249,7 @@ describe('settling a send the provider proves it received after the ack window', dispatch.mockResolvedValueOnce({ state: 'admitted' }) const params = sendParams('settle from provider echo') await host.send(CALLER, params) + await handedOver(params.envelope.clientOperationId) vi.spyOn(journal(), 'resolveDispatch').mockRejectedValueOnce( new Error('direct settlement write failed') ) @@ -259,6 +279,7 @@ describe('settling a send the provider proves it received after the ack window', it('leaves an already accepted send alone', async () => { const params = sendParams('ordinary send') await host.send(CALLER, params) + await vi.waitFor(() => expect(submissions()).toMatchObject([{ dispatchState: 'accepted' }])) await host.settleLateDispatch({ sessionId: SESSION, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-launch-send-after-create.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-launch-send-after-create.test.ts index 300f1423cb3..e7bd9a9ac85 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-launch-send-after-create.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-launch-send-after-create.test.ts @@ -57,12 +57,20 @@ async function launchAndDeliver(): Promise<{ text: 'fix the failing test' }) const sent = await send.mock.results[0]!.value - return { - messageId, - dispatchState: sent.ok - ? sent.value.submission.dispatchState - : `refused:${sent.refusal.code}:${sent.refusal.message}` + if (!sent.ok) { + return { messageId, dispatchState: `refused:${sent.refusal.code}:${sent.refusal.message}` } } + // Accepted first; the delivery loop hands it over, and that outcome is what reached the agent. + let dispatchState = sent.value.submission.dispatchState + await vi.waitFor(() => { + dispatchState = + host + .journalSnapshot(created.value.sessionId) + .submissions.find((entry) => entry.clientMessageId === sent.value.clientMessageId) + ?.dispatchState ?? 'missing' + expect(dispatchState).not.toBe('pending') + }) + return { messageId, dispatchState } } beforeEach(() => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts index 5e0d7c83616..f61dd491054 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts @@ -25,12 +25,18 @@ import { const CALLER = { callerKey: 'client-1' } +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => void): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + let root: string let store: AgentSessionRecordStore let host: StructuredAgentSessionHost let acquire: Mock let probe: Mock<() => Promise> let stopOwnerProcess: Mock<(pid: number, signal: 'SIGTERM' | 'SIGKILL') => void> +let dispatch: Mock function openHost(): void { host = new StructuredAgentSessionHost({ @@ -39,7 +45,7 @@ function openHost(): void { acquire, closeSession: vi.fn(async () => true), releaseAcquisition: vi.fn(async () => true), - dispatch: vi.fn(async () => ({ state: 'admitted' as const })), + dispatch, cancelTurn: vi.fn(async () => ({ cancelled: false })), answerPrompt: vi.fn(async () => undefined), setOption: vi.fn(async () => undefined) @@ -71,6 +77,24 @@ async function persistFromOlderBuild(lease: OlderBuildLease): Promise { openHost() } +/** A send is accepted at once; what became of it is the submission's state once the host's + * delivery settles it — handed over, or rejected with the reason the chat shows. */ +async function delivered(text: string) { + const sent = await send(text) + expect(sent).toMatchObject({ ok: true }) + const clientMessageId = sent.ok ? sent.value.clientMessageId : '' + const submission = () => + host + .journalSnapshot(SESSION) + .submissions.find((candidate) => candidate.clientMessageId === clientMessageId) + await eventually(() => + expect( + submission()?.dispatchState !== 'pending' || submission()?.handedOverAt !== undefined + ).toBe(true) + ) + return submission() +} + async function send(text: string) { const body = hostTestMessage(text) return host.send(CALLER, { @@ -93,6 +117,7 @@ beforeEach(async () => { resetHostTestOperationIds() probe = vi.fn(async () => ({ outcome: 'pid-absent' as const })) stopOwnerProcess = vi.fn() + dispatch = vi.fn(async () => ({ state: 'admitted' as const })) acquire = vi.fn(async ({ fence, spawnToken }) => ({ process: { hostId: 'local', pid: 4242, processStartTimeMs: NOW - 1_000, spawnToken }, link: { @@ -143,7 +168,8 @@ describe('a record an older build left mid terminal handoff', () => { handoffOperationId: null }) expect(store.getRecord(SESSION)?.lease).not.toHaveProperty('settlementRetryRequired') - expect(await send('after the upgrade')).toMatchObject({ ok: true }) + expect(await delivered('after the upgrade')).toMatchObject({ dispatchState: 'pending' }) + expect(dispatch).toHaveBeenCalledOnce() expect(acquire).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ runtimeKind: 'native', @@ -175,7 +201,8 @@ describe('a record an older build left mid terminal handoff', () => { handoffStage: null, handoffOperationId: null }) - expect(await send('after the upgrade')).toMatchObject({ ok: true }) + expect(await delivered('after the upgrade')).toMatchObject({ dispatchState: 'pending' }) + expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'live' }) }) @@ -194,7 +221,8 @@ describe('a record an older build left mid terminal handoff', () => { handoffStage: null, claimStatus: 'released' }) - expect(await send('after the upgrade')).toMatchObject({ ok: true }) + expect(await delivered('after the upgrade')).toMatchObject({ dispatchState: 'pending' }) + expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ runtimeKind: 'native', claimStatus: 'live' @@ -216,18 +244,26 @@ describe('a record an older build left mid terminal handoff', () => { claimStatus: 'conflicted', handoffStage: 'recovering' }) - // Sending and opening the chat both say what frees it: quitting that terminal agent. + // Sending and opening the chat both say what frees it: quitting that terminal agent. A send is + // accepted, then rejected by the start that cannot take the lease, and the chat's row says why. const quitTerminal = 'This chat is still open in a terminal agent (process 4242). Quit that agent to continue the chat here.' - expect(await send('while the terminal still runs')).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_conflict', message: quitTerminal } + expect(await delivered('while the terminal still runs')).toMatchObject({ + dispatchState: 'rejected' }) + expect( + host + .journalSnapshot(SESSION) + .items.flatMap((item) => + item.body.kind === 'status' && item.body.tone === 'error' ? [item.body.text] : [] + ) + ).toEqual([expect.stringContaining(quitTerminal)]) const fence = store.getRecord(SESSION)?.lease.runtimeFence ?? null expect(await host.attach(CALLER, hostTestAttachParams(fence))).toMatchObject({ ok: false, refusal: { code: 'agent_session_conflict', message: quitTerminal } }) + expect(dispatch).not.toHaveBeenCalled() expect(stopOwnerProcess).not.toHaveBeenCalled() expect(acquire).not.toHaveBeenCalled() @@ -236,7 +272,10 @@ describe('a record an older build left mid terminal handoff', () => { await host.hold(SESSION, 'surface-1') expect(stopOwnerProcess).not.toHaveBeenCalled() - expect(await send('after the terminal closed')).toMatchObject({ ok: true }) + expect(await delivered('after the terminal closed')).toMatchObject({ + dispatchState: 'pending' + }) + expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ runtimeKind: 'native', claimStatus: 'live', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts index 62be8900159..24f341d28da 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts @@ -107,7 +107,8 @@ export async function admitAndRunAgentSessionMutation( envelope, hostFingerprint, now: request.now(), - ...(plan.operationIdScope ? { operationIdScope: plan.operationIdScope } : {}) + ...(plan.operationIdScope ? { operationIdScope: plan.operationIdScope } : {}), + ...(plan.conversationWrite ? { conversationWrite: true } : {}) }) if (!admitted) { return refuseAgentSessionMutation(AGENT_SESSION_NOT_ATTACHED) @@ -141,7 +142,8 @@ export async function admitAndRunAgentSessionMutation( envelope, hostFingerprint, ledger: { decision: 'admit', row: admission.row }, - lease: record.lease + lease: record.lease, + ...(plan.conversationWrite ? { conversationWrite: true } : {}) }) if (rerun.decision === 'refused') { return refuseAgentSessionMutation(rerun.refusal) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts index 226881d38bf..fd56944f3bd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts @@ -29,8 +29,9 @@ export type MutationPlan = { method: string fields: Record operationIdScope?: 'global' + /** Admitted without the writer lease: see `admitAgentSessionMutation`. */ + conversationWrite?: true markUnknownBeforeRun?: boolean - beforeRun?: () => void run: (ctx: AgentSessionTurnContext) => Promise> replay: (ctx: AgentSessionTurnContext, outcome: AgentSessionOperationOutcome) => TValue | null rerunWhenReplayMissing?: (ctx: AgentSessionTurnContext) => boolean @@ -50,19 +51,22 @@ export function sendPlan(params: { return { method: 'agentSession.send', operationIdScope: 'global', + conversationWrite: true, markUnknownBeforeRun: true, // A control signal is not payload; it cannot alter durable replay. fields: { body: params.body }, - ...(params.beforeRun ? { beforeRun: params.beforeRun } : {}), recoverUnknownFromDurableState: true, // `retryUnknown` is a compatibility-only client signal. A recorded send // always replays and never reaches the provider twice. - run: (ctx) => - performSend(ctx, { + run: (ctx) => { + // Asked at acceptance: a send accepted after this one is queued behind it. + params.beforeRun?.() + return performSend(ctx, { clientMessageId, payloadFingerprint: params.envelope.payloadFingerprint, body: params.body - }), + }) + }, replay: (ctx, outcome) => { const submission = ctx.journal .submissions() @@ -101,6 +105,8 @@ export function cancelPlan(params: { }): MutationPlan { return { method: 'agentSession.cancel', + // Stop is a conversation write; a prompt or background-task cancel needs the live child. + ...(params.scope || params.prompt ? {} : { conversationWrite: true as const }), fields: { turnId: params.turnId, ...(params.scope ? { scope: params.scope } : {}), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.test.ts index b1eb5704f01..094ddffdcb2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.test.ts @@ -2,7 +2,6 @@ import { join } from 'node:path' import { afterEach, expect, it, vi } from 'vitest' import { AgentSessionPreDispatchError, - AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS, runSettledAgentSessionMutation } from './structured-agent-session-operation-settlement' import { @@ -42,6 +41,9 @@ async function context(): Promise { } } +/** Longer than any bookkeeping bound: a refusal must already be answered by then. */ +const SETTLED_WAIT_MS = 2_000 + afterEach(() => { vi.useRealTimers() vi.restoreAllMocks() @@ -80,7 +82,7 @@ it('returns a pre-dispatch refusal without waiting on redundant uncertainty pers }) try { await refusing.promise - await vi.advanceTimersByTimeAsync(AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS) + await vi.advanceTimersByTimeAsync(SETTLED_WAIT_MS) expect(returned).toBe(true) expect(writes).toHaveBeenCalledOnce() expect(hostTestState().dispatch).not.toHaveBeenCalled() @@ -129,9 +131,10 @@ it.each([1, 2])( } ) -// The pre-dispatch check judges only what the journal already holds; the send path never waits on -// the provider's stream barrier, so a sink that stalls or fails cannot delay or double a send. -it('dispatches without touching the event-stream barrier', async () => { +// A send's plan only accepts: it records the submission and never waits on the provider's stream +// barrier, so a sink that stalls or fails cannot delay or double a send. Handing it over is the +// delivery loop's. +it('accepts without touching the event-stream barrier or the provider', async () => { const ctx = await context() const { store } = hostTestState() vi.spyOn(store, 'recordOperationOutcome').mockResolvedValue() @@ -149,42 +152,33 @@ it('dispatches without touching the event-stream barrier', async () => { }) expect(result).toMatchObject({ ok: true }) expect(beforeRun).toHaveBeenCalledOnce() - expect(hostTestState().dispatch).toHaveBeenCalledOnce() - expect(ctx.journal.submissions()[0]?.dispatchState).toBe('accepted') + expect(hostTestState().dispatch).not.toHaveBeenCalled() + expect(ctx.journal.submissions()[0]).toMatchObject({ + dispatchState: 'pending', + handoverRecorded: true + }) expect(barrier).not.toHaveBeenCalled() }) -it('refuses a superseded send without waiting on a stalled refusal write, and never dispatches late', async () => { +it('refuses a superseded send at acceptance, recording and dispatching nothing', async () => { const ctx = await context() const { store } = hostTestState() vi.spyOn(store, 'recordOperationOutcome').mockResolvedValue() - const pending = Promise.withResolvers() - const refusing = Promise.withResolvers() - vi.spyOn(ctx.journal, 'resolveDispatch').mockImplementationOnce(() => { - refusing.resolve() - return pending.promise.then(() => ctx.journal.cursor()) - }) - vi.spyOn(console, 'warn').mockImplementation(() => {}) const beforeRun = vi.fn(() => { throw new AgentSessionPreDispatchError('agent_session_restart_work_superseded') }) const body = hostTestMessage('Continue the interrupted work') const operation = envelope('agentSession.send', { body }) vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) - const result = runSettledAgentSessionMutation({ + const result = await runSettledAgentSessionMutation({ store, operationCallerKey: 'test', envelope: operation, context: ctx, plan: sendPlan({ envelope: operation, body, beforeRun }) }).catch((error: unknown) => error) - await refusing.promise - await vi.advanceTimersByTimeAsync(AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS) - expect(await result).toBeInstanceOf(AgentSessionPreDispatchError) - expect(hostTestState().dispatch).not.toHaveBeenCalled() - expect(ctx.journal.submissions()[0]?.dispatchState).toBe('pending') - pending.resolve() - await vi.advanceTimersByTimeAsync(0) + expect(result).toBeInstanceOf(AgentSessionPreDispatchError) + expect(ctx.journal.submissions()).toEqual([]) expect(hostTestState().dispatch).not.toHaveBeenCalled() expect(vi.getTimerCount()).toBe(0) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.ts index baf871aef65..5d5009aa918 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.ts @@ -11,8 +11,6 @@ export class AgentSessionPreDispatchError extends Error { } } -export const AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS = 2_000 - export async function runSettledAgentSessionMutation(input: { store: AgentSessionRecordStore operationCallerKey: string @@ -33,10 +31,7 @@ export async function runSettledAgentSessionMutation(input: { if (input.plan.markUnknownBeforeRun) { await settle({ status: 'unknown' }) } - outcome = await input.plan.run({ - ...input.context, - ...(input.plan.beforeRun ? { beforeDispatch: input.plan.beforeRun } : {}) - }) + outcome = await input.plan.run(input.context) await settle( outcome.ok ? (input.plan.settledOutcome?.(outcome.value) ?? { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts index 7957ed326a8..853fa5784a0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts @@ -152,6 +152,11 @@ describe('structured session options and close', () => { envelope: envelope('agentSession.send', { body }), body }) + // Accepted, then handed over by the delivery loop; the status is read once it answered. + await vi.waitFor(() => expect(dispatchedModels).toEqual([DEFAULT_MODEL])) + await vi.waitFor(() => + expect(host.journalSnapshot(SESSION).submissions[0]?.dispatchState).toBe('accepted') + ) const events: AgentSessionStatusEvent[] = [] host.subscribeStatus({ id: 'session-list', emit: (event) => events.push(event) }) expect(events).toEqual([ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts index d2d5d746469..459cd60bcc3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts @@ -2,9 +2,11 @@ // child that still owes the user work. // // A Claude chat is published before its CLI answers initialize, and a message sent in that window -// is held until it does; evicting then refuses a message the user already sent. And a lead whose -// turn has settled can leave subagents, commands and monitors running inside the child; evicting -// then ends them silently. +// is accepted and stays queued until it does; the delivery loop hands it over once startup lands. +// Switching away from the chat starts the release clock; the clock must treat that queued message +// as work still owed, exactly as it treats a running turn, or it evicts the session and rejects a +// message the user already sent. And a lead whose turn has settled can leave subagents, commands +// and monitors running inside the child; evicting then ends them silently. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -65,7 +67,7 @@ beforeEach(async () => { lifecycle.push(host.handleAdapterEvent(mapped)) } }, - // As the runtime wires it: a held prompt's outcome reaches the journal out of band. + // As the runtime wires it: an admitted prompt's outcome reaches the journal out of band. onDispatchSettledLate: (settlement) => void host.settleLateDispatch(settlement), // Initialize answers only when the test says so. openConnection: async (launch, handlers) => { @@ -139,13 +141,21 @@ function dispatchState(clientMessageId: string): string | undefined { .submissions.find((entry) => entry.clientMessageId === clientMessageId)?.dispatchState } +/** The delivery loop hands a message over on its own serialized steps after startup lands; this + * yields to them without advancing the (possibly faked) release clock. */ +async function untilSent(connection: { sent: unknown[] }): Promise { + for (let turn = 0; turn < 2000 && connection.sent.length === 0; turn += 1) { + await new Promise((resolve) => setImmediate(resolve)) + } +} + /** Long enough for several grace windows to elapse, so "not evicted" means the clock declined. */ function waitOutSeveralGraceWindows(): Promise { return new Promise((resolve) => setTimeout(resolve, GRACE_MS * 20)) } describe('a chat left while its Claude CLI is still starting', () => { - it('keeps the session for a message it is holding, and delivers it once startup lands', async () => { + it('keeps the session for a message still queued, and delivers it once startup lands', async () => { await attachStarting() const held = await send('sent while starting') @@ -157,9 +167,12 @@ describe('a chat left while its Claude CLI is still starting', () => { expect(dispatchState(held)).toBe('pending') landInit() - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) - expect(claude.connections[0].sent).toEqual([expect.objectContaining({ type: 'user' })]) + await vi.waitFor( + () => expect(claude.connections[0].sent).toEqual([expect.objectContaining({ type: 'user' })]), + { timeout: 3000 } + ) await vi.waitFor(() => expect(dispatchState(held)).toBe('accepted')) }) @@ -178,8 +191,9 @@ describe('a chat left while its Claude CLI is still starting', () => { // Startup lands just before the clock's next tick. landInit() - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) + await untilSent(connection) expect(connection.sent).toEqual([expect.objectContaining({ type: 'user' })]) await vi.advanceTimersByTimeAsync(GRACE_MS - 1) @@ -195,8 +209,9 @@ describe('a chat left while its Claude CLI is still starting', () => { it('is released after the grace once its turn has finished', async () => { await attachStarting() landInit() - await adapter.drainStartup(SESSION) - await send('answered', 'accepted') + await adapter.awaitStarted(SESSION) + const answered = await send('answered') + await vi.waitFor(() => expect(dispatchState(answered)).toBe('accepted')) claude.connections[0].handlers.onMessage?.({ type: 'result', subtype: 'success', @@ -232,8 +247,9 @@ describe('a chat left while its settled lead still has background work running', async function settleTurnLeavingTask(taskType: string): Promise { await attachStarting() landInit() - await adapter.drainStartup(SESSION) - await send('fan out', 'accepted') + await adapter.awaitStarted(SESSION) + const fanOut = await send('fan out') + await vi.waitFor(() => expect(dispatchState(fanOut)).toBe('accepted')) frame({ type: 'system', subtype: 'task_started', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts index e0fa13b3362..d1ab41ccaef 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts @@ -12,6 +12,7 @@ import { attachFingerprintFields, type AgentSessionAttachParams } from './structured-agent-session-attach' +import { openTestAttachConversation } from './structured-agent-session-attach-test-conversation' import { performAttach } from './structured-agent-session-attach-flow' const NOW = 1_800_000_000_000 @@ -86,6 +87,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -133,6 +135,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -172,6 +175,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-drift', claimKeyId: 'key-1', @@ -224,6 +228,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -290,6 +295,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts new file mode 100644 index 00000000000..10d60791781 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts @@ -0,0 +1,691 @@ +// The provider child is its own record on the conversation: stopping it, losing it or failing to +// start it ends the child, never the conversation. Against the real host, store and journal, with a +// live subscriber opened before each action. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import type { + AgentSessionStatusSummary, + AgentSessionSubscribeEvent +} from '../../../shared/agent-session-wire' +import { + DISPATCH_REJECTED_CANCELLED, + DISPATCH_REJECTED_PROVIDER_CLOSED +} from '../../../shared/structured-agent-session-dispatch-rejection' +import { + providerStartupFailureOutcome, + unexpectedProviderExitOutcome +} from './structured-agent-session-dead-generation-settlement' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' +import { journalDirectoryFor } from '../agent-session-journal/journal-paths' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { stopStructuredAgentSessionAgentUnderSerialize } from './structured-agent-session-host-lifetime' +import { + HOST_TEST_LOCATION, + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let acquire: Mock +let dispatch: Mock +let adapterExtras: Partial + +function eventually(assertion: () => void | Promise): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + +function generation(): string { + return `generation-${acquire.mock.calls.length}` +} + +const spawnChild: StructuredAgentSessionAdapter['acquire'] = async ({ fence, spawnToken }) => ({ + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + acquisitionGeneration: generation(), + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex' as const, threadId: THREAD }, + origin: store.getRecord(SESSION)?.providerHandleChain.length + ? ('resumed' as const) + : ('created' as const), + mintedAtFence: fence, + observedAt: NOW + } +}) + +/** A Claude-shaped child: published at spawn, so it is `starting` until `started`. */ +const spawnStartingChild: StructuredAgentSessionAdapter['acquire'] = async (input) => ({ + ...(await spawnChild(input)), + providerChildPhase: 'starting' as const +}) + +function startHost(): void { + host = new StructuredAgentSessionHost({ + store, + adapter: { + acquire, + dispatch, + closeSession: vi.fn(async () => true), + releaseAcquisition: vi.fn(async () => true), + cancelTurn: vi.fn(async () => ({ cancelled: false })), + answerPrompt: vi.fn(async () => undefined), + setOption: vi.fn(async () => undefined), + ...adapterExtras + }, + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => `spawn-${acquire.mock.calls.length}`, + releaseGraceMs: 60_000, + now: () => NOW + }) +} + +async function restartHost(): Promise { + await host.flushAllStreamedEvents() + startHost() +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-child-record-')) + resetHostTestOperationIds() + adapterExtras = {} + acquire = vi.fn(spawnChild) + dispatch = vi.fn(async (input) => ({ + state: 'accepted' as const, + providerIdentity: { + provider: 'codex' as const, + threadId: THREAD, + turnId: `turn-${input.clientMessageId}`, + ordinal: dispatch.mock.calls.length + } + })) + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + startHost() + expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) + await host.close(SESSION) +}) + +afterEach(async () => { + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +function sendParams(text: string) { + const body = hostTestMessage(text) + return { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: 1, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + } +} + +async function accept(text: string): Promise { + const params = sendParams(text) + const sent = await host.send(CALLER, params) + expect(sent).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + return params.envelope.clientOperationId +} + +function stop() { + const turnId = 'turn-none' + return host.cancel(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.cancel', + sessionId: SESSION, + fields: { turnId } + }) + }, + turnId + }) +} + +function submission(id: string): AgentJournalSubmission | undefined { + return host.journalSnapshot(SESSION).submissions.find((entry) => entry.clientMessageId === id) +} + +function statusRows(): { itemId: string; text: string; tone?: string }[] { + return host.journalSnapshot(SESSION).items.flatMap((item) => + item.body.kind === 'status' + ? [ + { + itemId: item.itemId, + text: item.body.text, + ...(item.body.tone ? { tone: item.body.tone } : {}) + } + ] + : [] + ) +} + +/** The child the conversation has now, as its lifecycle events name it. */ +function currentChild() { + const child = conversation()?.child + if (!child?.generation) { + throw new Error('no child indexed') + } + return { sessionId: SESSION, fence: child.fence, acquisitionGeneration: child.generation } +} + +function exit(child: ReturnType, reason: string, startupUnproven?: true) { + return host.handleAdapterEvent({ + type: 'ended', + ...child, + reason, + cause: 'unexpected-exit', + ...(startupUnproven ? { startupUnproven } : {}) + }) +} + +function rejectedIn(events: AgentSessionSubscribeEvent[], id: string): boolean { + return events.some( + (event) => + event.type === 'batch' && + event.batch.submissions.some( + (entry) => entry.clientMessageId === id && entry.dispatchState === 'rejected' + ) + ) +} + +function conversation() { + return host['sessions'].get(SESSION) +} + +function subscribe(): AgentSessionSubscribeEvent[] { + const events: AgentSessionSubscribeEvent[] = [] + host.subscribe({ + id: 'sub-1', + sessionId: SESSION, + emit: (event) => events.push(structuredClone(event)) + }) + return events +} + +/** The chat's status row as a session list sees it, frame by frame. */ +function watchStatus(): AgentSessionStatusSummary[] { + const frames: AgentSessionStatusSummary[] = [] + host.subscribeStatus({ + id: 'list-1', + emit: (event) => { + if (event.type === 'status' && event.session.sessionId === SESSION) { + frames.push(event.session) + } + } + }) + return frames +} + +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((next) => { + resolve = next + }) + return { promise, resolve } +} + +describe('Stop on a child still proving its start', () => { + it('ends the child and keeps the conversation, its holders and its readers (R1)', async () => { + const ended = deferred() + adapterExtras = { + awaitStarted: vi.fn(() => ended.promise), + closeSession: vi.fn(async () => { + ended.resolve() + return true + }) + } + await restartHost() + acquire.mockImplementationOnce(spawnStartingChild) + await host.hold(SESSION, 'surface-1', { resume: false }) + const first = await accept('hello') + const journal = conversation()?.journal + const events = subscribe() + const frames = watchStatus() + await eventually(() => expect(conversation()?.child?.phase).toBe('starting')) + + expect(await stop()).toMatchObject({ ok: true, value: { cancelled: true } }) + + // The same conversation: no reopen, the holder kept, and the chat told it is idle again. + expect(conversation()?.journal).toBe(journal) + expect(conversation()?.child).toBeNull() + expect(conversation()?.lastEndedChild).toMatchObject({ cause: 'user-stop', rootGone: true }) + expect(host.isHeld(SESSION)).toBe(true) + expect(frames.at(-1)).not.toHaveProperty('hostExecutionPhase') + expect(frames.at(-1)).not.toHaveProperty('hostExecutionOwned') + expect(submission(first)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_CANCELLED + }) + // A Stop is not a failure: no row, and the loop is gone. + expect(statusRows()).toEqual([]) + await eventually(() => expect(host['conversationDelivery'].loop.isRunning(SESSION)).toBe(false)) + + const next = await accept('after stop') + await eventually(() => expect(submission(next)?.dispatchState).toBe('accepted')) + expect(conversation()?.journal).toBe(journal) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([next]) + // The reader opened before the Stop saw the next message delivered on the same stream. + expect( + events.some( + (event) => + event.type === 'batch' && + event.batch.submissions.some( + (entry) => entry.clientMessageId === next && entry.dispatchState === 'accepted' + ) + ) + ).toBe(true) + }) +}) + +describe('settling an earlier child before the next one takes its message', () => { + it("settles the earlier child's turn from its death evidence and leaves the queued message to the new child (R1)", async () => { + // The earlier child exited mid-turn; the released lease keeps only its death evidence. + await store.transitionHandoff(SESSION, (record) => ({ + ...record, + lease: { + ...record.lease, + deathEvidence: { kind: 'exit-observed', detail: 'provider exited', observedAt: NOW - 1_000 } + } + })) + const releasedFence = store.getRecord(SESSION)!.lease.runtimeFence + const journal = await openAgentSessionJournal({ + identity: { + sessionId: SESSION, + workspaceId: HOST_TEST_LOCATION.workspaceId, + hostId: HOST_TEST_LOCATION.executionHostId, + agent: 'codex', + providerHandle: { kind: 'codex', threadId: THREAD } + }, + journalDir: journalDirectoryFor(root, { + workspaceId: HOST_TEST_LOCATION.workspaceId, + sessionId: SESSION + }) + }) + await journal.appendItem( + { provider: 'codex', threadId: THREAD, turnId: 'earlier-turn', ordinal: 0 }, + { kind: 'turn', turnId: 'earlier-turn', state: 'running', startedAt: NOW - 5_000 }, + { fence: releasedFence } + ) + await journal.close() + const id = await accept('for the next child') + + await eventually(() => expect(submission(id)?.dispatchState).toBe('accepted')) + // The exit was observed, so its receipt ends the turn, and the chat says why it stopped. + const items = conversation()!.journal.snapshot().items + expect(items.map((item) => readAgentJournalTurn(item.body)).filter(Boolean)).toContainEqual( + expect.objectContaining({ + turnId: 'earlier-turn', + state: 'interrupted', + completedAt: NOW - 1_000 + }) + ) + expect( + items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) + ).toContain(unexpectedProviderExitOutcome('provider exited')) + // Handed over at the new child's fence, which the attach reserved after settling. + const newFence = store.getRecord(SESSION)!.lease.runtimeFence + expect(newFence).toBeGreaterThan(releasedFence) + expect(submission(id)?.fence).toBe(newFence) + expect(conversation()?.child).toMatchObject({ generation: generation(), fence: newFence }) + }) +}) + +describe('a published child that dies while it proves its start', () => { + const EXIT = 'claude stream-json exited (code 1)' + const TEXT = providerStartupFailureOutcome(EXIT) + + it.each([['the loop sees the start fail first'], ['the exit is processed first']])( + 'leaves one error row keyed by the start, and every queued message rejected with it: %s (R2)', + async (order) => { + const settled = deferred() + adapterExtras = { awaitStarted: vi.fn(() => settled.promise) } + await restartHost() + acquire.mockImplementation(spawnStartingChild) + const first = await accept('first') + const events = subscribe() + const second = await accept('second') + await eventually(() => expect(adapterExtras.awaitStarted).toHaveBeenCalled()) + const child = currentChild() + + if (order === 'the exit is processed first') { + await exit(child, EXIT, true) + settled.resolve(TEXT) + } else { + settled.resolve(TEXT) + await eventually(() => expect(submission(second)?.dispatchState).toBe('rejected')) + await exit(child, EXIT, true) + } + + await eventually(() => expect(submission(second)?.dispatchState).toBe('rejected')) + expect(statusRows()).toEqual([ + { + itemId: `orca:${encodeURIComponent(`start-failure:${child.acquisitionGeneration}`)}`, + text: TEXT, + tone: 'error' + } + ]) + expect(submission(first)).toMatchObject({ dispatchState: 'rejected', reason: TEXT }) + expect(submission(second)).toMatchObject({ dispatchState: 'rejected', reason: TEXT }) + expect(rejectedIn(events, second)).toBe(true) + expect(dispatch).not.toHaveBeenCalled() + expect(acquire).toHaveBeenCalledTimes(2) + } + ) +}) + +describe("a view's start that dies while a sent message waits on it", () => { + const EXIT = 'claude stream-json exited (code 1)' + const TEXT = providerStartupFailureOutcome(EXIT) + + it("is the message's own failed start: one error row, the message rejected, no second start (R2)", async () => { + adapterExtras = { awaitStarted: vi.fn(async () => TEXT) } + await restartHost() + acquire.mockImplementation(spawnStartingChild) + // Opening the tab: the view's hold starts a child that has not proven its start. + await host.hold(SESSION, 'surface-1') + const viewChild = currentChild() + const events = subscribe() + const params = sendParams('hello') + + // Accepted first; the view's child's exit is settled before the loop's first step. + const sent = host.send(CALLER, params) + const exited = exit(viewChild, EXIT, true) + expect(await sent).toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + await exited + const id = params.envelope.clientOperationId + + await eventually(() => expect(submission(id)?.dispatchState).toBe('rejected')) + await settleLoop() + expect(submission(id)?.reason).toBe(TEXT) + expect(statusRows()).toEqual([ + { + itemId: `orca:${encodeURIComponent(`start-failure:${viewChild.acquisitionGeneration}`)}`, + text: TEXT, + tone: 'error' + } + ]) + expect(rejectedIn(events, id)).toBe(true) + // The setup's child and the view's: nothing started again into the same failure. + expect(acquire).toHaveBeenCalledTimes(2) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('leaves a message sent after that start failed to a fresh start (R2)', async () => { + await restartHost() + acquire.mockImplementationOnce(spawnStartingChild) + await host.hold(SESSION, 'surface-1') + await exit(currentChild(), EXIT, true) + expect(statusRows()).toHaveLength(1) + + const id = await accept('after the failure') + + await eventually(() => expect(submission(id)?.dispatchState).toBe('accepted')) + expect(acquire).toHaveBeenCalledTimes(3) + }) + + it('starts again for a message whose proven child crashed: only a failed start settles it (R2)', async () => { + await restartHost() + await host.hold(SESSION, 'surface-1') + const params = sendParams('hello') + + const sent = host.send(CALLER, params) + const exited = exit(currentChild(), 'codex app-server crashed') + await sent + await exited + const id = params.envelope.clientOperationId + + await eventually(() => expect(submission(id)?.dispatchState).not.toBe('pending')) + expect(submission(id)?.dispatchState).toBe('accepted') + expect(acquire).toHaveBeenCalledTimes(3) + }) + + it('waits on a child started since the failed one, not on the failure (R2)', async () => { + adapterExtras = { awaitStarted: vi.fn(async () => undefined) } + await restartHost() + acquire.mockImplementation(spawnStartingChild) + await host.hold(SESSION, 'surface-1') + const params = sendParams('hello') + + // A client's attach lands after the first child's exit, before the loop's first step: a view + // no longer starts a child whose last start failed, but an attach still does. + const sent = host.send(CALLER, params) + const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + const exited = exit(currentChild(), EXIT, true) + const attached = host.attach(CALLER, hostTestAttachParams(exitedFence + 1)) + await sent + await exited + await expect(attached).resolves.toMatchObject({ ok: true }) + const id = params.envelope.clientOperationId + + await eventually(() => expect(submission(id)?.dispatchState).not.toBe('pending')) + expect(submission(id)?.dispatchState).toBe('accepted') + expect(acquire).toHaveBeenCalledTimes(3) + }) +}) + +describe('a child that ends before its message is handed over', () => { + it('starts one child for the message, then rejects it and stops (R2)', async () => { + // The child the loop starts exits between its start step and its handover step. + adapterExtras = { + awaitStarted: vi.fn(async () => { + await exit(currentChild(), 'codex app-server crashed') + }) + } + await restartHost() + const id = await accept('hello') + const events = subscribe() + + await eventually(() => expect(submission(id)?.dispatchState).toBe('rejected')) + expect(submission(id)?.reason).toContain('codex app-server crashed') + expect(statusRows()).toEqual([ + { itemId: expect.any(String), text: submission(id)?.reason, tone: 'error' } + ]) + expect(rejectedIn(events, id)).toBe(true) + await eventually(() => expect(host['conversationDelivery'].loop.isRunning(SESSION)).toBe(false)) + expect(acquire).toHaveBeenCalledTimes(2) + expect(dispatch).not.toHaveBeenCalled() + }) +}) + +describe('another child indexed while the loop waits on the one it started', () => { + it('hands nothing over until the child now there has proven its start (R2)', async () => { + const starts = new Map>>() + const startOf = (generation: string) => { + const start = starts.get(generation) ?? deferred() + starts.set(generation, start) + return start + } + adapterExtras = { + awaitStarted: vi.fn(() => startOf(currentChild().acquisitionGeneration).promise), + // The stop ends the child without settling the start the loop is waiting on. + closeSession: vi.fn(async () => true) + } + await restartHost() + acquire.mockImplementation(spawnStartingChild) + const first = await accept('first') + await eventually(() => expect(adapterExtras.awaitStarted).toHaveBeenCalledTimes(1)) + const stopped = currentChild() + expect(await stop()).toMatchObject({ ok: true }) + const second = await accept('second') + // A view's hold starts its own child before the loop's handover step runs. + await host.hold(SESSION, 'surface-1') + const replacement = currentChild() + expect(replacement.acquisitionGeneration).not.toBe(stopped.acquisitionGeneration) + + startOf(stopped.acquisitionGeneration).resolve() + await eventually(() => expect(adapterExtras.awaitStarted).toHaveBeenCalledTimes(2)) + expect(dispatch).not.toHaveBeenCalled() + + await host.handleAdapterEvent({ + type: 'started', + ...replacement, + reportedOptions: { model: 'sonnet' }, + restoreSkippedOptions: [] + }) + startOf(replacement.acquisitionGeneration).resolve() + + await eventually(() => expect(submission(second)?.dispatchState).toBe('accepted')) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([second]) + expect(submission(first)).toMatchObject({ reason: DISPATCH_REJECTED_CANCELLED }) + }) +}) + +describe('a quit with a message still queued', () => { + /** Read by the next launch, through the same open any reader takes. */ + async function afterRelaunch(id: string): Promise { + startHost() + await host.revealSession(SESSION) + return submission(id) + } + + it('settles a message no child ever had the way a chat close does (R2)', async () => { + // Quit has begun — its first step stops the delivery loops — when this message is accepted. + host['conversationDelivery'].loop.dispose() + const id = await accept('hello') + expect(conversation()?.child).toBeNull() + await host.flushAllStreamedEvents() + + expect(await afterRelaunch(id)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_PROVIDER_CLOSED + }) + }) + + it('waits for the start already in flight and stops the child it produced (R2)', async () => { + const starting = deferred() + const closeSession = vi.fn(async () => true) + adapterExtras = { closeSession } + await restartHost() + // The loop's start step is under way, but has not reached its attach yet. + const resolveRecovery = host['runtimeState'].resolveRecovery.bind(host['runtimeState']) + const recovering = vi.spyOn(host['runtimeState'], 'resolveRecovery') + recovering.mockImplementationOnce(async (sessionId) => { + await starting.promise + return resolveRecovery(sessionId) + }) + const id = await accept('hello') + await eventually(() => expect(recovering).toHaveBeenCalled()) + + const quit = host.flushAllStreamedEvents() + starting.resolve() + await quit + + expect(closeSession).toHaveBeenCalledWith(SESSION) + expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'released' }) + expect(dispatch).not.toHaveBeenCalled() + expect(await afterRelaunch(id)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_PROVIDER_CLOSED + }) + }) +}) + +describe('a send whose start failed, sent again with the same operation id', () => { + it('replays the recorded rejection and starts no second agent (R2)', async () => { + acquire.mockRejectedValueOnce(new Error('spawn claude ENOENT')) + const fenceBefore = store.getRecord(SESSION)!.lease.runtimeFence + const params = sendParams('hello') + // A failed start is a rejected message, never a refused send. + expect(await host.send(CALLER, params)).toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + const id = params.envelope.clientOperationId + await eventually(() => expect(submission(id)?.dispatchState).toBe('rejected')) + // The start moved the fence while the message was out. + expect(store.getRecord(SESSION)!.lease.runtimeFence).toBeGreaterThan(fenceBefore) + const starts = acquire.mock.calls.length + + const replay = await host.send(CALLER, params) + + expect(replay).toMatchObject({ + ok: true, + replayed: true, + value: { submission: { clientMessageId: id, dispatchState: 'rejected' } } + }) + await settleLoop() + expect(acquire).toHaveBeenCalledTimes(starts) + expect(dispatch).not.toHaveBeenCalled() + }) +}) + +async function settleLoop(): Promise { + await eventually(() => expect(host['conversationDelivery'].loop.isRunning(SESSION)).toBe(false)) +} + +describe('how a stopped child ends the start its loop was waiting on', () => { + /** A child the loop waits on, whose start the stop below does not settle, so a message sent + * after the stop is queued when the loop next looks. */ + async function stoppedWhileStarting(stop: () => Promise) { + const start = deferred() + adapterExtras = { + awaitStarted: vi.fn(() => start.promise), + closeSession: vi.fn(async () => true) + } + await restartHost() + acquire.mockImplementationOnce(spawnStartingChild) + await accept('first') + await eventually(() => expect(adapterExtras.awaitStarted).toHaveBeenCalledTimes(1)) + await stop() + const second = await accept('second') + adapterExtras.awaitStarted = undefined + start.resolve() + return second + } + + it("goes on after a user's Stop and delivers what was sent since (R2)", async () => { + const second = await stoppedWhileStarting(async () => { + expect(await stop()).toMatchObject({ ok: true }) + }) + + await eventually(() => expect(submission(second)?.dispatchState).toBe('accepted')) + expect(conversation()?.lastEndedChild).toMatchObject({ cause: 'user-stop', reason: null }) + expect(statusRows()).toEqual([]) + }) + + it('fails the start after a host stop, with the stop as the reason (R2)', async () => { + const reason = 'Claude never finished starting, so Orca stopped it.' + const second = await stoppedWhileStarting(() => + host['serialize'](SESSION, () => + stopStructuredAgentSessionAgentUnderSerialize(host['lifetimeContext'](), SESSION, { + cause: 'host-stop', + reason + }) + ) + ) + + await eventually(() => expect(submission(second)?.dispatchState).toBe('rejected')) + expect(submission(second)?.reason).toBe(reason) + expect(statusRows()).toEqual([{ itemId: expect.any(String), text: reason, tone: 'error' }]) + expect(dispatch).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts new file mode 100644 index 00000000000..6c469e4dacc --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts @@ -0,0 +1,78 @@ +// The provider child behind a conversation, kept as its own record on the conversation's entry. +// +// The entry is the conversation and outlives any number of children. A child enters only when an +// attach has fully succeeded, and leaves only through `endProviderChild`, which every ending shares: +// an exit, a failed re-attach, a Stop and an eviction. Each is matched on the child's generation +// and fence, so an ending that arrives late for an older child cannot end a newer one. + +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { + StructuredAgentSessionEndedChild, + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChild, + StructuredAgentSessionProviderChildIdentity +} from './structured-agent-session-host-types' + +type ChildBearer = Pick & { + journal: Pick +} + +/** The fence a conversation write carries: the record's, which is where the next child starts. A + * child's own writes carry `child.fence`, which equals it while that child holds the lease. */ +export function structuredAgentSessionConversationFence( + store: Pick, + sessionId: string +): number { + return store.getRecord(sessionId)?.lease.runtimeFence ?? 0 +} + +/** For the end of a successful attach only: a failed one never wrote a child to take back. */ +export function indexProviderChild( + session: ChildBearer, + child: StructuredAgentSessionProviderChild +): void { + session.child = child +} + +export function markProviderChildStarted( + session: ChildBearer, + identity: StructuredAgentSessionProviderChildIdentity +): boolean { + const child = matchingChild(session, identity) + if (child) { + child.phase = 'ready' + } + return child !== null +} + +export function endProviderChild( + session: ChildBearer, + ended: Omit +): boolean { + if (!matchingChild(session, ended)) { + return false + } + session.child = null + session.lastEndedChild = { ...ended, endedAt: session.journal.cursor() } + return true +} + +/** The conversation's last start died before it proved itself, and nothing started since. Only a + * send retries it: a view or an exit recovery would respawn into the same failure, adding a row. */ +export function failedProviderChildStart( + session: Pick +): StructuredAgentSessionEndedChild | null { + const ended = session.lastEndedChild + return !session.child && ended?.duringStartup && ended.cause !== 'user-stop' ? ended : null +} + +function matchingChild( + session: ChildBearer, + identity: StructuredAgentSessionProviderChildIdentity +): StructuredAgentSessionProviderChild | null { + const { child } = session + return child && child.generation === identity.generation && child.fence === identity.fence + ? child + : null +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts index 52699fd062f..bb5d29d4dea 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts @@ -102,7 +102,7 @@ describe('a publish-first Claude create whose init is slow', () => { expect(store.getRecord(SESSION)?.options?.model).toBeUndefined() expect(lastPhase()).toBe('starting') - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) expect(store.getRecord(SESSION)?.options?.model).toBe('claude-opus-9') @@ -116,7 +116,7 @@ describe('a publish-first Claude create whose init is slow', () => { ).resolves.toMatchObject({ ok: true }) expect(store.getRecord(SESSION)?.options?.model).toBe('opus') - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) expect(store.getRecord(SESSION)?.options?.model).toBe('opus') @@ -126,7 +126,7 @@ describe('a publish-first Claude create whose init is slow', () => { it('keeps the picked model across a resume whose new child starts on its own default', async () => { const params = claudeParams() await host.attach(CALLER, { ...params, options: { model: 'opus' } }) - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) await host.close(SESSION) const releasedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 @@ -138,7 +138,7 @@ describe('a publish-first Claude create whose init is slow', () => { expect(store.getRecord(SESSION)?.options?.model).toBe('opus') expect(lastPhase()).toBe('starting') - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) expect(store.getRecord(SESSION)?.options?.model).toBe('opus') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts index 55c718f967b..6d6d910cc55 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts @@ -16,6 +16,7 @@ import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { nativeSessionOptionsFromReport } from './structured-agent-session-option-restoration' +import { markProviderChildStarted } from './structured-agent-session-provider-child' export type StructuredAgentSessionProviderStartedContext = { deps: StructuredAgentSessionHostDeps @@ -35,13 +36,14 @@ export function settleStructuredAgentSessionProviderStarted( return context.serialize(event.sessionId, async () => { const session = context.sessions.get(event.sessionId) if ( - !session?.hasProviderChild || - session.fence !== event.fence || - session.acquisitionGeneration !== event.acquisitionGeneration + !session || + !markProviderChildStarted(session, { + generation: event.acquisitionGeneration, + fence: event.fence + }) ) { return } - session.providerChildPhase = 'ready' // Prompts held for the start are written now but open a turn only on their echo; a release // tick in between would stop the child before it runs them. context.restartReleaseGrace(event.sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.test.ts index 34e3fb8a4cf..514008e49f9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.test.ts @@ -48,6 +48,11 @@ const store = { } as unknown as AgentSessionRecordStore let journalRoot: string +const openDeps = () => ({ + store, + journalRoot, + adapter: {} +}) const opened: AgentSessionJournal[] = [] async function writeRemnant(name: string): Promise { @@ -73,29 +78,29 @@ describe('a session whose journal is still the pre-SQLite format', () => { it('is published, carrying the message that explains it', async () => { const transcript = await writeRemnant('log.jsonl') - const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID) + const restored = await restoreStructuredAgentSessionRead(openDeps(), SESSION_ID) expect(restored).not.toBeNull() - opened.push(restored!.journal) - const disclosed = restored!.journal + opened.push(restored!.session.journal) + const disclosed = restored!.session.journal .snapshot() .items.map((entry) => (entry.body.kind === 'status' ? entry.body.text : '')) expect(disclosed.join('')).toContain(transcript) // Publishing it costs no agent process; acquisition still waits for the user. - expect(restored!.hasProviderChild).toBe(false) + expect(restored!.session.child).toBeNull() }) it('is published for a remnant whose log is gone', async () => { await writeRemnant('snapshot.json') - const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID) + const restored = await restoreStructuredAgentSessionRead(openDeps(), SESSION_ID) expect(restored).not.toBeNull() - opened.push(restored!.journal) + opened.push(restored!.session.journal) }) it('still drops a session with neither a journal nor a remnant', async () => { - const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID) + const restored = await restoreStructuredAgentSessionRead(openDeps(), SESSION_ID) expect(restored).toBeNull() }) @@ -103,7 +108,7 @@ describe('a session whose journal is still the pre-SQLite format', () => { it('still drops a session with no record', async () => { await writeRemnant('log.jsonl') - const restored = await restoreStructuredAgentSessionRead(store, journalRoot, 'unknown-session') + const restored = await restoreStructuredAgentSessionRead(openDeps(), 'unknown-session') expect(restored).toBeNull() }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts index 65b913daf42..0e1441b729c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts @@ -1,104 +1,32 @@ -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { findJournalFileFormatRemnant } from '../agent-session-journal/journal-file-format-remnant' -import { loadJournal } from '../agent-session-journal/journal-open' -import { journalDirectoryFor } from '../agent-session-journal/journal-paths' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' +import { existsSync } from 'node:fs' +import { journalDatabaseFile, journalDirectoryFor } from '../agent-session-journal/journal-paths' import { - attachFingerprintFields, - journalIdentityFor, - type AgentSessionAttachParams -} from './structured-agent-session-attach' -import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' - -export type RestoredStructuredAgentSessionRead = { - journal: AgentSessionJournal - params: AgentSessionAttachParams - fence: number - hasProviderChild: false - providerChildPhase: 'ready' - acquisitionGeneration: null -} + openStructuredAgentSessionConversationJournal, + type OpenedStructuredAgentSessionConversation, + type StructuredAgentSessionConversationOpenDeps +} from './structured-agent-session-conversation-open' +/** + * A reader's open: the conversation's own open, for a session that has a journal to read. One + * with none — never written, or gone — stays unpublished rather than founding an empty one. + * Opening can still write: the crash boundary, and the row explaining an old-format history. + */ export async function restoreStructuredAgentSessionRead( - store: AgentSessionRecordStore, - journalRoot: string, + deps: StructuredAgentSessionConversationOpenDeps, sessionId: string -): Promise { - const record = store.getRecord(sessionId) +): Promise { + const record = deps.store.getRecord(sessionId) if (!record) { return null } - const params = attachParamsForRecord(record, { - clientOperationId: `read-restore:${record.sessionId}`, - expectedRuntimeFence: record.lease.runtimeFence - }) - const journalDir = journalDirectoryFor(journalRoot, { + const journalDir = journalDirectoryFor(deps.journalRoot, { workspaceId: record.location.workspaceId, sessionId }) - const loaded = loadJournal(journalDir, sessionId) - if (loaded?.corrupt) { + // A session still in the pre-SQLite format has no `journal.db`; the open imports it. + if (!existsSync(journalDatabaseFile(journalDir)) && !findJournalFileFormatRemnant(journalDir)) { return null } - // A session still in the pre-SQLite format has no `journal.db` to load. Dropping - // it here leaves it unpublished, which is also what prunes its tab out of the - // saved workspace — so the chat disappears with nowhere to explain itself. - if (!loaded && !findJournalFileFormatRemnant(journalDir)) { - return null - } - const journal = await openAgentSessionJournal({ - identity: journalIdentityFor(record, params), - journalDir, - // Omitted, not `null`: the store reads `null` as "replay already ran and - // found nothing" and founds a fresh epoch. In process the probe above is the - // previous statement, so the window is zero-width; this holds the line for a - // database another process creates in between. - ...(loaded ? { loaded } : {}) - }) - // Read restore opens the journal and nothing else: no adapter call, so no - // provider child. Opening it can still write — a session whose history is in - // the old format founds its epoch and commits the row explaining that here. - return { - journal, - params, - fence: record.lease.runtimeFence, - hasProviderChild: false, - providerChildPhase: 'ready', - acquisitionGeneration: null - } -} - -export function attachParamsForRecord( - record: AgentSessionRecord, - input: { - clientOperationId: string - expectedRuntimeFence: number - } -): AgentSessionAttachParams { - const params: AgentSessionAttachParams = { - envelope: { - sessionId: record.sessionId, - clientOperationId: input.clientOperationId, - expectedRuntimeFence: input.expectedRuntimeFence, - payloadFingerprint: '' - }, - location: record.location, - provider: record.provider, - agent: record.provider, - accountHome: record.accountHome, - runtimeKind: 'native' - } - return { - ...params, - envelope: { - ...params.envelope, - payloadFingerprint: computeAgentSessionPayloadFingerprint({ - method: 'agentSession.attach', - sessionId: record.sessionId, - fields: attachFingerprintFields(params) - }) - } - } + return openStructuredAgentSessionConversationJournal(deps, record) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts index a788e1ebb66..d1cc9408a62 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts @@ -19,15 +19,17 @@ describe('StructuredAgentSessionReadableRestorer', () => { (sessionId) => ({ sessionId }) as AgentSessionRecord ) const restorer = new StructuredAgentSessionReadableRestorer({ - store: { listRecords: () => records } as never, - journalRoot: '/tmp/journals', + openDeps: { + store: { getRecord: () => null, listRecords: () => records }, + journalRoot: '/tmp/journals', + adapter: {} + }, supportsRecord: () => true, reconcile: async () => null, resolveRecovery: async () => undefined, serialize: async (_sessionId, task) => task(), hasSession: () => false, - onReadable: () => undefined, - settleStaleState: async () => undefined + onReadable: () => undefined }) await restorer.restore(['visible-a', 'visible-b', 'background-a', 'background-b']) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts index 5775ecc5401..ed10f63e44c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts @@ -1,10 +1,7 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' -import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' -import type { RestoredStructuredAgentSessionRead } from './structured-agent-session-read-restore' +import type { StructuredAgentSessionReadRestoreDeps } from './structured-agent-session-restart-restore' import { restoreOneStructuredAgentSessionRead, - restoreOneStructuredAgentSessionReadUnderSerialize, restoreStructuredAgentSessionsOnRestart } from './structured-agent-session-restart-restore' @@ -12,19 +9,8 @@ export class StructuredAgentSessionReadableRestorer { private restorePromise: Promise | null = null constructor( - private readonly input: { - store: AgentSessionRecordStore - journalRoot: string + private readonly input: StructuredAgentSessionReadRestoreDeps & { supportsRecord: (record: AgentSessionRecord) => boolean - reconcile: (sessionId: string) => Promise - resolveRecovery: (sessionId: string) => Promise - serialize: (sessionId: string, task: () => Promise) => Promise - hasSession: (sessionId: string) => boolean - onReadable: (sessionId: string, restored: RestoredStructuredAgentSessionRead) => void - settleStaleState: ( - sessionId: string, - restored: RestoredStructuredAgentSessionRead - ) => Promise } ) {} @@ -55,19 +41,8 @@ export class StructuredAgentSessionReadableRestorer { return this.input.hasSession(sessionId) } - /** `restoreOne` for a caller already inside the session's serialize. Reconciliation is skipped - * on purpose: a lease this host has not adjudicated is the attach's problem, and a replay - * needs only the journal. */ - async restoreOneUnderSerialize(sessionId: string): Promise { - if (!this.supports(sessionId)) { - return false - } - await restoreOneStructuredAgentSessionReadUnderSerialize(this.input, sessionId) - return this.input.hasSession(sessionId) - } - private supports(sessionId: string): boolean { - const record = this.input.store.getRecord(sessionId) + const record = this.input.openDeps.store.getRecord(sessionId) return record !== null && this.input.supportsRecord(record) } @@ -75,7 +50,7 @@ export class StructuredAgentSessionReadableRestorer { const targetOrder = sessionIds ? new Map(sessionIds.map((sessionId, index) => [sessionId, index])) : null - const records = this.input.store + const records = this.input.openDeps.store .listRecords() .filter( (record) => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts index 87f4f9e3d65..0d873389890 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts @@ -106,6 +106,7 @@ async function createHarness(options: { attached?: boolean } = {}) { async function abandonHost(host: StructuredAgentSessionHost): Promise { host['runtimeState'].stopLeaseRenewal() host['holds'].dispose() + host['conversationDelivery'].loop.dispose() await Promise.all([...host['sessions'].values()].map((session) => session.journal.close())) host['sessions'].clear() } @@ -225,16 +226,20 @@ const UNREACHABLE = new Set([ // StructuredAgentSessionHost.mutate maps an absent record to AGENT_SESSION_NOT_ATTACHED. 'agentSession.setOption:agent_session_identity_required', 'agentSession.send:agent_session_identity_required', - // No structured-agent-session host branch emits agent_session_journal_unreadable. + // Only a send opens the conversation it writes to. 'agentSession.setOption:agent_session_journal_unreadable', - 'agentSession.send:agent_session_journal_unreadable', // Send reconstructs doubt from its global tombstone instead of refusing it. 'agentSession.send:agent_session_operation_unknown', // Only a send restarts a lost owner. 'agentSession.setOption:agent_session_owner_restart_failed', // A write names its target, not an owner generation; only an attach compares fences. 'agentSession.setOption:agent_session_checkpoint_stale', - 'agentSession.send:agent_session_checkpoint_stale' + 'agentSession.send:agent_session_checkpoint_stale', + // A send is a conversation write: admitted whoever owns the lease, and a start it needs that + // fails rejects the accepted message rather than refusing the call. + 'agentSession.send:agent_session_conflict', + 'agentSession.send:execution_owner_reconciling', + 'agentSession.send:agent_session_owner_restart_failed' ]) describe('agentSessionRefusalOperationState host oracle', () => { @@ -251,7 +256,7 @@ describe('agentSessionRefusalOperationState host oracle', () => { expect(stale.setOption).toHaveBeenCalledTimes(1) const conflict = await createHarness() - for (const method of ['agentSession.setOption', 'agentSession.send'] as const) { + for (const method of ['agentSession.setOption'] as const) { await setLease(conflict, (current) => ({ ...current, lease: { ...current.lease, handoffStage: 'new-owner-proving' } @@ -331,7 +336,7 @@ describe('agentSessionRefusalOperationState host oracle', () => { record(await assertHostAgreement(unknown, optionUnknown, 'agent_session_operation_unknown')) const reconciling = await createHarness() - for (const method of ['agentSession.setOption', 'agentSession.send'] as const) { + for (const method of ['agentSession.setOption'] as const) { await setLease(reconciling, (current) => ({ ...current, lease: { ...current.lease, unreconciled: true } @@ -348,16 +353,21 @@ describe('agentSessionRefusalOperationState host oracle', () => { ) } - const unrecoverable = await createHarness() - await unrecoverable.host.close(SESSION) - unrecoverable.host.deps.adapter.acquire = async () => { - throw new Error('no provider thread to resume') + const unreadable = await createHarness() + await unreadable.host.close(SESSION) + unreadable.host.deps.adapter.historyFilePath = async () => { + throw new Error('transcript unreadable') } + const unreadableSend = { method: 'agentSession.send' as const, operationId: operationId() } record( await assertHostAgreement( - unrecoverable, - { method: 'agentSession.send', operationId: operationId() }, - 'agent_session_owner_restart_failed' + unreadable, + unreadableSend, + 'agent_session_journal_unreadable', + async () => { + delete unreadable.host.deps.adapter.historyFilePath + return { harness: unreadable, spec: unreadableSend } + } ) ) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation-wait.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation-wait.test.ts new file mode 100644 index 00000000000..8ab9bb250ad --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation-wait.test.ts @@ -0,0 +1,63 @@ +// The restart continuation is accepted like any send, so its verdict is its delivery: a cold start +// longer than the legacy client wait must not turn a continuation that went through into an +// "unconfirmed" one. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import { structuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-wiring' +import { StructuredAgentSessionSendSettlement } from './structured-agent-session-send-settlement' + +const SESSION = 'session-1' +const MESSAGE = 'continuation-1' + +let submission: AgentJournalSubmission +const journal = { + submissions: (): AgentJournalSubmission[] => [submission], + cursor: () => ({ epoch: 'epoch-1', sequence: 1 }) +} + +beforeEach(() => { + vi.useFakeTimers() + submission = { + clientMessageId: MESSAGE, + fence: 2, + payloadFingerprint: 'fingerprint', + dispatchState: 'pending', + providerItemId: null, + reason: null, + submittedAt: 1, + resolvedAt: null, + handoverRecorded: true + } +}) + +afterEach(() => { + vi.useRealTimers() +}) + +describe('the restart continuation waits for its delivery (W18)', () => { + it('reaches accepted after a 40 s cold start, with nothing filed as unconfirmed', async () => { + const settlement = new StructuredAgentSessionSendSettlement(() => journal) + const surfaces = structuredAgentSessionRestartResumeSurfaces( + { + revealSession: async () => ({ readable: true }), + hold: async () => undefined, + release: () => undefined, + send: async () => { + throw new Error('not used') + }, + waitForSendSettlement: settlement.wait + }, + () => 0 + ) + + const verdict = surfaces.awaitSendSettlement(SESSION, MESSAGE) + await vi.advanceTimersByTimeAsync(40_000) + submission = { ...submission, dispatchState: 'accepted', providerItemId: 'item-1' } + settlement.publish(SESSION, journal) + + await expect(verdict).resolves.toMatchObject({ + value: { submission: { dispatchState: 'accepted' } } + }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.ts index c025c0dc46a..87a56efb7b9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.ts @@ -44,7 +44,9 @@ export type StructuredAgentSessionContinuationOutcome = { /** The slice of the host one continuation needs. Structural so this module never imports the host. */ export type StructuredAgentSessionContinuationHost = { - sessions: ReadonlyMap + sessions: ReadonlyMap + /** The fence a conversation write carries; null when the session is not open. */ + conversationFence: (sessionId: string) => number | null send: (input: { envelope: AgentSessionMutationEnvelope body: AgentJournalMessageItem @@ -72,7 +74,7 @@ export function restartContinuationDeps( marker: AgentSessionResumeMarker ): StructuredAgentSessionContinuationDeps { return { - currentFence: (sessionId) => host.sessions.get(sessionId)?.fence ?? null, + currentFence: host.conversationFence, send: (input) => host.send({ ...input, @@ -107,17 +109,18 @@ export function noteRestartReattachFailed( /** Writes a host-authored status note into the chat. */ function restartNoteWriter( - host: Pick + host: Pick ): StructuredAgentSessionContinuationDeps['note'] { return async (sessionId, text, tone) => { const session = host.sessions.get(sessionId) - if (!session) { + const fence = host.conversationFence(sessionId) + if (!session || fence === null) { return } await session.journal.appendItem( { provider: 'orca', clientMessageId: `restart-continuation:${sessionId}:${host.now()}` }, { kind: 'status', text, ...(tone ? { tone } : {}) }, - { fence: session.fence } + { fence } ) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-filing.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-filing.test.ts index 5dcc3ca7690..97830212b0a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-filing.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-filing.test.ts @@ -6,7 +6,6 @@ import { AGENT_SESSION_RESTART_NOT_CONNECTED_NOTE } from '../../../shared/agent-session-restart-continuation' import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' -import { AgentSessionJournal } from '../agent-session-journal/journal-store' import { latestStructuredAgentSessionUserItem } from '../../../shared/structured-agent-session-projection' import { StructuredAgentSessionResumeAdmission } from './structured-agent-session-restart-resume-runner' import { @@ -39,17 +38,17 @@ it('files a continuation superseded by a replayed message, lists it and says so await host.restartResume.list() await host.hold(SESSION, 'pane') const events = providerEvents(acquire) - const append = AgentSessionJournal.prototype.appendSubmission - vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission').mockImplementationOnce( - async function (this: AgentSessionJournal, input) { - const cursor = await append.call(this, input) - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'replayed-turn', ordinal: 1 }, - hostTestMessage('A queued notification the provider replayed') - ) - return cursor - } - ) + // The replay lands after the reattach and just before the continuation is accepted, which is + // where a send asks whether its offer still stands. + const send = host.send + vi.spyOn(host, 'send').mockImplementationOnce(async (caller, params) => { + events.appendItem( + { provider: 'codex', threadId: THREAD, turnId: 'replayed-turn', ordinal: 1 }, + hostTestMessage('A queued notification the provider replayed') + ) + await host.flushStreamedEvents(SESSION) + return send(caller, params) + }) const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts index 1bcb27ca4a2..1d7f0033a15 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts @@ -10,7 +10,6 @@ import { } from '../../runtime/agent-session-recovery-capsule' import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { parseAgentSessionResumeMarker } from '../../../shared/agent-session-resume-marker' -import { AgentSessionJournal } from '../agent-session-journal/journal-store' import { StructuredAgentSessionHost } from './structured-agent-session-host' import { StructuredAgentSessionResumeAdmission } from './structured-agent-session-restart-resume-runner' import { @@ -52,6 +51,8 @@ export async function interruptedRestart( previous.dispatch.mockResolvedValueOnce({ state: 'admitted' }) const body = hostTestMessage('Perform the original task') await previous.host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) + // Accepted first, handed over after: the work in flight is a send the provider took. + await vi.waitFor(() => expect(previous.dispatch).toHaveBeenCalledOnce()) } else if (work === 'children') { events.appendItem( { provider: 'codex', threadId: THREAD, turnId: 'settled-turn', ordinal: 1 }, @@ -150,15 +151,17 @@ export async function supersededRefusal(userAnswers?: 'before' | 'after') { if (!events) { throw new Error('missing resumed provider event sink') } - const append = AgentSessionJournal.prototype.appendSubmission - const writing = vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission') - writing.mockImplementationOnce(async function (this: AgentSessionJournal, input) { - const cursor = await append.call(this, input) + // The newer message lands after the reattach and just before the continuation is accepted, + // which is where a send asks whether its offer still stands. + const send = host.send + const writing = vi.spyOn(host, 'send') + writing.mockImplementationOnce(async (caller, params) => { events.appendItem( { provider: 'codex', threadId: THREAD, turnId: 'newer-turn', ordinal: 1 }, hostTestMessage('A newer task from another client') ) - return cursor + await host.flushStreamedEvents(SESSION) + return send(caller, params) }) const admit = StructuredAgentSessionResumeAdmission.prototype.run const admitting = vi.spyOn(StructuredAgentSessionResumeAdmission.prototype, 'run') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-ownership.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-ownership.test.ts index c3301861c01..ca5ba68dee0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-ownership.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-ownership.test.ts @@ -178,9 +178,8 @@ it('refuses at send admission once the user has sent a newer message', async () { outcome: 'refused', reason: 'agent_session_restart_work_superseded' } ]) expect(dispatch).not.toHaveBeenCalled() - expect(host.journalSnapshot(SESSION).submissions).toMatchObject([ - { dispatchState: 'rejected', reason: 'agent_session_restart_work_superseded' } - ]) + // Refused where it would have been accepted, so the chat records no continuation at all. + expect(host.journalSnapshot(SESSION).submissions).toEqual([]) host.release(SESSION, 'pane') expect(host.isHeld(SESSION)).toBe(false) }) @@ -249,6 +248,8 @@ it.each([false, true])( hostTestMessage('A newer task from another client'), { lifecycle: true } ) + // Journaled before the continuation's acceptance asks whether its offer still stands. + await host.flushStreamedEvents(SESSION) }) const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') @@ -257,8 +258,8 @@ it.each([false, true])( { outcome: 'refused', reason: 'agent_session_restart_work_superseded' } ]) expect(dispatch).not.toHaveBeenCalled() - expect(host.journalSnapshot(SESSION).submissions).toHaveLength(2) - expect(host.journalSnapshot(SESSION).submissions[1]?.dispatchState).toBe('rejected') + // Refused before acceptance: only the interrupted send is in the journal. + expect(host.journalSnapshot(SESSION).submissions).toHaveLength(1) host.release(SESSION, 'pane') expect(host.isHeld(SESSION)).toBe(false) // The offer is spent, but the refusal is kept as a durable failure: a retry finds it, and the @@ -560,7 +561,7 @@ it('fails closed on corrupt recovery storage while ordinary hold and send still expect( await host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) ).toMatchObject({ ok: true }) - expect(dispatch).toHaveBeenCalledTimes(1) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) // list; the action's read of offers and of failures; the post-action refresh of both. expect(warning).toHaveBeenCalledTimes(5) warning.mockRestore() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts index 56d5e0b2317..491a2670d0d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts @@ -15,6 +15,12 @@ import { restoreStructuredAgentSessionsOnRestart } from './structured-agent-session-restart-restore' +const NO_OPEN_DEPS = { + store: { getRecord: () => null, listRecords: () => [] }, + journalRoot: '/tmp/journals', + adapter: {} +} + describe('restart journal restoration', () => { beforeEach(() => restoreRead.mockReset()) @@ -22,17 +28,19 @@ describe('restart journal restoration', () => { const gate = Promise.withResolvers() let active = 0 let peak = 0 - restoreRead.mockImplementation(async (_store, _root, sessionId: string) => { + restoreRead.mockImplementation(async (_deps, sessionId: string) => { active += 1 peak = Math.max(peak, active) await gate.promise active -= 1 return { - journal: {}, - params: { location: { workspaceId: 'workspace-1' }, provider: 'codex' }, - fence: 1, - hasProviderChild: false, - sessionId + session: { + journal: {}, + params: { location: { workspaceId: 'workspace-1' }, provider: 'codex' }, + child: null, + sessionId + }, + reset: null } }) const records = Array.from( @@ -41,15 +49,13 @@ describe('restart journal restoration', () => { ) const restoration = restoreStructuredAgentSessionsOnRestart({ - store: {} as never, - journalRoot: '/tmp/journals', + openDeps: NO_OPEN_DEPS, records, reconcile: async () => null, resolveRecovery: async () => undefined, serialize: async (_sessionId, task) => task(), hasSession: () => false, - onReadable: () => undefined, - settleStaleState: async () => undefined + onReadable: () => undefined }) await vi.waitFor(() => expect(active).toBe(4)) @@ -82,61 +88,53 @@ describe('restart journal restoration', () => { runtimeKind: 'native' } const restored = { - journal: {}, - params, - fence: 4, - hasProviderChild: false, - acquisitionGeneration: null + session: { journal: {}, params, child: null }, + reset: null } - restoreRead.mockResolvedValue(restored) + // The open is what settles: it runs after recovery resolution and before the publish. + restoreRead.mockImplementation(async () => { + calls.push('open') + return restored + }) await restoreOneStructuredAgentSessionRead( { - store: {} as never, - journalRoot: '/tmp/journals', + openDeps: NO_OPEN_DEPS, reconcile: async () => null, resolveRecovery: async () => { calls.push('resolveRecovery') }, serialize: async (_sessionId, task) => task(), hasSession: () => false, - onReadable: () => { - calls.push('onReadable') - }, - settleStaleState: async (_sessionId, settled) => { - calls.push(settled === restored ? 'settleStaleState:restored' : 'settleStaleState') + onReadable: (_sessionId, readable) => { + calls.push(readable === restored ? 'onReadable:restored' : 'onReadable') } }, 'session-1' ) - expect(calls).toEqual(['resolveRecovery', 'settleStaleState:restored', 'onReadable']) + expect(calls).toEqual(['resolveRecovery', 'open', 'onReadable:restored']) }) it('does not settle again when a second restore finds the session already open', async () => { - const settleStaleState = vi.fn(async () => undefined) restoreRead.mockResolvedValue({ - journal: {}, - params: {}, - fence: 4, - hasProviderChild: false, - acquisitionGeneration: null + session: { journal: {}, params: {}, child: null }, + reset: null }) await restoreOneStructuredAgentSessionRead( { - store: {} as never, - journalRoot: '/tmp/journals', + openDeps: NO_OPEN_DEPS, reconcile: async () => null, resolveRecovery: async () => undefined, serialize: async (_sessionId, task) => task(), hasSession: () => true, - onReadable: () => undefined, - settleStaleState + onReadable: () => undefined }, 'session-1' ) - expect(settleStaleState).not.toHaveBeenCalled() + // The open is where the settlement runs, and a session already open is not opened again. + expect(restoreRead).not.toHaveBeenCalled() }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index 1b5d8864ae5..789eb37f45c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -13,28 +13,28 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' -import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { mapWithConcurrency } from '../../../shared/map-with-concurrency' -import { - restoreStructuredAgentSessionRead, - type RestoredStructuredAgentSessionRead -} from './structured-agent-session-read-restore' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { + OpenedStructuredAgentSessionConversation, + StructuredAgentSessionConversationOpenDeps +} from './structured-agent-session-conversation-open' +import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore' const JOURNAL_RESTORE_CONCURRENCY = 4 export type StructuredAgentSessionReadRestoreDeps = { - store: AgentSessionRecordStore - journalRoot: string + openDeps: StructuredAgentSessionConversationOpenDeps & { + store: Pick + } reconcile: (sessionId: string) => Promise resolveRecovery: (sessionId: string) => Promise serialize: (sessionId: string, task: () => Promise) => Promise hasSession: (sessionId: string) => boolean - onReadable: (sessionId: string, restored: RestoredStructuredAgentSessionRead) => void - /** Settles what a previous generation left running. Best effort: the next acquire re-derives it. */ - settleStaleState: ( + onReadable: ( sessionId: string, - restored: RestoredStructuredAgentSessionRead - ) => Promise + opened: OpenedStructuredAgentSessionConversation + ) => Promise | void } /** @@ -61,28 +61,19 @@ export async function restoreOneStructuredAgentSessionRead( /** The serialized half of the restore, for a caller already inside the session's serialize — a * send replaying into a session this host has closed, which needs the journal and no child. */ export async function restoreOneStructuredAgentSessionReadUnderSerialize( - input: Pick< - StructuredAgentSessionReadRestoreDeps, - 'store' | 'journalRoot' | 'hasSession' | 'onReadable' | 'settleStaleState' - >, + input: Pick, sessionId: string ): Promise { if (input.hasSession(sessionId)) { // A surface that took a hold mid-restore already attached this one. return } - const restored = await restoreStructuredAgentSessionRead( - input.store, - input.journalRoot, - sessionId - ) - if (!restored) { + const opened = await restoreStructuredAgentSessionRead(input.openDeps, sessionId) + if (!opened) { return } - // No child in this process writes to a journal with no map entry, so anything it shows running - // belongs to a generation that is gone. Settled before it is published, so no reader sees it run. - await input.settleStaleState(sessionId, restored) - input.onReadable(sessionId, restored) + // The open settled what a gone generation left running, so no reader sees it run. + await input.onReadable(sessionId, opened) } export async function restoreStructuredAgentSessionsOnRestart( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-host.ts index 29453e74652..0fa4ee8534d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-host.ts @@ -41,8 +41,9 @@ import { type StructuredAgentSessionContinuationOutcome } from './structured-agent-session-restart-continuation' import { createStructuredAgentSessionRestartWitnesses } from './structured-agent-session-restart-witnesses' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' -type LiveSession = { journal: AgentSessionJournal; hasProviderChild: boolean; fence: number } +type LiveSession = { journal: AgentSessionJournal; child: { fence: number } | null } export type StructuredAgentSessionRestartResumeSurfaces = { revealSession: (sessionId: string) => Promise<{ readable: boolean }> @@ -152,6 +153,10 @@ export function createStructuredAgentSessionRestartResume( const continuationHost: StructuredAgentSessionContinuationHost = { ...surfaces, sessions, + conversationFence: (sessionId) => + sessions.has(sessionId) + ? structuredAgentSessionConversationFence(deps.store, sessionId) + : null, stillResumable: (marker, options) => derive([marker], 'may-be-held', options).candidates.length === 1 } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts index e5b65009634..68823c078dc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts @@ -145,7 +145,7 @@ export type HarnessJournal = { appendItem: (envelope: unknown, body: { kind: string; text: string }) => Promise } -export type HarnessSession = { journal: HarnessJournal; hasProviderChild: boolean; fence?: number } +export type HarnessSession = { journal: HarnessJournal; child: { fence: number } | null } export function journal( items: AgentJournalRenderItem[], diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-wiring.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-wiring.ts index 07ec6b81599..66461a0c873 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-wiring.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-wiring.ts @@ -11,7 +11,9 @@ import type { AgentSessionMutationResult, AgentSessionSendResult } from '../../../shared/agent-session-wire' +import { MAX_TIMER_DELAY_MS } from '../../../shared/timer-delay' import type { StructuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-host' +import type { SendSettlementWaitOptions } from './structured-agent-session-send-settlement' /** The caller key the continuation sends under, so its writes are attributable to Orca itself. */ export const STRUCTURED_AGENT_SESSION_RESTART_CONTINUATION_CALLER = @@ -34,7 +36,8 @@ type RestartResumeHostBindings = { /** The host's existing settlement waiter; a send returns while its dispatch is still pending. */ waitForSendSettlement: ( sessionId: string, - clientMessageId: string + clientMessageId: string, + options: SendSettlementWaitOptions ) => Promise<{ value: AgentSessionSendResult } | undefined> } @@ -48,7 +51,10 @@ export function structuredAgentSessionRestartResumeSurfaces( release: host.release, send: (params) => host.send({ callerKey: STRUCTURED_AGENT_SESSION_RESTART_CONTINUATION_CALLER }, params), - awaitSendSettlement: host.waitForSendSettlement, + // Accepted like any send, so its verdict is its delivery, however long the start takes; the + // wait ends when the submission settles or the session closes. + awaitSendSettlement: (sessionId, clientMessageId) => + host.waitForSendSettlement(sessionId, clientMessageId, { budgetMs: MAX_TIMER_DELAY_MS }), onNoteFailed: () => console.warn('[structured-agent-session] restart continuation attribution failed'), now diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts index 1d6a8a1fc5e..5dc939d7e66 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts @@ -48,7 +48,7 @@ describe('deriving what was working at teardown', () => { it('marks a session this host was running a turn for', () => { const markers = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => undefined, @@ -74,7 +74,7 @@ describe('deriving what was working at teardown', () => { it('carries the update trigger so the surface can say the restart was not the user choice', () => { const [recorded] = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => undefined, @@ -89,7 +89,7 @@ describe('deriving what was working at teardown', () => { it('marks nothing for an idle session', () => { expect( markersAtTeardown({ - sessions: new Map([[SESSION, { journal: journal([]), hasProviderChild: true }]]), + sessions: new Map([[SESSION, { journal: journal([]), child: { fence: 1 } }]]), getRecord: () => record(), backgroundTasks: () => undefined, trigger: 'quit', @@ -103,7 +103,7 @@ describe('deriving what was working at teardown', () => { expect( markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => undefined, @@ -115,12 +115,12 @@ describe('deriving what was working at teardown', () => { }) // The user's stated fear. A journal restored for READING carries whatever `running` row an older - // crash left behind, and it is the live `hasProviderChild` — not that row — that decides. + // crash left behind, and it is the live `child` — not that row — that decides. it('marks nothing for a stale running row this host was not executing', () => { expect( markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: false }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: null }] ]), getRecord: () => record(), backgroundTasks: () => undefined, @@ -141,7 +141,7 @@ describe('deriving what was working at teardown', () => { SESSION, { journal: journal([turnItem('turn-1', 'running'), pendingApproval()]), - hasProviderChild: true + child: { fence: 1 } } ] ]), @@ -165,7 +165,7 @@ describe('deriving what was working at teardown', () => { it('marks a settled lead whose subagent was still running, anchored on its last turn', () => { const markers = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => [ @@ -188,7 +188,7 @@ describe('deriving what was working at teardown', () => { it('records only the live rows of the roster, bounded', () => { const [recorded] = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => [ @@ -213,7 +213,7 @@ describe('deriving what was working at teardown', () => { it('marks a settled lead whose only live work is a monitor', () => { const markers = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => [{ id: 'task-m', kind: 'monitor', name: 'ci-watch' }], @@ -230,7 +230,7 @@ describe('deriving what was working at teardown', () => { expect( markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => [ @@ -248,7 +248,7 @@ describe('deriving what was working at teardown', () => { it('records the identity root so an advancing Claude leaf cannot invalidate the marker', () => { const [recorded] = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: { fence: 1 } }] ]), getRecord: () => claudeRecord(null), backgroundTasks: () => undefined, @@ -264,7 +264,7 @@ describe('deriving what was working at teardown', () => { expect( markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: { fence: 1 } }] ]), getRecord: () => record({ chain: [] }), backgroundTasks: () => undefined, @@ -285,7 +285,7 @@ describe('deriving what was working at teardown', () => { SESSION, { journal: journal([], false, [submission('msg-1', 'pending')]), - hasProviderChild: true + child: { fence: 1 } } ] ]), @@ -310,7 +310,7 @@ describe('deriving what was working at teardown', () => { journal: journal([turnItem('turn-0', 'completed')], false, [ submission('msg-1', 'pending') ]), - hasProviderChild: true + child: { fence: 1 } } ] ]), @@ -324,6 +324,67 @@ describe('deriving what was working at teardown', () => { expect(recorded?.work).toEqual({ kind: 'submission', id: 'msg-1' }) }) + // Accepted while the agent was starting and never handed over: the chat shows working, but no + // agent had the message, and quit rejects it as never sent. + it('marks nothing for a session whose only work is a message still queued', () => { + const queued = { ...submission('msg-1', 'pending'), handoverRecorded: true as const } + expect( + markersAtTeardown({ + sessions: new Map([ + [ + SESSION, + { + journal: journal([turnItem('turn-0', 'completed')], false, [queued]), + child: { fence: 1 } + } + ] + ]), + getRecord: () => claudeRecord(null), + backgroundTasks: () => undefined, + trigger: 'quit', + teardownId: TEARDOWN_CURRENT, + now: NOW + }) + ).toEqual([]) + }) + + it('marks a handed-over message over a newer queued one, and a turn a queued one waits behind', () => { + const handedOver = { + ...submission('msg-1', 'pending'), + handoverRecorded: true as const, + handedOverAt: NOW + } + const queued = { ...submission('msg-2', 'pending'), handoverRecorded: true as const } + const markers = markersAtTeardown({ + sessions: new Map([ + [ + SESSION, + { + journal: journal([turnItem('turn-0', 'completed')], false, [handedOver, queued]), + child: { fence: 1 } + } + ], + [ + 'session-running', + { + journal: journal([turnItem('turn-1', 'running')], false, [queued]), + child: { fence: 1 } + } + ] + ]), + getRecord: () => claudeRecord(null), + backgroundTasks: () => undefined, + trigger: 'quit', + teardownId: TEARDOWN_CURRENT, + now: NOW + }) + + expect(markers.map((entry) => entry.work)).toEqual([ + { kind: 'submission', id: 'msg-1' }, + { kind: 'turn', id: 'turn-1' } + ]) + }) + // Once a turn exists it is the better identity: it is what eviction rewrites, so it is what the // journal can be asked about at launch. it('prefers the running turn over the send that opened it', () => { @@ -335,7 +396,7 @@ describe('deriving what was working at teardown', () => { journal: journal([turnItem('turn-1', 'running')], false, [ submission('msg-1', 'accepted') ]), - hasProviderChild: true + child: { fence: 1 } } ] ]), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts index 2e031b9f071..93a7962d4a2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts @@ -11,7 +11,7 @@ import { agentSessionLeaseIsReleased } from '../../../shared/agent-session-lease import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { randomUUID } from 'node:crypto' import type { AgentSessionAttachParams } from './structured-agent-session-attach' -import { attachParamsForRecord } from './structured-agent-session-read-restore' +import { attachParamsForRecord } from './structured-agent-session-conversation-open' export function isResumableStructuredAgentSessionRecord(record: AgentSessionRecord): boolean { return agentSessionLeaseIsReleased(record.lease) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.test.ts index 1172e993b96..7ba1c716769 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.test.ts @@ -46,23 +46,31 @@ function harness( return task() } const restorer = new StructuredAgentSessionReadableRestorer({ - store: { - getRecord: (sessionId: string) => records.find((r) => r.sessionId === sessionId) ?? null, - listRecords: () => records - } as never, - journalRoot: '/journals', + openDeps: { + store: { + getRecord: (sessionId: string) => records.find((r) => r.sessionId === sessionId) ?? null, + listRecords: () => records + }, + journalRoot: '/journals', + adapter: {} + }, supportsRecord: options.supports ?? (() => true), reconcile: async () => null, resolveRecovery: async () => undefined, serialize, hasSession: (sessionId) => live.has(sessionId), - onReadable: (sessionId, restored) => live.set(sessionId, restored), - settleStaleState: async () => undefined + onReadable: (sessionId, restored) => { + live.set(sessionId, restored) + } }) return { restorer, live, serializedIds } } -const readable = { journal: {}, params: {}, fence: 1 } as never +const readable = { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the restorer only indexes the session it is handed; no member of it is read here. + session: { journal: {}, params: {}, fence: 1 } as never, + reset: null +} afterEach(() => { vi.restoreAllMocks() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 097cd6b479a..c6ea9c1f83b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -18,7 +18,6 @@ import type { StructuredAgentSessionHostDeps, StructuredAgentSessionReveal } from './structured-agent-session-host-types' -import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' /** Throws its refusal as the code itself, matching `resumeHeldStructuredAgentSession`. */ export async function revealStructuredAgentSession( @@ -58,31 +57,15 @@ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, wiring: Omit< ConstructorParameters[0], - 'store' | 'journalRoot' | 'supportsRecord' | 'settleStaleState' + 'openDeps' | 'supportsRecord' > ): { restoreReadableSessions: (sessionIds?: readonly string[]) => Promise revealSession: (sessionId: string) => Promise - /** One session, for a caller already inside its serialize. */ - restoreReadableUnderSerialize: (sessionId: string) => Promise } { const restorer = new StructuredAgentSessionReadableRestorer({ - store: deps.store, - journalRoot: deps.journalRoot, + openDeps: deps, supportsRecord: (record) => adapterSupportsRecord(deps.adapter, record), - settleStaleState: async (sessionId, restored) => { - try { - await settleStaleStructuredAgentSessionState({ - journal: restored.journal, - sessionId, - fence: restored.fence, - acquisitionGeneration: null, - deathEvidence: deps.store.getRecord(sessionId)?.lease.deathEvidence ?? null - }) - } catch (error) { - deps.onEventSinkError?.({ sessionId, error }) - } - }, ...wiring }) const gate = new StructuredAgentSessionRestartRestoreGate() @@ -91,7 +74,6 @@ export function createStructuredAgentSessionHostRestore( revealSession: (sessionId) => revealStructuredAgentSession(deps, sessionId, wiring.hasSession, (id) => restorer.restoreOne(id) - ), - restoreReadableUnderSerialize: (sessionId) => restorer.restoreOneUnderSerialize(sessionId) + ) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts index b82248cb06f..32b05b08077 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts @@ -139,6 +139,14 @@ async function seed(acceptedSubmissions = false) { } }) ).toMatchObject({ ok: true }) + // Accepted into the conversation first; the delivery loop hands it over after. + await vi.waitFor(() => + expect( + host + .journalSnapshot(HOST_TEST_SESSION) + .submissions.find((entry) => entry.clientMessageId === clientOperationId)?.dispatchState + ).toBe('accepted') + ) if (i === 1) { selectedItemId = agentJournalSubmissionKey(clientOperationId) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts index 0f75c9a3322..2494535e019 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts @@ -120,11 +120,16 @@ describe('structured send idempotency', () => { await performSend(context, input) const replay = await performSend(context, input) + // Acceptance records the one submission; the reused id answers with it and writes nothing. + // Handing it over is the delivery loop's, never a second accept's. expect(replay).toMatchObject({ ok: true, - value: { clientMessageId: 'shared-send-id', submission: { dispatchState: 'accepted' } } + value: { + clientMessageId: 'shared-send-id', + submission: { dispatchState: 'pending', handoverRecorded: true } + } }) - expect(dispatch).toHaveBeenCalledOnce() + expect(dispatch).not.toHaveBeenCalled() expect(journal.submissions()).toHaveLength(1) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts new file mode 100644 index 00000000000..ffd06ae4dac --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts @@ -0,0 +1,137 @@ +// A send can be what opens a conversation this process has not read yet: a chat nobody has on +// screen after the app died, sent to from a phone or the CLI. Whatever that journal shows running +// belongs to a generation that is gone, so it is settled when the journal opens, not only when a +// new child starts: a start that then fails would leave the turn running for every reader. + +import { cp, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { afterEach, expect, it, vi } from 'vitest' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + adapter, + attach, + CALLER, + envelope, + hostTestState, + replaceHostTestState +} from './structured-agent-session-host-test-harness' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestMessage +} from './structured-agent-session-host-test-data' + +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => unknown): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + +const relaunchedRoots: string[] = [] + +afterEach(async () => { + await Promise.all( + relaunchedRoots.splice(0).map((dir) => rm(dir, { recursive: true, force: true })) + ) +}) + +/** A host that dies mid-turn, relaunched over a copy of its files taken at the crash. */ +async function relaunchAfterCrashMidTurn(probe: AgentSessionOwnerProbe) { + const dying = hostTestState() + await attach() + const events = dying.acquire.mock.calls[0]?.[0].events + if (!events) { + throw new Error('missing provider event sink') + } + events.appendItem( + { provider: 'codex', threadId: THREAD, turnId: 'crashed-turn', ordinal: 1 }, + { kind: 'turn', turnId: 'crashed-turn', state: 'running' } + ) + await dying.host.flushStreamedEvents(SESSION) + // An empty renewal queues behind every record write, so they are on disk. + await dying.store.renewLeases([]) + const relaunched = `${dying.root}-relaunched` + relaunchedRoots.push(relaunched) + // A dead process holds no lock. + await cp(dying.root, relaunched, { + recursive: true, + filter: (source) => !source.includes('.lock') + }) + const store = await AgentSessionRecordStore.open({ + directory: join(relaunched, 'store'), + hostId: 'local' + }) + const acquire = vi.fn(async () => { + throw new Error('claude: command not found') + }) + const host = new StructuredAgentSessionHost({ + store, + adapter: { ...adapter(), acquire }, + journalRoot: relaunched, + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-next', + probeOwner: async () => probe, + now: () => NOW + }) + await host.reconcileRestartLeases() + replaceHostTestState({ store, host }) + return { host, acquire } +} + +/** Neither proof of the old owner's death is an observed exit, so the turn ends `unverifiable`. */ +function turnStates(host: StructuredAgentSessionHost) { + return host + .journalSnapshot(SESSION) + .items.flatMap((item) => readAgentJournalTurn(item.body) ?? []) + .map((turn) => turn.state) +} + +// A host that cannot prove the old owner gone leaves its lease in recovery, still claimed, until +// the next acquire resolves it: the open is not that acquire, and the turn is no less gone. +const PROBES = [ + ['the old owner is proven gone', { outcome: 'pid-absent' }], + [ + 'nothing proves the old owner gone', + { outcome: 'indeterminate', reason: 'This host cannot probe structured session owners.' } + ] +] as const satisfies readonly (readonly [string, AgentSessionOwnerProbe])[] + +it.each(PROBES)( + 'settles a turn a dead generation left running when a send opens the chat and its start fails, when %s', + async (_when, probe) => { + const { host, acquire } = await relaunchAfterCrashMidTurn(probe) + expect(host.hasSession(SESSION)).toBe(false) + + const body = hostTestMessage('sent to a chat nobody has open') + const sendEnvelope = envelope('agentSession.send', { body }) + await expect(host.send(CALLER, { envelope: sendEnvelope, body })).resolves.toMatchObject({ + ok: true + }) + await eventually(() => + expect( + host + .journalSnapshot(SESSION) + .submissions.find((entry) => entry.clientMessageId === sendEnvelope.clientOperationId) + ).toMatchObject({ dispatchState: 'rejected' }) + ) + + expect(acquire).toHaveBeenCalledOnce() + expect(turnStates(host)).toEqual(['unverifiable']) + await host.flushAllStreamedEvents() + } +) + +it.each(PROBES)( + 'settles the same turn when a reader opens the chat, when %s', + async (_when, probe) => { + const { host } = await relaunchAfterCrashMidTurn(probe) + + await host.revealSession(SESSION) + + expect(turnStates(host)).toEqual(['unverifiable']) + await host.flushAllStreamedEvents() + } +) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts index a93a64b0dc2..8c8dfec04ce 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts @@ -1,11 +1,11 @@ -// A send, or a hold, that finds the session's provider child gone, against the real host. +// A send, or a hold, that finds the session's provider child gone, against the real host. The +// send is accepted at once; its delivery restarts the child, or rejects it with the reason. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest' import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' -import { agentSessionRefusalOperationState } from '../../../shared/agent-session-refusal-retry' import type { AgentSessionMutationEnvelope, AgentSessionSubscribeEvent @@ -24,6 +24,11 @@ import { } from './structured-agent-session-host-test-data' const CALLER = { callerKey: 'client-1' } + +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => void): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} // Long enough that no release fires mid-test; whether one is pending is asserted directly. const GRACE_MS = 60_000 @@ -105,15 +110,39 @@ function sendParams(text: string, operationId = hostTestOperationId()) { return { envelope, body } } -/** Every status row the chat shows, oldest first; none when the session is not even readable. */ -function journalStatuses(): string[] { - if (!host.hasSession(SESSION)) { - return [] - } - const history = host.history({ sessionId: SESSION, direction: 'tail' }) - return history.ok - ? history.page.items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) - : [] +/** Accepted at once, before any owner exists for it; answers the message id. */ +async function accept(params: ReturnType): Promise { + const result = await host.send(CALLER, params) + expect(result, JSON.stringify(result)).toMatchObject({ + ok: true, + replayed: false, + value: { submission: { dispatchState: 'pending', handoverRecorded: true } } + }) + return params.envelope.clientOperationId +} + +function submission(clientMessageId: string) { + return host + .journalSnapshot(SESSION) + .submissions.find((entry) => entry.clientMessageId === clientMessageId) +} + +/** The submission once delivery is done with it: handed over, or rejected unwritten. */ +async function settled(clientMessageId: string) { + await eventually(() => { + const current = submission(clientMessageId) + expect(current?.dispatchState !== 'pending' || current.handedOverAt !== undefined).toBe(true) + }) + return submission(clientMessageId) +} + +/** The failure rows a start the chat needed left, oldest first. */ +function errorStatuses(): string[] { + return host + .journalSnapshot(SESSION) + .items.flatMap((item) => + item.body.kind === 'status' && item.body.tone === 'error' ? [item.body.text] : [] + ) } /** The child timed out or exited: its lease is handed back and the host holds no session. */ @@ -130,15 +159,14 @@ describe('a send with no live owner', () => { it('restarts the owner once and delivers against it', async () => { await loseOwner() - const result = await host.send(CALLER, sendParams('after the child died')) + await accept(sendParams('after the child died')) - expect(result).toMatchObject({ ok: true, replayed: false }) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') }) - it('restarts the owner before the send is admitted, so the send is admitted once', async () => { + it('accepts the send before anything restarts, and the restart hands it over', async () => { await loseOwner() const order: string[] = [] const spawnChild = acquire.getMockImplementation()! @@ -152,21 +180,18 @@ describe('a send with no live owner', () => { return admit(args) }) - await expect(host.send(CALLER, sendParams('ensure first'))).resolves.toMatchObject({ - ok: true, - replayed: false - }) + await accept(sendParams('accept first')) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) - expect(order).toEqual(['acquire', 'admit']) + expect(order).toEqual(['admit', 'acquire']) }) it('leaves a live owner alone', async () => { acquire.mockClear() - await expect(host.send(CALLER, sendParams('owner is live'))).resolves.toMatchObject({ - ok: true - }) + await accept(sendParams('owner is live')) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).not.toHaveBeenCalled() }) @@ -184,36 +209,35 @@ describe('a send with no live owner', () => { } })) - await host.send(CALLER, sendParams('into a cleared chat')) + await expect(host.send(CALLER, sendParams('into a cleared chat'))).resolves.toMatchObject({ + ok: false, + refusal: { code: 'agent_session_operation_invalid' } + }) expect(acquire).not.toHaveBeenCalled() }) it('renews the idle window on journal activity in an unheld session', async () => { await loseOwner() - await expect(host.send(CALLER, sendParams('restart'))).resolves.toMatchObject({ ok: true }) + await accept(sendParams('restart')) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) const arm = vi.spyOn(host['holds']['clock'], 'arm') - await expect(host.send(CALLER, sendParams('more activity'))).resolves.toMatchObject({ - ok: true - }) + await accept(sendParams('more activity')) - expect(arm).toHaveBeenCalledWith(SESSION) + await eventually(() => expect(arm).toHaveBeenCalledWith(SESSION)) }) it('releases the restarted child on the usual clock only when no surface holds it', async () => { await loseOwner() - await expect(host.send(CALLER, sendParams('nobody is watching'))).resolves.toMatchObject({ - ok: true - }) - expect(host['holds'].isReleasePending(SESSION)).toBe(true) + await accept(sendParams('nobody is watching')) + await eventually(() => expect(host['holds'].isReleasePending(SESSION)).toBe(true)) await host.close(SESSION) // A reading surface that does not itself restart the agent. await host.hold(SESSION, 'desktop-chat:1', { resume: false }) - await expect(host.send(CALLER, sendParams('the chat is open'))).resolves.toMatchObject({ - ok: true - }) + await accept(sendParams('the chat is open')) + await eventually(() => expect(dispatch).toHaveBeenCalledTimes(2)) expect(host['holds'].isReleasePending(SESSION)).toBe(false) }) @@ -231,15 +255,15 @@ describe('a send with no live owner', () => { expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') acquire.mockClear() - await expect(host.send(CALLER, sendParams('after an exit'))).resolves.toMatchObject({ - ok: true - }) + await accept(sendParams('after an exit')) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).toHaveBeenCalledOnce() }) it('restarts nothing for a resend the journal already answers', async () => { const params = sendParams('sent once') - await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: false }) + await accept(params) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) // The child died during startup: the lease is handed back, the fence moves, and the session // stays readable. The client resends against the new fence. await host.handleAdapterEvent({ @@ -268,12 +292,9 @@ describe('a send with no live owner', () => { expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') // Retry rotates the id: a genuinely new send restarts the owner once. - await expect(host.send(CALLER, sendParams('sent once'))).resolves.toMatchObject({ - ok: true, - replayed: false - }) + await accept(sendParams('sent once')) + await eventually(() => expect(dispatch).toHaveBeenCalledTimes(2)) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledTimes(2) }) it('restarts nothing for a send the ledger holds but the journal never saw', async () => { @@ -320,7 +341,7 @@ describe('a send with no live owner', () => { expect(dispatch).not.toHaveBeenCalled() }) - it('admits a send that arrives after the restart has already claimed the lease', async () => { + it('accepts a send that arrives while a restart holds the queue, and hands both over in order', async () => { await loseOwner() const lostFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 let claimed = () => {} @@ -334,18 +355,22 @@ describe('a send with no live owner', () => { return spawnChild!(input) }) - const first = host.send(CALLER, sendParams('first')) + const first = await accept(sendParams('first')) await claim expect(store.getRecord(SESSION)?.lease.runtimeFence).toBe(lostFence + 1) + // Written against the lost owner's fence, which admits it: a send is a conversation write. const late = sendParams('second') late.envelope.expectedRuntimeFence = lostFence const second = host.send(CALLER, late) release() - expect(await first).toMatchObject({ ok: true }) expect(await second).toMatchObject({ ok: true }) + await eventually(() => expect(dispatch).toHaveBeenCalledTimes(2)) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledTimes(2) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([ + first, + late.envelope.clientOperationId + ]) }) it('shares one restart between concurrent sends', async () => { @@ -358,8 +383,8 @@ describe('a send with no live owner', () => { ]) expect(results.map((result) => result.ok)).toEqual([true, true, true]) + await eventually(() => expect(dispatch).toHaveBeenCalledTimes(3)) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledTimes(3) }) it('shares one restart between a hold and a send that arrive in the same gap', async () => { @@ -372,21 +397,22 @@ describe('a send with no live owner', () => { expect(held).toMatchObject({ status: 'fulfilled' }) expect(sent).toMatchObject({ status: 'fulfilled', value: { ok: true } }) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledOnce() expect(host['holds'].isHeld(SESSION)).toBe(true) expect(host['holds'].isReleasePending(SESSION)).toBe(false) }) it('replays into a closed session without spawning anything', async () => { const params = sendParams('sent once') - await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: false }) + await accept(params) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) await loseOwner() await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: true }) await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: true }) - // The journal was made readable for the answer; the record's lease was left as it was. + // The conversation was opened for the answer; the record's lease was left as it was. expect(host.hasSession(SESSION)).toBe(true) expect(acquire).not.toHaveBeenCalled() expect(dispatch).toHaveBeenCalledOnce() @@ -394,104 +420,83 @@ describe('a send with no live owner', () => { expect(host['holds'].isReleasePending(SESSION)).toBe(false) }) - it("refuses with the restart's own cause, in the answer and in the chat", async () => { + it("rejects the accepted message with the restart's own cause, and says so in the chat once", async () => { await loseOwner() acquire.mockRejectedValue(new Error('Not signed in. Run codex login')) const params = sendParams('while signed out') - - const result = await host.send(CALLER, params) - - expect(result).toEqual({ - ok: false, - refusal: { - code: 'agent_session_owner_restart_failed', - message: "Codex couldn't restart: Not signed in. Run codex login.", - // The failed attach proved its child gone: nothing runs for this session. - ownerVerdict: 'exited' - } - }) - expect(dispatch).not.toHaveBeenCalled() - expect(hostErrors).not.toEqual([]) - expect(agentSessionRefusalOperationState('agent_session_owner_restart_failed')).toBe( - 'settled-rejected' - ) - // Refused before admission: the ledger holds nothing a resend would replay. - expect(store.getOperationRow(CALLER.callerKey, params.envelope.clientOperationId)).toBeNull() - // The same status row a failed start leaves, so the reason outlives the error strip. - expect(journalStatuses()).toEqual([ + const cause = 'The provider stopped before it finished starting: Not signed in. Run codex login.' - ]) + + const id = await accept(params) + + expect(await settled(id)).toMatchObject({ dispatchState: 'rejected', reason: cause }) + expect(dispatch).not.toHaveBeenCalled() + // Accepted, so the ledger answers a resend with the rejection rather than a second attempt. + expect( + store.getOperationRow(CALLER.callerKey, params.envelope.clientOperationId) + ).toMatchObject({ outcome: { status: 'succeeded' } }) + // One row, in the error tone, so the reason outlives the error strip. + expect(errorStatuses()).toEqual([cause]) }) - it('restarts again for a Retry of the refused send, under its own id or a new one', async () => { + it('restarts again for a Retry under a new id, and replays a resend of the same id', async () => { await loseOwner() acquire.mockRejectedValue(new Error('Not signed in')) const params = sendParams('while signed out') - await expect(host.send(CALLER, params)).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } - }) + await settled(await accept(params)) + expect(acquire).toHaveBeenCalledTimes(1) - // A client that resends the same id gets another attempt, and the chat no second row. + // A client that resends the same id gets the recorded rejection, and the chat no second row. await expect(host.send(CALLER, params)).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } + ok: true, + replayed: true, + value: { submission: { dispatchState: 'rejected' } } + }) + expect(acquire).toHaveBeenCalledTimes(1) + expect(errorStatuses()).toHaveLength(1) + + // The outbox's Retry rotates the id: a fresh attempt, with its own row. + expect(await settled(await accept(sendParams('while signed out')))).toMatchObject({ + dispatchState: 'rejected' }) expect(acquire).toHaveBeenCalledTimes(2) - expect(journalStatuses()).toHaveLength(1) - - // The outbox's Retry rotates the id: also a fresh attempt. - await expect(host.send(CALLER, sendParams('while signed out'))).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } - }) - expect(acquire).toHaveBeenCalledTimes(3) + expect(errorStatuses()).toHaveLength(2) expect(dispatch).not.toHaveBeenCalled() }) it('restarts and delivers a later send once the cause clears', async () => { await loseOwner() acquire.mockRejectedValueOnce(new Error('Not signed in')) - await expect(host.send(CALLER, sendParams('while signed out'))).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } + expect(await settled(await accept(sendParams('while signed out')))).toMatchObject({ + dispatchState: 'rejected' }) // The user signed in; nothing about the failed attempt is remembered. - await expect(host.send(CALLER, sendParams('signed in now'))).resolves.toMatchObject({ - ok: true, - replayed: false - }) + await accept(sendParams('signed in now')) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).toHaveBeenCalledTimes(2) - expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') }) it('suggests a new chat only when this host has nothing to restart the chat from', async () => { await loseOwner() acquire.mockRejectedValue(new Error('Not signed in')) - const failed = await host.send(CALLER, sendParams('restart fails')) - expect(failed).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } - }) - expect(failed.ok ? '' : failed.refusal.message).not.toMatch(/new chat/) + const failed = await settled(await accept(sendParams('restart fails'))) + expect(failed).toMatchObject({ dispatchState: 'rejected' }) + expect(failed?.reason).not.toMatch(/new chat/) // The adapter cannot run this record where it lives: no retry would bring it back. host.deps.adapter.supportsLocation = () => false - const unresumable = await host.send(CALLER, sendParams('cannot resume here')) + const unresumable = await settled(await accept(sendParams('cannot resume here'))) expect(unresumable).toMatchObject({ - ok: false, - refusal: { - code: 'agent_session_owner_restart_failed', - message: - "Codex couldn't restart: This execution host cannot resume the requested structured agent session. Start a new chat to continue." - } + dispatchState: 'rejected', + reason: + "Codex couldn't restart: This execution host cannot resume the requested structured agent session. Start a new chat to continue." }) }) - it('runs the send as the lease stands when the restart met a lease someone else is settling', async () => { + it('rejects the message with the cause when the restart met a lease someone else is settling', async () => { await loseOwner() vi.spyOn(host['holds'], 'ensureProviderChild').mockResolvedValueOnce({ ok: false, @@ -501,33 +506,30 @@ describe('a send with no live owner', () => { } }) - const result = await host.send(CALLER, sendParams('owner being settled')) + const id = await accept(sendParams('owner being settled')) - // The ordinary lease check answers, retryably; nothing terminal and nothing in the chat. - expect(result).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_ownership_unknown' } - }) + const cause = "Codex couldn't restart: Another runtime is still adjudicating this lease." + expect(await settled(id)).toMatchObject({ dispatchState: 'rejected', reason: cause }) expect(acquire).not.toHaveBeenCalled() - expect(journalStatuses()).toEqual([]) + expect(errorStatuses()).toEqual([cause]) }) - it('runs the send as the lease stands when the restart itself faults', async () => { + it('rejects the message, and reports the fault, when the restart itself faults', async () => { await loseOwner() vi.spyOn(host['holds'], 'ensureProviderChild').mockRejectedValueOnce( new Error('spawn-token mint failed') ) - const result = await host.send(CALLER, sendParams('bookkeeping failed')) + const id = await accept(sendParams('bookkeeping failed')) - expect(result).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_ownership_unknown' } + expect(await settled(id)).toMatchObject({ + dispatchState: 'rejected', + reason: "Codex couldn't restart: spawn-token mint failed." }) expect(hostErrors).toContainEqual( expect.objectContaining({ message: 'spawn-token mint failed' }) ) - expect(journalStatuses()).toEqual([]) + expect(errorStatuses()).toHaveLength(1) }) it('keeps a second surface holder taken during an auto-restart, and starts nothing for it', async () => { @@ -663,20 +665,22 @@ describe('a send with no live owner', () => { }) }) - it('leaves a lease it cannot adjudicate alone', async () => { + it('adjudicates a lease this host has not reconciled before its delivery resumes it', async () => { await loseOwner() await store.transitionHandoff(SESSION, (current) => ({ ...current, lease: { ...current.lease, unreconciled: true } })) - const result = await host.send(CALLER, sendParams('owner unverifiable')) + // The send's start is the same serialized resume a hold runs, reconciliation first. + await accept(sendParams('owner unverified')) - expect(result).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_ownership_unknown' } + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) + expect(acquire).toHaveBeenCalledOnce() + expect(store.getRecord(SESSION)?.lease).toMatchObject({ + unreconciled: false, + claimStatus: 'live' }) - expect(acquire).not.toHaveBeenCalled() }) }) @@ -689,20 +693,17 @@ describe('a write fenced to an owner the pane has not seen replaced', () => { const params = sendParams('after the release') params.envelope.expectedRuntimeFence = seenFence - expect(await host.send(CALLER, params)).toMatchObject({ - ok: true, - replayed: false, - value: { submission: { dispatchState: 'accepted' } } - }) + const id = await accept(params) + expect(await settled(id)).toMatchObject({ dispatchState: 'accepted' }) expect(acquire).toHaveBeenCalledOnce() expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease.runtimeFence).toBeGreaterThan(seenFence + 1) }) - // Clients resend a refused message when the fence they hold moves, and a refused restart leaves - // no ledger row, so a frame carrying each failed attempt's fence would resend it forever. - it("keeps the pane's fence on the row a failed restart publishes", async () => { + // Clients resend a refused message when the fence they hold moves. A failed start is a + // rejected message now, never a refused send, and the pane keeps the fence it subscribed under. + it("keeps the pane's fence on the rows a failed start publishes, and rejects the message", async () => { const seenFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 const frames: AgentSessionSubscribeEvent[] = [] host.subscribe({ id: 'pane', sessionId: SESSION, emit: (event) => frames.push(event) }) @@ -710,11 +711,12 @@ describe('a write fenced to an owner the pane has not seen replaced', () => { acquire.mockRejectedValueOnce(new Error('Not signed in')) const subscribed = frames.length - expect(await host.send(CALLER, sendParams('while signed out'))).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } - }) + const id = await accept(sendParams('while signed out')) + expect(await settled(id)).toMatchObject({ + dispatchState: 'rejected', + reason: expect.stringContaining('Not signed in') + }) expect(store.getRecord(SESSION)?.lease.runtimeFence).toBeGreaterThan(seenFence + 1) const published = frames.slice(subscribed) expect(published.length).toBeGreaterThan(0) @@ -736,7 +738,8 @@ describe('a write fenced to an owner the pane has not seen replaced', () => { return spawnChild(input) }) - const first = host.send(CALLER, sendParams('starts the agent')) + const firstParams = sendParams('starts the agent') + const first = host.send(CALLER, firstParams) await claim const cancelFields = { turnId: 'turn-1' } const stop = host.cancel(CALLER, { @@ -760,7 +763,14 @@ describe('a write fenced to an owner the pane has not seen replaced', () => { expect(await first).toMatchObject({ ok: true }) expect(await stop).toMatchObject({ ok: true, replayed: false }) expect(await second).toMatchObject({ ok: true, replayed: false }) + // The Stop withdrew the message its start held; the one typed after it is delivered. + expect(await settled(late.envelope.clientOperationId)).toMatchObject({ + dispatchState: 'accepted' + }) + expect(submission(firstParams.envelope.clientOperationId)).toMatchObject({ + dispatchState: 'rejected' + }) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledTimes(2) + expect(dispatch).toHaveBeenCalledTimes(1) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts index 09986f2e129..1fa923e971f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts @@ -1,23 +1,8 @@ -// What a send needs from the session before its lease is checked. -// -// A provider child that exits or fails to start hands its lease back. Before this, a send to that -// session was refused `agent_session_ownership_unknown` — which a client reads as "not admitted -// yet" and resends forever — and only a surface hold could ever make a new child. Now the send -// makes sure it has an owner as a step of its own serialized admission: a released lease where -// resume is allowed gets a child first; anything else runs as it is and meets the lease check. -// A restart that fails refuses with a code the client stops auto-retrying on, carrying the -// restart's own cause, and writes that cause into the chat the way a start that failed does, so -// the user sees why. A manual Retry or a new send is a fresh attempt: a refusal before admission -// leaves no ledger row behind. -// -// The ledger's answer comes first, so a send it already holds a row for restarts nothing: -// admission replays or refuses it whoever owns the session now, and a closed session is made -// readable for that, never given a child. Otherwise each resend of a message whose child died at -// startup would spawn another child that dies the same way. -// -// A child that has not proven its start is still the owner: the send is admitted against it and -// the adapter holds the message until startup lands, or rejects it with the child's own reason -// when the child dies first. The exit settlement writes that reason into the chat. +// What a send or a Stop needs from the session before the ledger places its row: the +// conversation open. Nothing here needs an owner — a send is accepted into the conversation and +// the delivery loop makes the session ready — so a refusal before acceptance is only one the +// conversation itself makes: a rewind or conversation command in doubt, a cleared conversation, +// or a journal that cannot be opened. import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { @@ -25,45 +10,39 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import type { AgentSessionWireRefusalCode } from '../../../shared/agent-session-wire-refusals' -import { boundJournalStatusText } from '../agent-session-journal/journal-prompt-body-bounds' import { TUI_AGENT_DISPLAY_NAMES } from '../../../shared/tui-agent-display-names' import { ownerRestartFailedOutcome, providerStartupFailureOutcome } from './structured-agent-session-dead-generation-settlement' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' -import type { StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations' -import type { AgentSessionMutationSessionPreparation } from './structured-agent-session-mutation-admission' -import { isResumableStructuredAgentSessionRecord } from './structured-agent-session-resume-eligibility' +import { + AGENT_SESSION_NOT_ATTACHED, + type AgentSessionMutationSessionPreparation +} from './structured-agent-session-mutation-admission' import { rewindRefusal } from './structured-rewind-refusal' /** - * What a refused resume means for the send that ran it. `transient`: the resume met a lease - * someone else is settling, which is not proof it cannot resume — the send runs as the lease - * stands and admission reports it. `failed`: the restart itself failed; the send answers with the - * cause and stops the client's retry loop, and the user may clear the cause and retry. - * `unresumable`: this host has nothing to restart the chat from — no record, or none it can run — - * so only a new chat continues. A new wire code does not compile until it is classified here. + * Whether a refused start leaves the chat anything to start again from. `unresumable`: this host + * has nothing to restart it from — no record, or none it can run — so only a new chat continues. + * A new wire code does not compile until it is classified here. */ -const RESUME_REFUSAL_OUTCOME: Record< - AgentSessionWireRefusalCode, - 'transient' | 'failed' | 'unresumable' -> = { - execution_owner_reconciling: 'transient', - agent_session_conflict: 'transient', - agent_session_checkpoint_stale: 'transient', - agent_session_ownership_unknown: 'transient', - agent_session_operation_capacity: 'transient', - structured_agent_session_unsupported: 'unresumable', - agent_session_operation_conflict: 'failed', - agent_session_operation_expired: 'failed', - agent_session_operation_invalid: 'failed', - agent_session_operation_unknown: 'failed', - agent_session_item_revision_stale: 'failed', - agent_session_already_resolved: 'failed', - agent_session_identity_required: 'unresumable', - agent_session_journal_unreadable: 'failed', - agent_session_owner_restart_failed: 'failed' +const START_REFUSAL_RESUMABLE: Record = { + execution_owner_reconciling: true, + agent_session_conflict: true, + agent_session_checkpoint_stale: true, + agent_session_ownership_unknown: true, + agent_session_operation_capacity: true, + structured_agent_session_unsupported: false, + agent_session_operation_conflict: true, + agent_session_operation_expired: true, + agent_session_operation_invalid: true, + agent_session_operation_unknown: true, + agent_session_item_revision_stale: true, + agent_session_already_resolved: true, + agent_session_identity_required: false, + agent_session_journal_unreadable: true, + agent_session_owner_restart_failed: true } /** Why the record refuses any send right now, whoever owns it; null when a send may run. */ @@ -93,133 +72,41 @@ export function structuredAgentSessionSendBlock( return null } -/** Whether this send is the one that must bring the owner back: no child, a lease handed back - * cleanly, and nothing on the record that refuses the send anyway. Live, unverifiable, still - * reserved, or handed off: that lease is not this send's to replace. */ -export function structuredAgentSessionSendNeedsOwner( - session: StructuredAgentSessionHostSession | undefined, - record: AgentSessionRecord -): boolean { - return ( - session?.hasProviderChild !== true && - isResumableStructuredAgentSessionRecord(record) && - structuredAgentSessionSendBlock(record) === null - ) -} - -type SendPreparationContext = Pick< - StructuredAgentSessionMutationContext, - 'deps' | 'sessions' | 'holds' | 'restoreReadable' -> - -export async function prepareStructuredAgentSessionSend( - context: SendPreparationContext, - envelope: AgentSessionMutationEnvelope, - ledger: 'admit' | 'replay', - record: AgentSessionRecord +/** The conversation a send or a Stop writes to, opened when this host holds it closed. */ +export async function openConversationForWrite( + openConversation: (sessionId: string) => Promise, + envelope: AgentSessionMutationEnvelope ): Promise { - const { sessionId } = record - if (ledger !== 'admit') { - if (!context.sessions.has(sessionId)) { - await context.restoreReadable(sessionId) + try { + if (await openConversation(envelope.sessionId)) { + return { ok: true } } - return { ok: true } - } - if (structuredAgentSessionSendNeedsOwner(context.sessions.get(sessionId), record)) { - const refusal = await restartOwnerForSend(context, envelope, record) - if (refusal) { - return { ok: false, refusal } + return { ok: false, refusal: AGENT_SESSION_NOT_ATTACHED } + } catch (error) { + return { + ok: false, + refusal: { + code: 'agent_session_journal_unreadable', + message: `The conversation could not be opened: ${ + error instanceof Error ? error.message : String(error) + }` + } } } - return { ok: true } } -/** One restart attempt. Answers with the refusal that ends the send, or null when the send goes - * on to admission — after a child, after a transient refusal, or after a fault in the restart's - * own bookkeeping, which is reported and never gates the user's action. */ -async function restartOwnerForSend( - context: SendPreparationContext, - envelope: AgentSessionMutationEnvelope, - record: AgentSessionRecord -): Promise { - const { sessionId } = envelope - let resumed: Awaited> - try { - resumed = await context.holds.ensureProviderChild(sessionId) - } catch (error) { - context.deps.onEventSinkError?.({ sessionId, error }) - return null +/** What the chat says, in its row and on every message it rejects, when the delivery loop could + * not make the session ready. A child that died starting reads as any start that died does. */ +export function structuredAgentSessionStartFailureText( + record: AgentSessionRecord | null, + cause: AgentSessionWireRefusal +): string { + if (cause.ownerVerdict === 'exited') { + return providerStartupFailureOutcome(cause.message) } - const outcome = resumed.ok ? null : RESUME_REFUSAL_OUTCOME[resumed.refusal.code] - if (resumed.ok || outcome === 'transient') { - return null - } - const refusal = ownerRestartFailedRefusal(record, resumed.refusal, outcome !== 'unresumable') - context.deps.onEventSinkError?.({ - sessionId, - error: new Error(`${resumed.refusal.code}: ${resumed.refusal.message}`) + return ownerRestartFailedOutcome({ + agentName: record ? TUI_AGENT_DISPLAY_NAMES[record.provider] : 'The agent', + reason: cause.message, + resumable: START_REFUSAL_RESUMABLE[cause.code] }) - // A restart whose child died starting leaves the row any start that died leaves, so the chat - // reads the same whether the send met that death before admission or after it. - await recordFailedRestart( - context, - envelope, - resumed.refusal.ownerVerdict === 'exited' - ? providerStartupFailureOutcome(resumed.refusal.message) - : refusal.message - ) - return refusal -} - -/** The client stops on the code; the message carries the restart's own cause, and the verdict — - * when the failed attach proved its child gone — tells a client nothing runs for the session. */ -function ownerRestartFailedRefusal( - record: AgentSessionRecord, - cause: AgentSessionWireRefusal, - resumable: boolean -): AgentSessionWireRefusal { - return { - code: 'agent_session_owner_restart_failed', - message: ownerRestartFailedOutcome({ - agentName: TUI_AGENT_DISPLAY_NAMES[record.provider], - reason: cause.message, - resumable - }), - ...(cause.ownerVerdict ? { ownerVerdict: cause.ownerVerdict } : {}) - } -} - -/** The same status row a start that failed leaves in the chat, so the reason outlives the error - * strip. The journal is made readable for it when the failed attach left none behind. Keyed by - * the send, not the clock: a resend of the same id that fails again adds no second row. */ -async function recordFailedRestart( - context: SendPreparationContext, - envelope: AgentSessionMutationEnvelope, - text: string -): Promise { - const { sessionId } = envelope - try { - if (!context.sessions.has(sessionId)) { - await context.restoreReadable(sessionId) - } - const session = context.sessions.get(sessionId) - if (!session) { - return - } - const settlementId = `failed-restart:${envelope.clientOperationId}` - await session.journal.appendLifecycleBatch({ - settlementId, - fence: session.fence, - recovered: true, - mutations: [ - { - kind: 'item', - identity: { provider: 'orca', clientMessageId: settlementId }, - body: { kind: 'status', text: boundJournalStatusText(text) } - } - ] - }) - } catch (error) { - context.deps.onEventSinkError?.({ sessionId, error }) - } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts index e2e2fb81102..211d8684e15 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts @@ -1,12 +1,11 @@ // A session that published and then lost its child before startup (not signed in, say) keeps a // released lease and a chat the user can still type into. The send is the user asking for the -// child back: the host restarts it before admitting the write and delivers against the new owner, -// instead of parking the message behind a lease nothing would ever re-acquire. +// child back: the host accepts the message, and its delivery restarts the child and hands the +// message to the new owner, instead of parking it behind a lease nothing would ever re-acquire. // // A child is published before it has proven its start, and it owns the send from that moment: the -// message is admitted against it and the adapter holds it for the start. When the child exits -// first, the exit settlement rejects the message and writes the cause into the chat, once, and -// the next send is a fresh restart. +// message is handed to it. When the child exits first, the exit settlement rejects the message and +// writes the cause into the chat, once, and the next send is a fresh restart. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -28,6 +27,11 @@ import { } from './structured-agent-session-host-test-data' const CALLER = { callerKey: 'client-1' } + +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => void): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} const EXIT_REASON = 'Claude Code is not signed in. Sign in with the Claude CLI' let root: string @@ -53,7 +57,7 @@ function sendEnvelope( } } -/** A send is admitted against the child it meets, proven or not; the adapter holds the rest. */ +/** A send is accepted at once and handed to the child delivery finds or starts. */ async function send( text: string, fence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 @@ -63,9 +67,11 @@ async function send( expect(sent, JSON.stringify(sent)).toMatchObject({ ok: true, replayed: false, - value: { submission: { dispatchState: 'pending' } } + value: { submission: { dispatchState: 'pending', handoverRecorded: true } } }) - return sent.ok ? sent.value.clientMessageId : '' + const clientMessageId = sent.ok ? sent.value.clientMessageId : '' + await eventually(() => expect(submission(clientMessageId)?.handedOverAt).toBeDefined()) + return clientMessageId } /** The child of the current acquisition, as the adapter would identify it in a lifecycle event. */ @@ -166,8 +172,7 @@ describe('a send into a published session whose child ended before startup', () await send('hello again', releasedFence) - // A send still fenced to the lost owner is admitted once against the restarted one, with no - // stale round trip. + // Accepted at the lost owner's fence and handed to the child delivery started, once. expect(acquire).toHaveBeenCalledTimes(2) expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') expect(dispatch).toHaveBeenCalledOnce() @@ -207,7 +212,7 @@ describe('a send into a published session whose child ended before startup', () // One spawn per user action: nothing restarted it a second time. expect(acquire).toHaveBeenCalledTimes(2) - // Retry is a fresh action: it restarts once and is admitted against the new child. + // Retry is a fresh action: it restarts once and is handed to the new child. await send('signed in now') expect(acquire).toHaveBeenCalledTimes(3) expect(dispatch).toHaveBeenCalledTimes(2) @@ -216,7 +221,7 @@ describe('a send into a published session whose child ended before startup', () }) describe('a send while the child of the first start is still proving itself', () => { - it('is admitted against the starting child, and nothing restarts it', async () => { + it('is handed to the starting child, and nothing restarts it', async () => { await send('hello') expect(dispatch).toHaveBeenCalledOnce() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts index b461d508f42..ff9765d1cee 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts @@ -61,7 +61,7 @@ describe('structured send settlement compatibility wait', () => { it('removes an abandoned wait on transport cancellation', async () => { const settlements = new StructuredAgentSessionSendSettlement(() => journal('pending')) const controller = new AbortController() - const pending = settlements.wait('session-1', 'client-1', controller.signal) + const pending = settlements.wait('session-1', 'client-1', { signal: controller.signal }) controller.abort(new Error('transport closed')) await expect(pending).rejects.toThrow('transport closed') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts index 6150e3412a6..bc265833c42 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts @@ -3,8 +3,12 @@ import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import type { AgentSessionSendResult } from '../../../shared/agent-session-wire' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +/** What a settlement read needs of a journal. */ +type SendSettlementJournal = Pick + type SettledSend = { cursor: AgentJournalCursor value: AgentSessionSendResult @@ -12,8 +16,19 @@ type SettledSend = { type SendSettlement = SettledSend | 'pending' | 'missing' +/** What ends a wait: the provider's answer, or only the host handing the message over. */ +export type SendSettlementPoint = 'answered' | 'handed-over' + +export type SendSettlementWaitOptions = { + signal?: AbortSignal + /** How long to observe; unanswered by then resolves undefined. */ + budgetMs?: number + until?: SendSettlementPoint +} + type SendSettlementWaiter = { clientMessageId: string + until: SendSettlementPoint resolve: (result: SettledSend | undefined) => void reject: (error: Error) => void timer: ReturnType @@ -23,12 +38,15 @@ type SendSettlementWaiter = { // Known legacy clients abandon the RPC after 15s without cancelling its socket dispatch. const SEND_SETTLEMENT_WAIT_TIMEOUT_MS = 30_000 +/** Long enough for a cold provider start; a longer one replays through the same wait. */ +export const STRUCTURED_AGENT_SESSION_START_WAIT_MS = 120_000 const MAX_SEND_SETTLEMENT_WAITERS_PER_SESSION = 64 const MAX_SEND_SETTLEMENT_WAITERS = 1_024 function settledSend( - journal: AgentSessionJournal, + journal: SendSettlementJournal, clientMessageId: string, + until: SendSettlementPoint, submission: AgentJournalSubmission | undefined = journal .submissions() .find((candidate) => candidate.clientMessageId === clientMessageId) @@ -36,9 +54,11 @@ function settledSend( if (!submission) { return 'missing' } - return submission.dispatchState === 'pending' - ? 'pending' - : { cursor: journal.cursor(), value: { clientMessageId, submission } } + const waiting = + until === 'handed-over' + ? isQueuedAgentJournalSubmission(submission) + : submission.dispatchState === 'pending' + return waiting ? 'pending' : { cursor: journal.cursor(), value: { clientMessageId, submission } } } function abortError(signal: AbortSignal): Error { @@ -52,17 +72,19 @@ export class StructuredAgentSessionSendSettlement { private readonly waiters = new Map>() private waiterCount = 0 - constructor(private readonly journalFor: (sessionId: string) => AgentSessionJournal) {} + constructor(private readonly journalFor: (sessionId: string) => SendSettlementJournal) {} wait = ( sessionId: string, clientMessageId: string, - signal?: AbortSignal + options: SendSettlementWaitOptions = {} ): Promise => { + const { signal } = options + const until = options.until ?? 'answered' if (signal?.aborted) { return Promise.reject(abortError(signal)) } - const immediate = settledSend(this.journalFor(sessionId), clientMessageId) + const immediate = settledSend(this.journalFor(sessionId), clientMessageId, until) if (immediate === 'missing') { return Promise.reject(new Error('agent session send disappeared before settlement')) } @@ -79,12 +101,13 @@ export class StructuredAgentSessionSendSettlement { return new Promise((resolve, reject) => { const waiter: SendSettlementWaiter = { clientMessageId, + until, resolve, reject, timer: setTimeout(() => { this.remove(sessionId, waiter) resolve(undefined) - }, SEND_SETTLEMENT_WAIT_TIMEOUT_MS) + }, options.budgetMs ?? SEND_SETTLEMENT_WAIT_TIMEOUT_MS) } waiter.timer.unref?.() const session = existingSession ?? new Set() @@ -106,7 +129,7 @@ export class StructuredAgentSessionSendSettlement { }) } - publish(sessionId: string, journal: AgentSessionJournal): void { + publish(sessionId: string, journal: SendSettlementJournal): void { const waiters = this.waiters.get(sessionId) if (!waiters) { return @@ -118,6 +141,7 @@ export class StructuredAgentSessionSendSettlement { const result = settledSend( journal, waiter.clientMessageId, + waiter.until, submissions.get(waiter.clientMessageId) ) if (result !== 'pending') { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send.test.ts index 638d8a8679f..1a6e5606c1c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send.test.ts @@ -32,6 +32,28 @@ beforeEach(() => { ;({ store, host, dispatch } = hostTestState()) }) +function hostJournal(): AgentSessionJournal { + return ( + host as unknown as { sessions: Map } + ).sessions.get(SESSION)!.journal +} + +/** A send is accepted, then the session's delivery loop hands it over: wait for the handover's + * outcome, or with `handedOver`, only for the handover itself (an admitted send stays pending). */ +async function delivered(clientMessageId: string, options: { handedOver?: true } = {}) { + let submission: ReturnType[number] | undefined + await vi.waitFor(() => { + submission = hostJournal() + .submissions() + .find((entry) => entry.clientMessageId === clientMessageId) + expect(submission?.handedOverAt).toBeDefined() + if (!options.handedOver) { + expect(submission?.dispatchState).not.toBe('pending') + } + }) + return submission! +} + describe('send', () => { it('writes the submission before dispatching and resolves it accepted', async () => { await attach() @@ -43,7 +65,15 @@ describe('send', () => { if (!result.ok) { throw new Error(`expected a send, got ${result.refusal.code}`) } - expect(result.value.submission.dispatchState).toBe('accepted') + // Answered once accepted; the delivery loop hands it over after. + expect(result.value.submission).toMatchObject({ + dispatchState: 'pending', + handoverRecorded: true + }) + expect(result.value.submission.handedOverAt).toBeUndefined() + await expect(delivered(result.value.clientMessageId)).resolves.toMatchObject({ + dispatchState: 'accepted' + }) expect(dispatch).toHaveBeenCalledTimes(1) const page = host.history({ sessionId: SESSION, direction: 'tail' }) expect(page.ok && page.page.items).toHaveLength(1) @@ -75,11 +105,11 @@ describe('send', () => { await attach() dispatch.mockRejectedValueOnce(new Error('socket closed')) const body = hostTestMessage('add a retry') - const result = await host.send(CALLER, { - envelope: envelope('agentSession.send', { body }), - body + const params = { envelope: envelope('agentSession.send', { body }), body } + await host.send(CALLER, params) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'unknown' }) - expect(result).toMatchObject({ ok: true, value: { submission: { dispatchState: 'unknown' } } }) }) it('replays a retried send from the journal without dispatching twice', async () => { @@ -87,6 +117,7 @@ describe('send', () => { const body = hostTestMessage('add a retry') const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId) const retry = await host.send(CALLER, params) expect(retry).toMatchObject({ ok: true, replayed: true }) expect(dispatch).toHaveBeenCalledTimes(1) @@ -98,10 +129,9 @@ describe('send', () => { const body = hostTestMessage('possibly delivered') const params = { envelope: envelope('agentSession.send', { body }), body } - const first = await host.send(CALLER, params) - expect(first).toMatchObject({ - ok: true, - value: { submission: { dispatchState: 'unknown' } } + await host.send(CALLER, params) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'unknown' }) // A thrown adapter call is indistinguishable from a lost reply, so Retry // replays the recorded outcome. @@ -129,6 +159,7 @@ describe('send', () => { const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId) await expect(host.send(CALLER, { ...params, retryUnknown: true })).resolves.toMatchObject({ ok: true, value: { @@ -151,21 +182,18 @@ describe('send', () => { const body = hostTestMessage('never written') const params = { envelope: envelope('agentSession.send', { body }), body } - await expect(host.send(CALLER, params)).resolves.toMatchObject({ - ok: true, - value: { - submission: { dispatchState: 'rejected', reason: 'provider_write_failed: broken pipe' } - } + await host.send(CALLER, params) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'rejected', + reason: 'provider_write_failed: broken pipe' }) // What the user's Retry does with a rejection: a fresh client message id, // which is a first delivery by construction and cannot duplicate the frame // that never left the process. - await expect( - host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) - ).resolves.toMatchObject({ - ok: true, - replayed: false, - value: { submission: { dispatchState: 'accepted' } } + const rotated = { envelope: envelope('agentSession.send', { body }), body } + await expect(host.send(CALLER, rotated)).resolves.toMatchObject({ ok: true, replayed: false }) + await expect(delivered(rotated.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'accepted' }) expect(dispatch).toHaveBeenCalledTimes(2) const state = host.history({ sessionId: SESSION, direction: 'tail' }) @@ -183,10 +211,9 @@ describe('send', () => { const body = hostTestMessage('a message the provider may already hold') const params = { envelope: envelope('agentSession.send', { body }), body } - const first = await host.send(CALLER, params) - expect(first).toMatchObject({ - ok: true, - value: { submission: { dispatchState: 'unknown' } } + await host.send(CALLER, params) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'unknown' }) // No `unknown` is re-delivered under its own id, whatever its reason says, // so Retry replays the recorded outcome instead of writing again. @@ -203,6 +230,7 @@ describe('send', () => { const body = hostTestMessage('settled for good') const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId) const journal = ( host as unknown as { sessions: Map } ).sessions.get(SESSION)!.journal @@ -223,7 +251,7 @@ describe('send', () => { expect(journal.receiptFor(params.envelope.clientOperationId)).not.toBeNull() }) - it('leaves an admitted send pending and writes no dispatch row', async () => { + it('leaves an admitted send pending once handed over', async () => { await attach() dispatch.mockImplementationOnce(async () => ({ state: 'admitted' as const })) const body = hostTestMessage('queued behind a running turn') @@ -233,9 +261,9 @@ describe('send', () => { ok: true, value: { submission: { dispatchState: 'pending', reason: null, resolvedAt: null } } }) - const journal = ( - host as unknown as { sessions: Map } - ).sessions.get(SESSION)!.journal + await delivered(params.envelope.clientOperationId, { handedOver: true }) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) + const journal = hostJournal() expect(journal.pendingSubmissions()).toHaveLength(1) }) @@ -245,6 +273,8 @@ describe('send', () => { const body = hostTestMessage('written, never acknowledged') const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId, { handedOver: true }) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) const journal = ( host as unknown as { sessions: Map } ).sessions.get(SESSION)!.journal @@ -281,6 +311,8 @@ describe('send', () => { const params = { envelope: envelope('agentSession.send', { body }), body } await expect(host.send(CALLER, params)).rejects.toThrow('operation settlement failed') + // The submission was recorded before the ledger write failed, so it is still delivered. + await delivered(params.envelope.clientOperationId) await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: true, @@ -333,9 +365,8 @@ describe('send', () => { await expect(host.send(CALLER, params)).rejects.toThrow('operation settlement failed') settlement.mockRestore() - const journal = ( - host as unknown as { sessions: Map } - ).sessions.get(SESSION)!.journal + await delivered(params.envelope.clientOperationId) + const journal = hostJournal() await journal.rollEpoch('schema_unreadable', store.getRecord(SESSION)?.lease.runtimeFence ?? 1) expect(journal.submissions()).toHaveLength(0) @@ -362,6 +393,7 @@ describe('send', () => { const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId) expect(dispatch).toHaveBeenCalledTimes(1) await store.recordOperationOutcome({ callerKey: CALLER.callerKey, @@ -393,6 +425,8 @@ describe('send', () => { const body = hostTestMessage('written, then the child died') const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId, { handedOver: true }) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) const journal = ( host as unknown as { sessions: Map } ).sessions.get(SESSION)!.journal @@ -411,21 +445,25 @@ describe('send', () => { it('advances an explicit retry after a ledger-unknown send is reconciled in the journal', async () => { await attach() - const journal = ( - host as unknown as { sessions: Map } - ).sessions.get(SESSION)!.journal - vi.spyOn(journal, 'resolveDispatch').mockRejectedValueOnce(new Error('journal resolve failed')) + const journal = hostJournal() + const resolve = journal.resolveDispatch.bind(journal) + // The handover row lands; the provider's answer, written after the adapter took the + // message, does not. + vi.spyOn(journal, 'resolveDispatch') + .mockImplementationOnce(resolve) + .mockRejectedValueOnce(new Error('journal resolve failed')) const body = hostTestMessage('possibly delivered before persistence failed') const params = { envelope: envelope('agentSession.send', { body }), body } - await expect(host.send(CALLER, params)).rejects.toThrow('journal resolve failed') - expect(journal.submissions()).toMatchObject([ - { clientMessageId: params.envelope.clientOperationId, dispatchState: 'unknown' } - ]) + await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true }) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'unknown' + }) + // Acceptance is what the ledger answers for; delivery is the journal's to say. expect( store.listOperationRows().find((row) => row.operationId === params.envelope.clientOperationId) ?.outcome - ).toEqual({ status: 'unknown' }) + ).toMatchObject({ status: 'succeeded' }) expect(dispatch).toHaveBeenCalledTimes(1) await journal.markPendingSubmissionsUnknown(store.getRecord(SESSION)?.lease.runtimeFence ?? 1) @@ -446,7 +484,7 @@ describe('send', () => { expect(journal.submissions()).toHaveLength(1) }) - it('admits a send fenced to another generation, and replays it by id once the fence catches up', async () => { + it('admits a send fenced to another generation, delivers it once, and replays it by id', async () => { const record = await attach() const body = hostTestMessage('add a retry') const params = { @@ -457,10 +495,9 @@ describe('send', () => { ), body } - expect(await host.send(CALLER, params)).toMatchObject({ - ok: true, - replayed: false, - value: { submission: { dispatchState: 'accepted' } } + expect(await host.send(CALLER, params)).toMatchObject({ ok: true, replayed: false }) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'accepted' }) const retry = { ...params, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts index f0dd60965ee..711e38b29b8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts @@ -314,6 +314,8 @@ describe('settled attach retry', () => { } const first = await host.send(CALLER, unknownParams) expect(first).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + // Handed over before the host dies: that is what makes the restart's answer doubt. + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) await host.flushAllStreamedEvents() store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) @@ -343,7 +345,7 @@ describe('settled attach retry', () => { if (!sent.ok) { throw new Error(`unexpected restored send refusal: ${sent.refusal.message}`) } - expect(dispatch).toHaveBeenCalledTimes(2) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(2)) const restoredHistory = host.history({ sessionId: SESSION, direction: 'tail' }) if (!restoredHistory.ok) { throw new Error(`unexpected restored history reset: ${restoredHistory.reset}`) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts new file mode 100644 index 00000000000..00594966101 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts @@ -0,0 +1,59 @@ +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' +import { boundJournalStatusText } from '../agent-session-journal/journal-prompt-body-bounds' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' + +/** + * The one row a start that failed leaves in the chat, whoever saw it fail: an error row, so the + * reason outlives any error strip. Keyed by the start — the child's generation, or the oldest + * message it was for when no child was ever published — so a second report of the same failure + * revises the row instead of adding one. + */ +export function structuredAgentSessionStartFailureRow( + startKey: string, + text: string +): JournalLifecycleMutationInput { + return { + kind: 'item', + identity: { provider: 'orca', clientMessageId: `start-failure:${startKey}` }, + body: { kind: 'status', text: boundJournalStatusText(text), tone: 'error' } + } +} + +/** + * A start the delivery loop needed and did not get: the start's row, and every queued message + * rejected with the same words. Writes nothing when nothing is still queued: a start whose + * messages Stop withdrew did not fail anyone. + */ +export async function recordStructuredAgentSessionStartFailure( + session: Pick & { fence: number }, + failure: { startKey: string | null; text: string } +): Promise { + const oldest = oldestQueuedSubmission(session) + if (!oldest) { + return + } + const startKey = failure.startKey ?? oldest.clientMessageId + await session.journal.appendLifecycleBatch({ + settlementId: `start-failure:${startKey}`, + fence: session.fence, + recovered: true, + mutations: [structuredAgentSessionStartFailureRow(startKey, failure.text)] + }) + await session.journal.rejectQueuedSubmissions(session.fence, failure.text) +} + +export function oldestQueuedSubmission( + session: Pick +): ReturnType[number] | undefined { + let oldest: ReturnType + for (const submission of session.journal.submissions()) { + if ( + isQueuedAgentJournalSubmission(submission) && + (oldest === undefined || (submission.acceptedSequence ?? 0) < (oldest.acceptedSequence ?? 0)) + ) { + oldest = submission + } + } + return oldest +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts index 0fd544ec9ee..88774c63bed 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts @@ -19,10 +19,9 @@ function startedSession(): StructuredAgentSessionUnexpectedExitSession & { journal: { appendLifecycleBatch: ReturnType } } { return { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), // Nothing ran: the start failed before any response or acknowledged prompt. snapshot: () => ({ items: [] }), appendLifecycleBatch: vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })), @@ -86,7 +85,9 @@ describe('a provider that ends before it finished starting', () => { expect.objectContaining({ mutations: [ expect.objectContaining({ - body: { kind: 'status', text: providerStartupFailureOutcome(REASON) } + // The same row the delivery loop writes for a failed start: an error, keyed by it. + identity: { provider: 'orca', clientMessageId: `start-failure:${GENERATION}` }, + body: { kind: 'status', text: providerStartupFailureOutcome(REASON), tone: 'error' } }) ] }) @@ -104,7 +105,10 @@ describe('a provider that ends before it finished starting', () => { }) it("reads a start that failed off the host's own phase when the provider omits the flag", async () => { - const session = { ...startedSession(), providerChildPhase: 'starting' as const } + const session = { + ...startedSession(), + child: { generation: GENERATION, fence: 7, phase: 'starting' as const } + } const ticket = await settleUnexpectedStructuredAgentSessionExit(contextFor(session), ended) @@ -113,7 +117,9 @@ describe('a provider that ends before it finished starting', () => { expect.objectContaining({ mutations: [ expect.objectContaining({ - body: { kind: 'status', text: providerStartupFailureOutcome(REASON) } + // The same row the delivery loop writes for a failed start: an error, keyed by it. + identity: { provider: 'orca', clientMessageId: `start-failure:${GENERATION}` }, + body: { kind: 'status', text: providerStartupFailureOutcome(REASON), tone: 'error' } }) ] }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts index e4fa6ebed1b..c130aaf9d59 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts @@ -2,17 +2,11 @@ import type { AgentSessionJournal } from '../agent-session-journal/journal-store export function indexedStatusFeedSession(session: { journal: AgentSessionJournal - hasProviderChild?: boolean - providerChildPhase?: 'starting' | 'ready' - fence?: number + child?: { phase: 'starting' | 'ready' } | null }) { return { journal: session.journal, - fence: session.fence ?? 1, - ...(session.hasProviderChild !== undefined - ? { hasProviderChild: session.hasProviderChild } - : {}), - ...(session.providerChildPhase ? { providerChildPhase: session.providerChildPhase } : {}), + ...(session.child !== undefined ? { child: session.child } : {}), params: { location: { executionHostId: 'local' as const, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts index 65c7f54fb30..e5da2eece97 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { agentSessionRecordFixture } from '../../../shared/agent-session-record.test-fixture' import type { AgentSessionBackgroundTask, AgentSessionStatusEvent, @@ -33,6 +34,8 @@ const USER_IDENTITY = { ordinal: 1 } as const +type Indexed = Parameters[0] + let root: string const journals = createTrackedJournalOpener() @@ -82,7 +85,8 @@ function feedFor( } } } as unknown as ReadonlyMap>, - getRecord: () => record as AgentSessionRecord | null, + // A partial record still has a lease: the feed reads the conversation's fence off it. + getRecord: () => (record ? { ...agentSessionRecordFixture(), ...record } : null), now: () => (now += 1) }) const events: AgentSessionStatusEvent[] = [] @@ -93,17 +97,17 @@ function feedFor( describe('StructuredAgentSessionStatusFeed', () => { it('projects whether the owned child has proven its start, and nothing once it is not owned', async () => { const journal = await openJournal() - const session = { journal, hasProviderChild: true, providerChildPhase: 'starting' as const } + const session = { journal, child: { phase: 'starting' as const } } const sessions = new Map[0]>([[SESSION, session]]) const { feed, events, dispose } = feedFor(sessions) expect(events.at(-1)).toMatchObject({ type: 'snapshot', sessions: [{ hostExecutionOwned: true, hostExecutionPhase: 'starting' }] }) - sessions.set(SESSION, { ...session, providerChildPhase: 'ready' }) + sessions.set(SESSION, { ...session, child: { phase: 'ready' } }) feed.publish(SESSION, journal) expect(events.at(-1)).toMatchObject({ session: { hostExecutionPhase: 'ready' } }) - sessions.set(SESSION, { ...session, hasProviderChild: false }) + sessions.set(SESSION, { ...session, child: null }) feed.publish(SESSION, journal) expect(events.at(-1)).not.toMatchObject({ session: { hostExecutionPhase: expect.any(String) } }) dispose() @@ -111,7 +115,7 @@ describe('StructuredAgentSessionStatusFeed', () => { it('publishes provider ownership transitions without changing journal time', async () => { const journal = await openJournal() - const sessions = new Map([[SESSION, { journal, hasProviderChild: true }]]) + const sessions = new Map([[SESSION, { journal, child: { phase: 'ready' } }]]) const { feed, events, dispose } = feedFor(sessions) events.length = 0 await journal.appendItem( @@ -130,7 +134,7 @@ describe('StructuredAgentSessionStatusFeed', () => { throw new Error('status publication missing') } const journalTime = firstStatus.session.updatedAt - sessions.get(SESSION)!.hasProviderChild = false + sessions.get(SESSION)!.child = null feed.publish(SESSION, journal) expect(events.at(-1)).toEqual({ type: 'status', @@ -167,8 +171,10 @@ describe('StructuredAgentSessionStatusFeed', () => { it('stops projecting an old-host unknown submission after the owner fence advances', async () => { const journal = await openJournal() - const session = { journal, fence: 1 } - const { feed, events } = feedFor(new Map([[SESSION, session]])) + // The conversation's fence is the record's: a child's end moves it. + const lease = agentSessionRecordFixture().lease + const record = agentSessionRecordFixture({ ...lease, runtimeFence: 1 }) + const { feed, events } = feedFor(new Map([[SESSION, { journal }]]), record) await journal.appendSubmission({ clientMessageId: 'old-host', payloadFingerprint: 'fp', @@ -183,7 +189,7 @@ describe('StructuredAgentSessionStatusFeed', () => { }) feed.publish(SESSION) expect(events.at(-1)).toMatchObject({ session: { status: 'working' } }) - session.fence = 2 + record.lease.runtimeFence = 2 feed.publish(SESSION) expect(events.at(-1)).toMatchObject({ session: { status: 'idle' } }) }) @@ -793,7 +799,7 @@ describe('the status sink sees the roster the broadcast cache deliberately lacks it('receives every change once, ownership revocation, and the forget edge', async () => { const journal = await openJournal() - const sessions = new Map([[SESSION, { journal, hasProviderChild: true }]]) + const sessions = new Map([[SESSION, { journal, child: { phase: 'ready' } }]]) const { sink, published, forgotten } = sinkFor() const { feed } = feedFor(sessions, null, undefined, undefined, sink) await journal.appendItem( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts index eef1c0e06f9..6e572f0366d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts @@ -42,9 +42,7 @@ export type StructuredAgentSessionStatusSubscriber = { type StatusFeedSession = { journal: AgentSessionJournal params: { location: AgentSessionRecord['location']; provider: AgentSessionRecord['provider'] } - hasProviderChild?: boolean - providerChildPhase?: StructuredAgentSessionProviderChildPhase - fence?: number + child?: { phase: StructuredAgentSessionProviderChildPhase } | null } export type StructuredAgentSessionStatusFeedDeps = { @@ -239,7 +237,9 @@ export class StructuredAgentSessionStatusFeed { // An unreadable journal projects as "no turn": the chat itself shows the reset. const cursor = journal.cursor() const readOnly = journal.isReadOnly - const fence = session.fence + const record = this.deps.getRecord(sessionId) + // The conversation's fence, which a child's end moves: its unanswered sends stop counting. + const fence = record?.lease.runtimeFence let projection = this.journalProjections.get(journal) if ( !projection || @@ -263,7 +263,6 @@ export class StructuredAgentSessionStatusFeed { } this.journalProjections.set(journal, projection) } - const record = this.deps.getRecord(sessionId) const providerSession = structuredAgentSessionProviderSessionMetadata(record) // The journal has no model: the record's acknowledged options are where a mid-session // switch lands, so the row follows whichever is in force. @@ -278,13 +277,8 @@ export class StructuredAgentSessionStatusFeed { sessionId, workspaceId: session.params.location.workspaceId, agent: session.params.provider, - ...(session.hasProviderChild - ? { - hostExecutionOwned: true as const, - ...(session.providerChildPhase - ? { hostExecutionPhase: session.providerChildPhase } - : {}) - } + ...(session.child + ? { hostExecutionOwned: true as const, hostExecutionPhase: session.child.phase } : {}), ...projection.summary, ...(record?.rewind?.phase === 'prepared' || record?.rewind?.phase === 'provider-succeeded' diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-subagent-recency.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-subagent-recency.test.ts index cf908932667..4053988233f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-subagent-recency.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-subagent-recency.test.ts @@ -63,7 +63,9 @@ async function openSession() { const roster: { tasks: AgentSessionBackgroundTask[] } = { tasks: [] } const server = new AgentHookServer() const feed = new StructuredAgentSessionStatusFeed({ - sessions: new Map([[SESSION, indexedStatusFeedSession({ journal, hasProviderChild: true })]]), + sessions: new Map([ + [SESSION, indexedStatusFeedSession({ journal, child: { phase: 'ready' } })] + ]), getRecord: () => null, now: () => 1, readBackgroundTasks: () => ({ state: 'monitoring', tasks: roster.tasks }), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts index 25c95b48dbe..cc145ad7995 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts @@ -295,6 +295,8 @@ describe('a chat that closes', () => { if (!result.ok) { throw new Error('send was refused') } + // Handed over first: a message still queued at close is rejected as never sent instead. + await vi.waitFor(() => expect(dispatch).toHaveBeenCalled()) const settlement = host.waitForSendSettlement(SESSION, result.value.clientMessageId) await host.close(SESSION) @@ -319,12 +321,10 @@ describe('a chat that closes', () => { }) .mockImplementation(closeJournal) - await expect(host.close(SESSION)).rejects.toMatchObject({ - step: 'forget-session', - cause: expect.objectContaining({ message: 'journal close result lost' }) - }) + // The child stopped and its lease went back; only the conversation's close is left to retry. + await expect(host.close(SESSION)).rejects.toThrow('journal close result lost') expect(host.hasSession(SESSION)).toBe(true) - expect(host['sessions'].get(SESSION)?.hasProviderChild).toBe(false) + expect(host['sessions'].get(SESSION)?.child).toBeNull() expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'released', ownerProcess: null @@ -362,7 +362,7 @@ describe('a chat that closes', () => { await expect(host.close(SESSION)).rejects.toMatchObject({ step: 'drain-published' }) // The child is proven gone, but the wind-down it owes is not done: nothing settled, no release. - expect(session!.hasProviderChild).toBe(false) + expect(session!.child).toBeNull() expect(store.getRecord(SESSION)?.lease.claimStatus).not.toBe('released') await expect(host.close(SESSION)).resolves.toBeUndefined() @@ -492,6 +492,12 @@ describe('a session evicted and opened again', () => { }) }) +function submissionState(clientMessageId: string): string | undefined { + return host + .journalSnapshot(SESSION) + .submissions.find((entry) => entry.clientMessageId === clientMessageId)?.dispatchState +} + describe('an unexpected provider exit', () => { it('publishes terminal settlement to a waiting older client', async () => { await attach() @@ -508,6 +514,8 @@ describe('an unexpected provider exit', () => { if (!result.ok) { throw new Error('send was refused') } + // Accepted first; the exit must meet a message the provider was handed. + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledOnce()) const settlement = host.waitForSendSettlement(SESSION, result.value.clientMessageId) const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 @@ -558,12 +566,14 @@ describe('an unexpected provider exit', () => { await host.hold(SESSION, SURFACE) dispatch.mockRejectedValueOnce(new Error('provider delivery became unknown')) const unknownBody = hostTestMessage('message with unknown delivery') + const unknownEnvelope = envelope('agentSession.send', { body: unknownBody }) await expect( - host.send(CALLER, { - envelope: envelope('agentSession.send', { body: unknownBody }), - body: unknownBody - }) - ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'unknown' } } }) + host.send(CALLER, { envelope: unknownEnvelope, body: unknownBody }) + ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + // Accepted, then handed over by the delivery loop, where the thrown dispatch becomes doubt. + await vi.waitFor(() => + expect(submissionState(unknownEnvelope.clientOperationId)).toBe('unknown') + ) const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 await host.handleAdapterEvent({ @@ -592,9 +602,12 @@ describe('an unexpected provider exit', () => { providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-next', ordinal: 1 } }) const body = hostTestMessage('a distinct next message') - await expect( - host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) - ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } }) + const nextEnvelope = envelope('agentSession.send', { body }) + await expect(host.send(CALLER, { envelope: nextEnvelope, body })).resolves.toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + await vi.waitFor(() => expect(submissionState(nextEnvelope.clientOperationId)).toBe('accepted')) expect(dispatch).toHaveBeenCalledTimes(2) }) @@ -659,8 +672,11 @@ describe('an unexpected provider exit', () => { } await expect(host.send(CALLER, unknownParams)).resolves.toMatchObject({ ok: true, - value: { submission: { dispatchState: 'unknown' } } + value: { submission: { dispatchState: 'pending' } } }) + await vi.waitFor(() => + expect(submissionState(unknownParams.envelope.clientOperationId)).toBe('unknown') + ) const runtimeState = ( host as unknown as { runtimeState: { lifecycleBarrier: () => Promise<{ ok: false; error: Error }> } @@ -704,9 +720,12 @@ describe('an unexpected provider exit', () => { providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-next', ordinal: 1 } }) const body = hostTestMessage('a distinct next message after failed-barrier recovery') - await expect( - host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) - ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } }) + const nextEnvelope = envelope('agentSession.send', { body }) + await expect(host.send(CALLER, { envelope: nextEnvelope, body })).resolves.toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + await vi.waitFor(() => expect(submissionState(nextEnvelope.clientOperationId)).toBe('accepted')) expect(dispatch).toHaveBeenCalledTimes(2) }) @@ -733,10 +752,10 @@ describe('an unexpected provider exit', () => { } ).sessions.get(SESSION) expect(session).toBeDefined() - // The dead generation's handle never accepts its settlement. - vi.spyOn(session!.journal, 'appendLifecycleBatch').mockRejectedValue( - new Error('settlement still unavailable') - ) + // The conversation's one handle refuses every write of the exit's settlement. + const refusing = vi + .spyOn(session!.journal, 'appendLifecycleBatch') + .mockRejectedValue(new Error('settlement still unavailable')) const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 await host.handleAdapterEvent({ @@ -756,15 +775,20 @@ describe('an unexpected provider exit', () => { runtimeFence: exitedFence + 1, deathEvidence: { kind: 'exit-observed', detail: 'provider exited', observedAt: NOW } }) + // Nothing retries the settlement; the journal writes again, and the next acquire re-derives it. + refusing.mockRestore() dispatch.mockResolvedValueOnce({ state: 'accepted', providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-next', ordinal: 1 } }) const body = hostTestMessage('sent after a settlement that never landed') - await expect( - host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) - ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } }) + const sentEnvelope = envelope('agentSession.send', { body }) + await expect(host.send(CALLER, { envelope: sentEnvelope, body })).resolves.toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + await vi.waitFor(() => expect(submissionState(sentEnvelope.clientOperationId)).toBe('accepted')) expect(acquire).toHaveBeenCalledTimes(2) // The new child's acquire settled the turn from the release's evidence: ended at the exit's // receipt, with the exit's own reason in the row. @@ -790,7 +814,7 @@ describe('a quit over an eviction that never got its retry', () => { failNextDrain() await expect(host.close(SESSION)).rejects.toMatchObject({ step: 'drain-published' }) - expect(host['sessions'].get(SESSION)?.hasProviderChild).toBe(false) + expect(host['sessions'].get(SESSION)?.child).toBeNull() expect(store.getRecord(SESSION)?.lease.claimStatus).not.toBe('released') await host.flushAllStreamedEvents() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts index 794b4fe8df9..5eac140868f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts @@ -25,11 +25,7 @@ import type { } from './structured-agent-session-adapter' import { providerStartupFailureRejection } from './structured-agent-session-dead-generation-settlement' import { validatePendingPrompt } from './structured-agent-session-prompt-state' -import { withTimeout } from '../../../shared/promise-timeout-fallback' -import { - AgentSessionPreDispatchError, - AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS -} from './structured-agent-session-operation-settlement' +import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' export { performSetOption } from './structured-agent-session-turns-options' export { performPrompt } from './structured-agent-session-turns-prompt' @@ -47,8 +43,6 @@ export type AgentSessionTurnContext = { publish: () => void /** Drains provider lifecycle already accepted by the execution host. */ flushStreamedEvents: () => Promise - /** Re-derives authorization after submission persistence, immediately before provider dispatch. */ - beforeDispatch?: () => void /** What the host holds about the child this dispatch is for, read at the moment it is needed. */ providerChildPhase?: () => StructuredAgentSessionProviderChildPhase | undefined now: () => number @@ -67,10 +61,10 @@ function invalid(message: string): { ok: false; refusal: AgentSessionWireRefusal * accepted nothing (input is written only after it initializes), so a dispatch it could not * take is provably unwritten and is rejected with the cause the adapter gave. */ async function dispatchSafely( - ctx: AgentSessionTurnContext, + ctx: AgentSessionHandoverContext, clientMessageId: string, body: AgentJournalMessageItem, - requestedAt: number | undefined + requestedAt: number ): Promise { try { return await ctx.adapter.dispatch({ @@ -78,13 +72,9 @@ async function dispatchSafely( clientMessageId, body, fence: ctx.fence, - ...(ctx.beforeDispatch ? { beforeDispatch: async () => ctx.beforeDispatch?.() } : {}), - ...(requestedAt === undefined ? {} : { requestedAt }) + requestedAt }) } catch (error) { - if (error instanceof AgentSessionPreDispatchError) { - throw error - } if (ctx.providerChildPhase?.() === 'starting') { return { state: 'rejected', reason: providerStartupFailureRejection(error) } } @@ -111,6 +101,8 @@ async function appendStatus( * the whole content of the word — and one message reached the model five times * when this was a judgement call instead of an invariant. A distinct send after * a terminal rejection uses a fresh id, which is a first delivery. + * + * Accepting only records the message; the session's delivery loop hands it over. */ export async function performSend( ctx: AgentSessionTurnContext, @@ -133,79 +125,10 @@ export async function performSend( } } try { - await ctx.journal.appendSubmission({ ...input, fence: ctx.fence }) + await ctx.journal.appendSubmission({ ...input, fence: ctx.fence, handoverRecorded: true }) } catch { return invalid('The message could not be recorded and was not sent.') } - - // The row just written is the send's instant on the host clock; the turn this - // dispatch opens records it so the live counter never re-anchors at turn-open. - const requestedAt = ctx.journal - .submissions() - .find((entry) => entry.clientMessageId === input.clientMessageId)?.submittedAt - const outcome = await dispatchSafely(ctx, input.clientMessageId, input.body, requestedAt).catch( - async (error: unknown) => { - if (error instanceof AgentSessionPreDispatchError) { - const recorded = await withTimeout( - ctx.journal - .resolveDispatch({ - clientMessageId: input.clientMessageId, - state: 'rejected', - reason: error.message, - fence: ctx.fence - }) - .then(() => true), - AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS, - false - ) - if (!recorded) { - console.warn('[structured-agent-session] pre-dispatch refusal persistence failed') - } - } - throw error - } - ) - // An admission needs no dispatch row: the submission is already pending. - if (outcome.state === 'admitted') { - return { - ok: true, - value: { - clientMessageId: input.clientMessageId, - submission: requireSubmission(ctx, input.clientMessageId) - } - } - } - try { - await ctx.journal.resolveDispatch( - outcome.state === 'accepted' - ? { - clientMessageId: input.clientMessageId, - state: 'accepted', - providerIdentity: outcome.providerIdentity, - fence: ctx.fence - } - : { - clientMessageId: input.clientMessageId, - state: outcome.state, - reason: outcome.reason, - fence: ctx.fence - } - ) - } catch (error) { - // A failed resolution must not strand a pending row; an unknown result is - // explicitly replayable. - try { - await ctx.journal.resolveDispatch({ - clientMessageId: input.clientMessageId, - state: 'unknown', - reason: DISPATCH_DOUBT_PERSISTENCE_FAILED, - fence: ctx.fence - }) - } catch { - // Nothing further to record; the pending row is settled on the next attach. - } - throw error - } return { ok: true, value: { @@ -215,6 +138,66 @@ export async function performSend( } } +export type AgentSessionHandoverContext = Pick< + AgentSessionTurnContext, + 'sessionId' | 'journal' | 'fence' | 'adapter' | 'providerChildPhase' +> + +/** + * Hands one queued submission to the provider. The `dispatch{pending}` row goes first: a crash + * after it leaves a message in doubt, never one that reads as queued and so provably unwritten. + */ +export async function handOverSubmission( + ctx: AgentSessionHandoverContext, + submission: AgentJournalSubmission +): Promise { + const { clientMessageId } = submission + const body = ctx.journal.itemBody(agentJournalSubmissionKey(clientMessageId)) + if (body?.kind !== 'message') { + await ctx.journal.resolveDispatch({ + clientMessageId, + state: 'rejected', + reason: 'The message could not be read back and was not sent.', + fence: ctx.fence + }) + return + } + await ctx.journal.resolveDispatch({ clientMessageId, state: 'pending', fence: ctx.fence }) + // The row written at acceptance is the send's instant on the host clock; the turn this + // dispatch opens records it so the live counter never re-anchors at turn-open. + const outcome = await dispatchSafely(ctx, clientMessageId, body, submission.submittedAt) + // An admission needs no dispatch row: the submission is already pending. + if (outcome.state === 'admitted') { + return + } + try { + await ctx.journal.resolveDispatch( + outcome.state === 'accepted' + ? { + clientMessageId, + state: 'accepted', + providerIdentity: outcome.providerIdentity, + fence: ctx.fence + } + : { clientMessageId, state: outcome.state, reason: outcome.reason, fence: ctx.fence } + ) + } catch (error) { + // A failed resolution must not strand a pending row; an unknown result is + // explicitly replayable. + try { + await ctx.journal.resolveDispatch({ + clientMessageId, + state: 'unknown', + reason: DISPATCH_DOUBT_PERSISTENCE_FAILED, + fence: ctx.fence + }) + } catch { + // Nothing further to record; the pending row is settled on the next open. + } + throw error + } +} + function requireSubmission( ctx: AgentSessionTurnContext, clientMessageId: string diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unanswered-dispatch-release.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unanswered-dispatch-release.test.ts index 79f21a349b4..952fa70657d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unanswered-dispatch-release.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unanswered-dispatch-release.test.ts @@ -32,10 +32,12 @@ function contextWith(submissions: AgentJournalSubmission[]) { return { epoch: 'e', sequence: 1 } }) } - // The mutation reads only `journal.submissions`, `journal.resolveDispatch` and `fence`. - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all three are supplied here; the rest of the host session is unreachable from this mutation. - const session = { journal, fence: FENCE } as unknown as ReleaseSession - const context: ReleaseContext = { sessions: new Map([['s-1', session]]) } + // The mutation reads only `journal.submissions`, `journal.resolveDispatch` and the record fence. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all three are supplied here; the rest of the session and the record is unreachable from this mutation. + const context = { + sessions: new Map([['s-1', { journal }]]), + deps: { store: { getRecord: () => ({ lease: { runtimeFence: FENCE } }) } } + } as unknown as ReleaseContext return { context, resolved, journal } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index 98901380aa3..6419e1cfb1a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -34,11 +34,18 @@ function recoveryContext(input: { handoffStage?: AgentSessionRecord['lease']['handoffStage'] resumeCapable?: boolean }) { - const session = { - hasProviderChild: false, - fence: 8, - acquisitionGeneration: input.generation ?? GENERATION - } as StructuredAgentSessionHostSession + const session: Pick = { + child: null, + lastEndedChild: { + generation: input.generation ?? GENERATION, + fence: 7, + cause: 'exit', + reason: null, + duringStartup: false, + rootGone: true, + endedAt: { epoch: 'epoch-1', sequence: 0 } + } + } const record = { lease: { runtimeFence: 8, @@ -121,11 +128,11 @@ describe('provider-exit recovery tickets', () => { .fn() .mockRejectedValueOnce(new Error('journal unavailable')) .mockResolvedValue({ epoch: 'epoch-1', sequence: 2 }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the exit reads only the child record and these journal methods; the rest of the session is unreachable from it. const session = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), snapshot: () => ({ items: [lifecycleItem('turn-1', 1, { state: 'running', startedAt: 1_000 })] }), @@ -193,11 +200,11 @@ describe('provider-exit recovery tickets', () => { lifecycleItem('turn-1', 1, { state: 'completed', startedAt: 10, completedAt: 20 }), lifecycleItem('turn-2', 2, { state: 'running', startedAt: 30 }) ] + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the exit reads only the child record and these journal methods; the rest of the session is unreachable from it. const session = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), snapshot: () => ({ items }), appendLifecycleBatch, markPendingSubmissionsUnknown: vi.fn(async () => []) @@ -242,7 +249,7 @@ describe('provider-exit recovery tickets', () => { 7, 'provider_exited_before_acknowledgement' ) - expect(session.hasProviderChild).toBe(false) + expect(session.child).toBeNull() // The running row is revised to interrupted at exit receipt, never tombstoned. expect(appendLifecycleBatch).toHaveBeenCalledExactlyOnceWith({ settlementId: `dead-generation:provider-exit:${SESSION}:7:${GENERATION}`, @@ -299,10 +306,9 @@ describe('provider-exit recovery tickets', () => { sequence: 3 })) const session: StructuredAgentSessionUnexpectedExitSession = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), snapshot: () => ({ items }), appendLifecycleBatch, markPendingSubmissionsUnknown: vi.fn(async () => []), @@ -356,10 +362,9 @@ describe('provider-exit recovery tickets', () => { it('settles a submission the dead child never acknowledged', async () => { const markPendingSubmissionsUnknown = vi.fn(async () => ['client-1']) const session: StructuredAgentSessionUnexpectedExitSession = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), snapshot: () => ({ items: [] }), appendLifecycleBatch: vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })), markPendingSubmissionsUnknown, @@ -404,10 +409,9 @@ describe('provider-exit recovery tickets', () => { it('releases without offering a restart while terminal settlement is failing', async () => { const session: StructuredAgentSessionUnexpectedExitSession = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), markPendingSubmissionsUnknown: vi.fn(async () => []), rejectPendingSubmissions: vi.fn(async () => []), snapshot: () => ({ @@ -442,8 +446,7 @@ describe('provider-exit recovery tickets', () => { const result = await settleUnexpectedStructuredAgentSessionExit(context, event) expect(result).toBeNull() - expect(session.hasProviderChild).toBe(false) - expect(session.fence).toBe(8) + expect(session.child).toBeNull() expect(publishFence).toHaveBeenCalledTimes(1) expect(release).toHaveBeenCalledTimes(2) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts index f21de56f2ba..541daec0153 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts @@ -1,8 +1,10 @@ -import type { - StructuredAgentSessionEndedEvent, - StructuredAgentSessionProviderChildPhase -} from './structured-agent-session-adapter' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionEndedEvent } from './structured-agent-session-adapter' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { + endProviderChild, + failedProviderChildStart +} from './structured-agent-session-provider-child' import { releaseStoredStructuredAgentSessionOwnerAfterUnexpectedExit, type StructuredAgentSessionLeaseStore @@ -28,13 +30,10 @@ export type StructuredAgentSessionRecoveryTicket = { stableSettlementId: string } -export type StructuredAgentSessionUnexpectedExitSession = { - journal: DeadGenerationJournal - hasProviderChild: boolean - fence: number - acquisitionGeneration: string | null - providerChildPhase?: StructuredAgentSessionProviderChildPhase -} +export type StructuredAgentSessionUnexpectedExitSession = Pick< + StructuredAgentSessionHostSession, + 'child' | 'lastEndedChild' +> & { journal: DeadGenerationJournal & Pick } export type StructuredAgentSessionUnexpectedExitContext< TSession extends StructuredAgentSessionUnexpectedExitSession = StructuredAgentSessionHostSession @@ -64,24 +63,37 @@ export async function settleUnexpectedStructuredAgentSessionExit< const observedAt = event.observedAt ?? context.now() return context.serialize(unexpectedEvent.sessionId, async () => { const session = context.sessions.get(unexpectedEvent.sessionId) + const child = session?.child if ( - !session?.hasProviderChild || - session.fence !== unexpectedEvent.fence || - session.acquisitionGeneration !== unexpectedEvent.acquisitionGeneration + !session || + !child || + child.fence !== unexpectedEvent.fence || + child.generation !== unexpectedEvent.acquisitionGeneration ) { return null } - const record = context.store.getRecord(unexpectedEvent.sessionId) - if (!record || record.lease.handoffStage !== null) { - // An acquisition or recovery already owns this lease's transition. - session.hasProviderChild = false - context.publishStatus?.(unexpectedEvent.sessionId) - return null - } // The host's own phase decides, so a provider that omits the flag still gets a start that // failed told as one: the row says so, and nothing resumes into the same failure. const exitedDuringStartup = - unexpectedEvent.startupUnproven === true || session.providerChildPhase === 'starting' + unexpectedEvent.startupUnproven === true || child.phase === 'starting' + const endChild = (): void => { + endProviderChild(session, { + generation: child.generation, + fence: child.fence, + cause: 'exit', + reason: unexpectedEvent.reason, + duringStartup: exitedDuringStartup, + // The adapter publishes an exit only once it saw the root go, first-hand or proven. + rootGone: true + }) + context.publishStatus?.(unexpectedEvent.sessionId) + } + const record = context.store.getRecord(unexpectedEvent.sessionId) + if (!record || record.lease.handoffStage !== null) { + // An acquisition or recovery already owns this lease's transition. + endChild() + return null + } let settlementFailed = false const stableSettlementId = providerExitSettlementId(unexpectedEvent) @@ -101,7 +113,8 @@ export async function settleUnexpectedStructuredAgentSessionExit< settlementFailed = !(await retryUnexpectedExitSettlement({ context, event: unexpectedEvent, - session, + journal: session.journal, + fence: child.fence, stableSettlementId, verdict: { state: 'interrupted', completedAt: observedAt }, exitedDuringStartup, @@ -123,7 +136,7 @@ export async function settleUnexpectedStructuredAgentSessionExit< sessionId: unexpectedEvent.sessionId, expectedFence: unexpectedEvent.fence, expectedAcquisitionGeneration: unexpectedEvent.acquisitionGeneration, - acquisitionGeneration: session.acquisitionGeneration, + acquisitionGeneration: child.generation, now: context.now(), exitObservedAt: observedAt, // Bare cause: whatever this settlement could not write is settled from it later, by the @@ -133,10 +146,8 @@ export async function settleUnexpectedStructuredAgentSessionExit< } catch (error) { context.onBarrierError?.(unexpectedEvent.sessionId, error) } finally { - session.hasProviderChild = false - context.publishStatus?.(unexpectedEvent.sessionId) + endChild() if (released) { - session.fence = released.lease.runtimeFence context.publishFence(unexpectedEvent.sessionId, session) } } @@ -144,8 +155,10 @@ export async function settleUnexpectedStructuredAgentSessionExit< if (settlementFailed || !released) { return null } - // Resuming a start that failed would respawn into the same failure; the next send retries. - if (exitedDuringStartup || !context.hasResumeCapableHolder(unexpectedEvent.sessionId)) { + if ( + failedProviderChildStart(session) || + !context.hasResumeCapableHolder(unexpectedEvent.sessionId) + ) { return null } return { @@ -162,10 +175,7 @@ export function isStructuredAgentSessionRecoveryTicketCurrent( store: Pick sessions: Map< string, - Pick< - StructuredAgentSessionUnexpectedExitSession, - 'hasProviderChild' | 'fence' | 'acquisitionGeneration' - > + Pick > hasResumeCapableHolder: (sessionId: string) => boolean }, @@ -174,9 +184,9 @@ export function isStructuredAgentSessionRecoveryTicketCurrent( const session = context.sessions.get(ticket.sessionId) const record = context.store.getRecord(ticket.sessionId) return ( - session?.hasProviderChild === false && - session.fence === ticket.releasedFence && - session.acquisitionGeneration === ticket.deadAcquisitionGeneration && + session !== undefined && + session.child === null && + session.lastEndedChild?.generation === ticket.deadAcquisitionGeneration && record?.lease.runtimeFence === ticket.releasedFence && record.lease.claimStatus === 'released' && record.lease.handoffStage === null && @@ -187,22 +197,25 @@ export function isStructuredAgentSessionRecoveryTicketCurrent( async function retryUnexpectedExitSettlement(input: { context: Pick event: UnexpectedExitLifecycleEvent - session: Pick + journal: DeadGenerationJournal + fence: number stableSettlementId: string verdict: StructuredAgentSessionTurnVerdict exitedDuringStartup: boolean showUnexpectedExitOutcome?: boolean }): Promise { return settleStructuredAgentSessionDeadGeneration({ - journal: input.session.journal, + journal: input.journal, sessionId: input.event.sessionId, - fence: input.session.fence, + fence: input.fence, settlementId: input.stableSettlementId, verdict: input.verdict, pendingSubmissionReason: 'provider_exited_before_acknowledgement', showUnexpectedExitOutcome: input.showUnexpectedExitOutcome, unexpectedExitReason: input.event.reason, - exitedDuringStartup: input.exitedDuringStartup, + ...(input.exitedDuringStartup + ? { exitedDuringStartup: { generation: input.event.acquisitionGeneration } } + : {}), onError: input.context.onBarrierError }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-view-start-after-failed-start.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-view-start-after-failed-start.test.ts new file mode 100644 index 00000000000..b8e2d73c27f --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-view-start-after-failed-start.test.ts @@ -0,0 +1,207 @@ +// A Claude chat whose CLI exits before it finishes starting leaves one red row per start. A view +// opening, or coming back to, a chat whose last start failed used to start the CLI again, so every +// look at the chat added an identical row. Only a send retries a failed start: it is the user asking. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import { ClaudeStructuredSessionAdapter } from '../../claude/claude-structured-session-adapter' +import { + fakeClaude, + PROVIDER_SESSION_ID +} from '../../claude/claude-structured-session-test-support' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { structuredClaudeLifecycleEvent } from '../../runtime/structured-claude-runtime-adapter' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => unknown): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + +const CALLER = { callerKey: 'client-1' } +const SURFACE = 'desktop-chat:1' +const LAUNCH_FAILURE = + 'claude stream-json exited (code 1): qa-shim: simulated claude launch failure' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let adapter: ClaudeStructuredSessionAdapter +let claude: ReturnType +let lifecycle: Promise[] +/** Every initialize waits on it: a start that must outlast a step does not race a timer. */ +let initGate: Promise + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-view-start-after-failed-start-')) + resetHostTestOperationIds() + lifecycle = [] + initGate = Promise.resolve() + // Every start spawns, is published, and exits before it answers initialize. + claude = fakeClaude({ initDelayMs: 20, exitBeforeInit: LAUNCH_FAILURE }) + adapter = new ClaudeStructuredSessionAdapter({ + resolveLaunch: async () => ({ + pathToClaudeCodeExecutable: 'claude', + options: {}, + cwd: root, + claudeConfigDir: join(root, 'claude-home'), + providerSessionId: PROVIDER_SESSION_ID, + resumeLeafUuid: null, + resumesTranscript: (store.getRecord(SESSION)?.providerHandleChain.length ?? 0) > 0, + continuesChain: (store.getRecord(SESSION)?.providerHandleChain.length ?? 0) > 0 + }), + onEvent: (event) => { + const mapped = structuredClaudeLifecycleEvent(event) + if (mapped) { + lifecycle.push(host.handleAdapterEvent(mapped)) + } + }, + openConnection: async (launch, handlers) => { + const connection = await claude.openConnection(launch, handlers) + const initialize = connection.initializationResult + return Object.assign(connection, { + initializationResult: async () => { + await initGate + return initialize() + } + }) + }, + readProcessStartTime: async () => 1_700_000_000_000, + now: () => NOW + }) + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + host = new StructuredAgentSessionHost({ + store, + adapter: Object.assign(adapter, { supportsCreate: () => true }), + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => `spawn-${claude.connections.length + 1}`, + now: () => NOW + }) +}) + +afterEach(async () => { + await adapter.closeAll() + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +/** Waits until the adapter has published every exit it saw and the host settled each one. */ +async function settleExits(): Promise { + await eventually(async () => { + await adapter.drainObservedExits() + await Promise.all(lifecycle) + expect(host.journalSnapshot(SESSION).items.length).toBeGreaterThan(0) + }) + await Promise.all(lifecycle) +} + +/** The chat as it renders: the user's messages and the error rows, in journal order. */ +function timeline(): string[] { + return host + .journalSnapshot(SESSION) + .items.flatMap((item) => + item.body.kind === 'message' + ? ['message'] + : item.body.kind === 'status' && item.body.tone === 'error' + ? [item.body.text] + : [] + ) +} + +async function send(text: string): Promise { + const body = hostTestMessage(text) + const sent = await host.send(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: store.getRecord(SESSION)?.lease.runtimeFence ?? 1, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + }) + expect(sent).toMatchObject({ ok: true }) + return sent.ok ? sent.value.clientMessageId : '' +} + +describe('a fresh chat whose Claude start fails', () => { + // QA saw three failed starts from opening the chat alone: the create's, and the view's. + it.each([ + ['after the create already died', false], + ['while the create is still starting', true] + ] as const)( + 'starts once for the open and once for a send, one row each, when the view binds %s', + async (_when, createStillStarting) => { + // Released only once the views bound, so no runner is slow enough to let the create die first. + let releaseCreate = (): void => {} + const createGate = new Promise((resolve) => { + releaseCreate = resolve + }) + if (!createStillStarting) { + releaseCreate() + } + initGate = createGate + claude = fakeClaude({ exitBeforeInit: LAUNCH_FAILURE }) + await expect( + host.attach( + CALLER, + hostTestAttachParams(null, { + provider: 'claude', + agent: 'claude', + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: join(root, 'claude-home') }, + providerHandle: { kind: 'claude', sessionId: PROVIDER_SESSION_ID, leafUuid: null } + }) + ) + ).resolves.toMatchObject({ ok: true }) + if (!createStillStarting) { + await settleExits() + } + // Two surfaces bind, as a pane and a second window do; exit recovery sees a holder. + await host.hold(SESSION, SURFACE) + await host.hold(SESSION, 'desktop-chat:2') + if (createStillStarting) { + // The views bound to the create's child itself, before it exited. + expect(timeline()).toEqual([]) + releaseCreate() + } + await settleExits() + const startFailure = `The provider stopped before it finished starting: ${LAUNCH_FAILURE}.` + // Opening the chat: the create's start, once, and its row. + expect(claude.connections).toHaveLength(1) + expect(timeline()).toEqual([startFailure]) + + const sent = await send('reply with exactly: alpha') + await eventually(() => + expect( + host.journalSnapshot(SESSION).submissions.find((s) => s.clientMessageId === sent) + ).toMatchObject({ dispatchState: 'rejected', reason: startFailure }) + ) + await settleExits() + // The send's own start, once, and one row for it below the message. + expect(claude.connections).toHaveLength(2) + expect(timeline()).toEqual([startFailure, 'message', startFailure]) + + // Switching away and back re-takes the view's hold; it starts nothing and adds no row. + host.release(SESSION, SURFACE) + await host.hold(SESSION, SURFACE) + await settleExits() + expect(claude.connections).toHaveLength(2) + expect(timeline()).toEqual([startFailure, 'message', startFailure]) + } + ) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts index 0778d271129..66e3cd2cca2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts @@ -398,7 +398,7 @@ describe('already-wedged profiles become usable on load', () => { ['a restart eviction', false], ['a proven eviction by recovery', true] ] as const)( - 'settles the turn %s left even when the handle it was restored with never writes', + 'settles the turn %s left at the next acquire when the read restore could not write it', async (_origin, ownerOutlivedRestart) => { await seedStore( wedgedRecord({ claimStatus: 'live', handoffStage: null, ownerProcess: DEAD_OWNER }) @@ -415,15 +415,13 @@ describe('already-wedged profiles become usable on load', () => { : { outcome: 'pid-absent' }, stopOwnerProcess }) - // The read restore's settlement fails, and the handle it restored with never writes again. + // The read restore's settlement fails, and nothing retries it. const failing = vi .spyOn(AgentSessionJournal.prototype, 'appendLifecycleBatch') .mockRejectedValue(new Error('journal unavailable')) await host.restoreReadableSessions() failing.mockRestore() - vi.spyOn(restoredJournal(), 'appendLifecycleBatch').mockRejectedValue( - new Error('journal unavailable') - ) + expect(activeStructuredAgentSessionTurnId(restoredJournal().snapshot().items)).toBe('turn-1') expect(stopOwnerProcess).toHaveBeenCalledTimes(ownerOutlivedRestart ? 1 : 0) expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'released', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts index 8eac02dc17b..e68df2c09a6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts @@ -17,6 +17,7 @@ import { agentSessionProviderHandleRoot } from '../../../shared/agent-session-provider-handle' import { latestStructuredAgentSessionUserItem } from '../../../shared/structured-agent-session-projection' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionResumeMarker, @@ -135,8 +136,7 @@ function liveTasks( type WorkingCandidateSession = { journal: AgentSessionJournal /** Only this host generation's own child counts. A restored-for-reading journal has none. */ - hasProviderChild: boolean - fence?: number + child: { fence: number } | null } /** The offer one session is owed, taken right before teardown stops its provider child; null when @@ -154,16 +154,24 @@ export function structuredAgentSessionWorkingAtStop(input: { }): AgentSessionResumeMarker | null { const { sessionId, session } = input // A journal this host cannot read tells us nothing about what the turn was doing. - if (!session?.hasProviderChild || session.journal.isReadOnly) { + if (!session?.child || session.journal.isReadOnly) { return null } const snapshot = session.journal.snapshot() + // A queued message reached no agent, so it is no work to resume: quit rejects it as never sent. + const handedOver = snapshot.submissions.filter( + (submission) => !isQueuedAgentJournalSubmission(submission) + ) const roster = input.backgroundTasks(sessionId) - const status = structuredAgentSessionShownStatus(snapshot, roster, session.fence) + const status = structuredAgentSessionShownStatus( + { items: snapshot.items, submissions: handedOver }, + roster, + session.child.fence + ) if (status.state === 'done') { return null } - const work = structuredAgentSessionResumeWork(snapshot.items, snapshot.submissions) + const work = structuredAgentSessionResumeWork(snapshot.items, handedOver) const head = agentSessionProviderHandleChainHead( input.getRecord(sessionId)?.providerHandleChain ?? [] ) diff --git a/src/main/native-chat/agent-session-wire/structured-compaction-recovery.ts b/src/main/native-chat/agent-session-wire/structured-compaction-recovery.ts index 2199da05392..64f1ee652e7 100644 --- a/src/main/native-chat/agent-session-wire/structured-compaction-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-compaction-recovery.ts @@ -9,12 +9,21 @@ export async function recoverInterruptedCompaction( fence: number ): Promise { const command = store.getRecord(sessionId)?.conversationCommand - if ( - command?.command !== 'compact' || - command.phase !== 'prepared' || - command.runtimeFence === undefined || - command.runtimeFence === fence - ) { + if (command?.runtimeFence === undefined || command.runtimeFence === fence) { + return + } + await settleInterruptedCompaction(store, sessionId, journal, fence) +} + +/** A compaction still prepared whose child can no longer finish it: its outcome is unknown. */ +export async function settleInterruptedCompaction( + store: AgentSessionRecordStore, + sessionId: string, + journal: AgentSessionJournal, + fence: number +): Promise { + const command = store.getRecord(sessionId)?.conversationCommand + if (command?.command !== 'compact' || command.phase !== 'prepared') { return } const error = 'Previous compaction completion could not be confirmed after session recovery.' diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts index 78b9f4a59d0..fe6e5501975 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts @@ -48,9 +48,12 @@ export function conversationCommandBlocked( : 'Wait for background tasks to finish before using this command.' } if ( - ctx.journal - .submissions() - .some((entry) => entry.dispatchState === 'pending' || entry.dispatchState === 'unknown') + ctx.journal.submissions().some( + (entry) => + entry.dispatchState === 'pending' || + // Doubt left by an earlier child is not this one's work in flight. + (entry.dispatchState === 'unknown' && entry.recovered !== true && entry.fence === ctx.fence) + ) ) { return 'Resolve pending or unconfirmed messages before using this command.' } diff --git a/src/main/runtime/agent-session-operation-admission.ts b/src/main/runtime/agent-session-operation-admission.ts index e7a4165120e..f16d48efb9d 100644 --- a/src/main/runtime/agent-session-operation-admission.ts +++ b/src/main/runtime/agent-session-operation-admission.ts @@ -36,6 +36,7 @@ export type AgentSessionMutationOperationAdmission = { hostFingerprint: string now: number operationIdScope?: 'global' + conversationWrite?: true } export type AgentSessionMutationOperationDecision = { @@ -149,7 +150,8 @@ export function admitAgentSessionMutationOperation( envelope: args.envelope, hostFingerprint: args.hostFingerprint, ledger: ledger.decision, - lease: record.lease + lease: record.lease, + ...(args.conversationWrite ? { conversationWrite: true } : {}) }) if (ledger.decision.decision === 'admit' && admission.decision === 'refused') { ledger.rows.delete(agentSessionOperationKey(operation.callerKey, operation.operationId)) diff --git a/src/main/runtime/claude-structured-send-held-for-startup.test.ts b/src/main/runtime/claude-structured-send-held-for-startup.test.ts index 19e05811055..da1135f588f 100644 --- a/src/main/runtime/claude-structured-send-held-for-startup.test.ts +++ b/src/main/runtime/claude-structured-send-held-for-startup.test.ts @@ -1,8 +1,8 @@ // A Claude chat is published the moment its child spawns, before the CLI has answered initialize. -// A send in that window — into a fresh start, or into the restart a send itself asked for after -// a start that failed — is admitted and held until the child proves its start. When the CLI dies -// first, the held message is rejected with the CLI's own diagnostic, the chat shows the cause -// once, and nothing is left as a delivery nobody can confirm. Against the production runtime, +// A send in that window — into a fresh start, or into the restart the delivery loop makes for a +// send after a start that failed — is accepted and stays queued until the child proves its start. +// When the CLI dies first, the queued message is rejected with the CLI's own diagnostic, the chat +// shows the cause once, and nothing is left as a delivery nobody can confirm. Against the production runtime, // adapter, record store and host, with only the CLI process scripted. import { afterEach, describe, expect, it, vi } from 'vitest' @@ -39,7 +39,7 @@ async function send(host: StructuredAgentSessionHost, text: string): Promise { expect(statusRows(host)).toEqual([expect.stringContaining('not signed in')]) const releasedFence = fence(host) - // The send asks for the child back and is held for its start; the CLI dies again first. + // The delivery loop asks for the child back and the message waits for its start; the CLI + // dies again first. const held = await send(host, 'hello?') - expect(claude.children(SESSION)).toHaveLength(2) - expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(2)) + await vi.waitFor(() => + expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + ) await failLatestStart(host, 2) // Rejected with the cause, not left in doubt; one row for this attempt names it. @@ -111,8 +114,8 @@ describe('a send into a Claude chat whose CLI keeps failing at startup', () => { // The user signs in and retries: one restart, proven, written to the CLI. claude.behave(SESSION, {}) await send(host, 'hello again') - expect(claude.children(SESSION)).toHaveLength(3) - expect(fence(host)).toBe(releasedFence + 3) + await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(3)) + await vi.waitFor(() => expect(fence(host)).toBe(releasedFence + 3)) await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) await vi.waitFor(() => expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') @@ -122,7 +125,7 @@ describe('a send into a Claude chat whose CLI keeps failing at startup', () => { }) describe('a send while the first Claude start is still answering initialize', () => { - it('is held, and written once the CLI proves its start', async () => { + it('is queued, and written once the CLI proves its start', async () => { claude.behave(SESSION, { initHangs: true }) const host = await claude.install() await host.attach(CALLER, claude.attachParams(SESSION, null)) @@ -130,7 +133,7 @@ describe('a send while the first Claude start is still answering initialize', () await send(host, 'hello') expect(claude.child(SESSION).calls).not.toContain('send') - // The CLI answers: startup lands and the held message is written to the proven child. + // The CLI answers: startup lands and the queued message is written to the proven child. claude.child(SESSION).answerInit() await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) diff --git a/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts b/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts index 0d4b4909637..817b00144bc 100644 --- a/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts +++ b/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts @@ -1,10 +1,9 @@ -// A send restarts a chat's Claude child and is admitted against it while it is still starting. -// When that child dies before the send's dispatch reaches the adapter, the adapter has no session -// to hold the message for, so the dispatch throws. A child that never proved its start accepted -// nothing — input is only written after initialize — so the send settles `rejected` with the -// child's own diagnostic, never as a delivery nobody can confirm, and a client that was -// subscribed the whole time receives the failure row and the rejected submission over the wire. -// Against the production runtime, adapter, record store and host, with only the CLI scripted. +// A send is accepted into a chat whose Claude child is gone, and its delivery restarts the child. +// When that child dies before it proves its start, the message was never handed to it — delivery +// waits for the start — so the send settles `rejected` with the child's own diagnostic, never as a +// delivery nobody can confirm, and a client that was subscribed the whole time receives the +// failure row and the rejected submission over the wire. Against the production runtime, adapter, +// record store and host, with only the CLI scripted. import { afterEach, describe, expect, it, vi } from 'vitest' import { computeAgentSessionPayloadFingerprint } from '../../shared/agent-session-mutation-envelope' @@ -18,6 +17,11 @@ const SESSION = 'claude-send-restart-dies-first' const CALLER = { callerKey: 'client-1' } const DIAGNOSTIC = 'claude stream-json exited (code 1): claude: not signed in (rig)' +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => void): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + let claude = createScriptedClaudeRuntime([SESSION]) let operations = 0 /** The dispatch state each send was answered with, before any exit settled it. */ @@ -86,24 +90,14 @@ function submission(host: StructuredAgentSessionHost, clientMessageId: string) { } async function failLatestStart(host: StructuredAgentSessionHost, count: number): Promise { - await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(count)) + await eventually(() => expect(claude.children(SESSION)).toHaveLength(count)) claude.child(SESSION).exit(new Error(DIAGNOSTIC)) await waitForStructuredAgentSessionRecovery() - await vi.waitFor(() => + await eventually(() => expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') ) } -/** The restarted child dies the instant the send's dispatch reaches the adapter. */ -function killChildAtDispatch(host: StructuredAgentSessionHost): void { - const adapter = host.deps.adapter - const dispatch = adapter.dispatch.bind(adapter) - vi.spyOn(adapter, 'dispatch').mockImplementationOnce((input) => { - claude.child(SESSION).exit(new Error(DIAGNOSTIC)) - return dispatch(input) - }) -} - /** Everything a subscriber received, flattened to the rows and submissions it was shown. */ function received(events: AgentSessionSubscribeEvent[]) { const statusTexts: string[] = [] @@ -129,7 +123,7 @@ function received(events: AgentSessionSubscribeEvent[]) { return { statusTexts, submissions, fences } } -describe('a send whose restarted Claude child dies before the dispatch reaches the adapter', () => { +describe('a send whose restarted Claude child dies before it proves its start', () => { it('settles rejected with the diagnostic, keeps one failure row, and a Retry is one new attempt', async () => { claude.behave(SESSION, { initHangs: true }) const host = await claude.install() @@ -139,18 +133,13 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t await failLatestStart(host, 1) const releasedFence = fence(host) - killChildAtDispatch(host) const sent = await send(host, 'hello?') - // The send's own answer already says it was not delivered; it does not wait for the exit. - expect(answered.get(sent)).toBe('rejected') - expect(claude.children(SESSION)).toHaveLength(2) - await waitForStructuredAgentSessionRecovery() - await vi.waitFor(() => - expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') - ) + // Accepted: the answer comes before the restart it needs. + expect(answered.get(sent)).toBe('pending') + await failLatestStart(host, 2) // Provably not delivered, with the cause; not "unconfirmed". - await vi.waitFor(() => + await eventually(() => expect(submission(host, sent)).toMatchObject({ dispatchState: 'rejected', // Worded for the user: the red line under the composer shows it as it stands. @@ -168,40 +157,14 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t // Retry under a new id: one restart, and once the CLI is healthy the message is written. claude.behave(SESSION, {}) await send(host, 'hello again') - expect(claude.children(SESSION)).toHaveLength(3) - await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) + await eventually(() => expect(claude.children(SESSION)).toHaveLength(3)) + await eventually(() => expect(claude.child(SESSION).calls).toContain('send')) expect(claude.child(SESSION).calls.filter((call) => call === 'send')).toHaveLength(1) expect(statusRows(host)).toHaveLength(2) }) - it('names the diagnostic even when the exit was fully processed before the dispatch arrived', async () => { - claude.behave(SESSION, { initHangs: true }) - const host = await claude.install() - await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({ - ok: true - }) - await failLatestStart(host, 1) - const adapter = host.deps.adapter - const dispatch = adapter.dispatch.bind(adapter) - vi.spyOn(adapter, 'dispatch').mockImplementationOnce(async (input) => { - claude.child(SESSION).exit(new Error(DIAGNOSTIC)) - // The adapter settles and publishes the exit; the host's own settlement waits behind this send. - await new Promise((resolve) => setTimeout(resolve, 300)) - return dispatch(input) - }) - - const sent = await send(host, 'hello?') - expect(answered.get(sent)).toBe('rejected') - await waitForStructuredAgentSessionRecovery() - expect(submission(host, sent)).toMatchObject({ - dispatchState: 'rejected', - reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.` - }) - expect(statusRows(host)).toHaveLength(2) - }) - // A restart refused because its child died before it was handed over leaves one row, from the - // send, in the words any failed start uses. + // delivery, in the words any failed start uses, and rejects the message with them. it.each(['spawn', 'start-time-read'] as const)( 'leaves one row for a restart whose child exits at %s', async (at) => { @@ -213,10 +176,13 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t await failLatestStart(host, 1) claude.behave(SESSION, { exitsDuringSpawn: { diagnostic: DIAGNOSTIC, at } }) - await expect(attempt(host, 'hello?')).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } - }) + const sent = await send(host, 'hello?') + await eventually(() => + expect(submission(host, sent)).toMatchObject({ + dispatchState: 'rejected', + reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.` + }) + ) await waitForStructuredAgentSessionRecovery() expect(statusRows(host)).toEqual([ @@ -242,14 +208,10 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t }) try { await failLatestStart(host, 1) - killChildAtDispatch(host) const sent = await send(host, 'hello?') - await waitForStructuredAgentSessionRecovery() - await vi.waitFor(() => - expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') - ) + await failLatestStart(host, 2) - await vi.waitFor(() => { + await eventually(() => { const seen = received(events) expect(seen.submissions.get(sent)).toBe('rejected') expect(seen.statusTexts).toContainEqual( @@ -285,7 +247,7 @@ describe('a chat whose Claude CLI keeps failing to start, seen by a subscriber o return rows } - it('shows one row naming the cause per failed attempt, admitted or refused, and none once the CLI is fixed', async () => { + it('shows one row naming the cause per failed attempt, however the start died, and none once the CLI is fixed', async () => { claude.behave(SESSION, { initHangs: true }) const host = await claude.install() const events: AgentSessionSubscribeEvent[] = [] @@ -301,37 +263,38 @@ describe('a chat whose Claude CLI keeps failing to start, seen by a subscriber o }) try { await failLatestStart(host, 1) - await vi.waitFor(() => expect([...shownRows(events).values()]).toEqual([STARTUP_FAILURE])) + await eventually(() => expect([...shownRows(events).values()]).toEqual([STARTUP_FAILURE])) - // Send: admitted against the restarted child, which dies before starting. - killChildAtDispatch(host) - const sent = await attempt(host, 'hello?') - await waitForStructuredAgentSessionRecovery() - expect(sent).toMatchObject({ - ok: true, - value: { submission: { dispatchState: 'rejected', reason: STARTUP_FAILURE } } - }) - await vi.waitFor(() => + // Send: accepted, and its delivery restarts the child, which dies before starting. + const sent = await send(host, 'hello?') + await failLatestStart(host, 2) + await eventually(() => + expect(submission(host, sent)).toMatchObject({ + dispatchState: 'rejected', + reason: STARTUP_FAILURE + }) + ) + await eventually(() => expect([...shownRows(events).values()]).toEqual([STARTUP_FAILURE, STARTUP_FAILURE]) ) - // Retry while still broken: this restart dies before its child is handed over, so the send - // is refused before admission. Still one row, saying the same thing. + // Retry while still broken: this restart dies before its child is handed over, so the + // delivery's start is refused. Still one row, saying the same thing, on the rejected message. claude.behave(SESSION, { exitsDuringSpawn: { diagnostic: 'claude stream-json exited (code 1): claude: not signed in (rig)', at: 'start-time-read' } }) - await expect(attempt(host, 'hello?')).resolves.toMatchObject({ - ok: false, - refusal: { - code: 'agent_session_owner_restart_failed', - message: expect.stringMatching(/couldn't restart: .*not signed in \(rig\)/) - } - }) + const retried = await send(host, 'hello?') + await eventually(() => + expect(submission(host, retried)).toMatchObject({ + dispatchState: 'rejected', + reason: STARTUP_FAILURE + }) + ) await waitForStructuredAgentSessionRecovery() - await vi.waitFor(() => + await eventually(() => expect([...shownRows(events).values()]).toEqual([ STARTUP_FAILURE, STARTUP_FAILURE, @@ -342,7 +305,7 @@ describe('a chat whose Claude CLI keeps failing to start, seen by a subscriber o // The CLI is fixed: Retry delivers and adds no row. claude.behave(SESSION, {}) await expect(attempt(host, 'hello?')).resolves.toMatchObject({ ok: true }) - await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) + await eventually(() => expect(claude.child(SESSION).calls).toContain('send')) expect(shownRows(events).size).toBe(3) } finally { unsubscribe() diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts index 91bf1158e07..c530a1e84ab 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts @@ -91,7 +91,7 @@ describe('structured mailbox pointer host', () => { value: { submission: { dispatchState } } }) ) - hostRef.current = { send } + hostRef.current = { send, waitForSendSettlement: async () => undefined } await expect( createStructuredMailboxPointerHost().send({ sessionId: 's1', @@ -107,6 +107,28 @@ describe('structured mailbox pointer host', () => { expect(send.mock.calls[0]![1]!.retryUnknown).toBeUndefined() }) + it('consumes mail once an accepted nudge is delivered while the worker starts (W10)', async () => { + hostRef.current = { + send: async () => ({ + ok: true, + value: { clientMessageId: 'op1', submission: { dispatchState: 'pending' } } + }), + waitForSendSettlement: async () => ({ + value: { clientMessageId: 'op1', submission: { dispatchState: 'accepted' } } + }) + } + await expect( + createStructuredMailboxPointerHost().send({ + sessionId: 's1', + dispatchId: 'd1', + operationId: 'op1', + expectedRuntimeFence: 1, + payloadFingerprint: 'fp', + body: { kind: 'message', role: 'user', blocks: [] } + } as never) + ).resolves.toEqual({ kind: 'sent', state: 'accepted' }) + }) + it('scopes direct peer mail to the session when there is no dispatch to scope to', async () => { // Direct mail is addressed to the worker's own handle, so there may be no dispatch at all. // The ledger is keyed on (callerKey, operationId): a key derived from the session keeps that diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts index b722952df92..907a7bcc097 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts @@ -6,6 +6,7 @@ * the send and reports what the host said. */ +import { ORCHESTRATION_READINESS_TIMEOUT_MS } from '../../../shared/orchestration-timing-budgets' import { AGENT_SESSION_NOT_ATTACHED } from '../../native-chat/agent-session-wire/structured-agent-session-mutation-admission' import { getStructuredAgentSessionHost } from '../../native-chat/agent-session-wire/structured-agent-session-registry' import type { StructuredMailboxPointerHost } from './structured-mailbox-pointer-delivery' @@ -94,8 +95,19 @@ export function createStructuredMailboxPointerHost(): StructuredMailboxPointerHo ? { kind: 'unattached' } : { kind: 'sent', state: 'rejected' } } - // `pending` is not yet an acknowledgement; only `accepted` may consume mail. - const state = result.value.submission.dispatchState + // `pending` is not yet an acknowledgement; only `accepted` may consume mail. Accepted is not + // delivered, so wait out a start; a wait that runs out parks for the next journal edge. + const submission = + result.value.submission.dispatchState === 'pending' + ? (( + await host + .waitForSendSettlement(input.sessionId, result.value.clientMessageId, { + budgetMs: ORCHESTRATION_READINESS_TIMEOUT_MS + }) + .catch(() => undefined) + )?.value.submission ?? result.value.submission) + : result.value.submission + const state = submission.dispatchState return { kind: 'sent', state: state === 'accepted' ? 'accepted' : state === 'rejected' ? 'rejected' : 'unknown' diff --git a/src/main/runtime/rpc/methods/orchestration-structured-worker-session.test.ts b/src/main/runtime/rpc/methods/orchestration-structured-worker-session.test.ts index 493f9d092c3..86568a0435f 100644 --- a/src/main/runtime/rpc/methods/orchestration-structured-worker-session.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-structured-worker-session.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentJournalSubmission } from '../../../../shared/agent-session-journal-types' import { dispatchWriteFailureReason } from '../../../../shared/structured-agent-session-dispatch-rejection' const hostRef: { current: unknown } = { current: null } @@ -224,20 +225,69 @@ describe('structured worker session hold', () => { }) describe('structured worker dispatch preamble', () => { - function hostWithSubmission(submission: Record) { + type PreambleHost = Parameters[0]['host'] + type Settled = Pick + + function submissionOf(settled: Settled): AgentJournalSubmission { return { - deps: { store: { getRecord: () => ({ lease: { runtimeFence: 7 } }) } }, - send: async () => ({ ok: true, value: { clientMessageId: 'c1', submission } }) - } as never + clientMessageId: 'c1', + fence: 7, + payloadFingerprint: 'fingerprint', + providerItemId: null, + submittedAt: 1, + resolvedAt: null, + ...settled + } } - const send = (host: never) => + function hostWithSubmission(submission: Settled, delivered?: Settled): PreambleHost { + return { + deps: { store: { getRecord: () => ({ lease: { runtimeFence: 7 } }) } }, + send: async () => ({ + ok: true, + replayed: false, + fence: 7, + cursor: { epoch: 'epoch-1', sequence: 1 }, + value: { clientMessageId: 'c1', submission: submissionOf(submission) } + }), + // What the submission settled as while the worker's agent started; undefined when the + // start outlasted the wait. + waitForSendSettlement: async () => + delivered + ? { + cursor: { epoch: 'epoch-1', sequence: 2 }, + value: { clientMessageId: 'c1', submission: submissionOf(delivered) } + } + : undefined + } + } + + const send = (host: PreambleHost) => sendStructuredWorkerPreamble({ host, sessionId: 's1', dispatchId: 'd1', preamble: 'spec' }) it('reports the preamble delivered only on an accepted submission', async () => { await expect( send(hostWithSubmission({ dispatchState: 'accepted', reason: null })) - ).resolves.toBeUndefined() + ).resolves.toBe('accepted') + }) + + it('waits for an accepted preamble to be delivered, and reports that delivery (W10)', async () => { + await expect( + send( + hostWithSubmission( + { dispatchState: 'pending', reason: null }, + { dispatchState: 'accepted', reason: null } + ) + ) + ).resolves.toBe('accepted') + }) + + it('reports a preamble still held for an agent that outlasted the wait, without failing the start (W10)', async () => { + // Held, not lost: the host delivers it when the agent starts. Throwing here tore the worker + // down, which rejected the preamble the start was about to deliver. + await expect( + send(hostWithSubmission({ dispatchState: 'pending', reason: null })) + ).resolves.toBe('pending') }) it('never claims delivery for a submission the provider never acknowledged', async () => { @@ -245,15 +295,13 @@ describe('structured worker dispatch preamble', () => { // into `unknown`, and `performSend` still returns ok. Reporting that as `dispatch_input: // accepted` marks the worker ready with no task, and the coordinator blocks in // `check --wait --types worker_done` until it times out. - for (const dispatchState of ['unknown', 'pending'] as const) { - const error = await send( - hostWithSubmission({ dispatchState, reason: 'provider child exited' }) - ).catch((thrown: unknown) => thrown) - expect((error as { code?: string }).code).toBe('operation_unknown') - // The wiring, not just the throw: this is the code that makes the start receipt - // `outcome_unknown` with the worker-show / worker-abandon recovery commands. - expect(isUnknownWorkerStartOutcome(error, 'dispatch_input')).toBe(true) - } + const error = await send( + hostWithSubmission({ dispatchState: 'unknown', reason: 'provider child exited' }) + ).catch((thrown: unknown) => thrown) + expect((error as { code?: string }).code).toBe('operation_unknown') + // The wiring, not just the throw: this is the code that makes the start receipt + // `outcome_unknown` with the worker-show / worker-abandon recovery commands. + expect(isUnknownWorkerStartOutcome(error, 'dispatch_input')).toBe(true) }) it('keeps a rejected preamble a proven failure under a code of its own', async () => { diff --git a/src/main/runtime/rpc/methods/orchestration-structured-worker-session.ts b/src/main/runtime/rpc/methods/orchestration-structured-worker-session.ts index 5bde461a059..06f657924b1 100644 --- a/src/main/runtime/rpc/methods/orchestration-structured-worker-session.ts +++ b/src/main/runtime/rpc/methods/orchestration-structured-worker-session.ts @@ -14,6 +14,7 @@ import { randomUUID } from 'node:crypto' import { isDefinitiveAgentSessionCreateRefusal } from '../../../../shared/agent-session-definitive-refusal' import type { AgentJournalMessageItem } from '../../../../shared/agent-session-journal-types' +import { ORCHESTRATION_READINESS_TIMEOUT_MS } from '../../../../shared/orchestration-timing-budgets' import type { StructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-host' import { getStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' import type { OrcaRuntimeService } from '../../orca-runtime' @@ -213,13 +214,22 @@ export async function discardStructuredWorkerSession( retireSettledStructuredWorkerTab(sessionId, runtime) } -/** Delivers the dispatch preamble as the worker's first turn. */ +/** What a preamble send reads of the host. */ +type StructuredWorkerPreambleHost = Pick< + StructuredAgentSessionHost, + 'send' | 'waitForSendSettlement' +> & { + deps: { store: { getRecord: (sessionId: string) => { lease: { runtimeFence: number } } | null } } +} + +/** Delivers the dispatch preamble as the worker's first turn. `pending`: the worker's agent had + * not taken it within the wait; the host still holds it for that agent, and never re-sends it. */ export async function sendStructuredWorkerPreamble(args: { - host: StructuredAgentSessionHost + host: StructuredWorkerPreambleHost sessionId: string dispatchId: string preamble: string -}): Promise { +}): Promise<'accepted' | 'pending'> { const body: AgentJournalMessageItem = { kind: 'message', role: 'user', @@ -244,9 +254,19 @@ export async function sendStructuredWorkerPreamble(args: { if (!result.ok) { throw new Error(`The dispatch preamble was refused: ${result.refusal.message}`) } - const submission = result.value.submission - if (submission.dispatchState === 'accepted') { - return + // Accepted is not delivered: the worker's agent may still be starting. + const submission = + result.value.submission.dispatchState === 'pending' + ? (( + await args.host + .waitForSendSettlement(args.sessionId, result.value.clientMessageId, { + budgetMs: ORCHESTRATION_READINESS_TIMEOUT_MS + }) + .catch(() => undefined) + )?.value.submission ?? result.value.submission) + : result.value.submission + if (submission.dispatchState === 'accepted' || submission.dispatchState === 'pending') { + return submission.dispatchState } if (submission.dispatchState === 'rejected') { // A rejection is a verdict, not a mystery: the preamble provably did not happen. diff --git a/src/main/runtime/rpc/methods/orchestration-worker-mode-opacity.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-mode-opacity.test.ts index aa8c7c58574..48ffe07d5d0 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-mode-opacity.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-mode-opacity.test.ts @@ -42,6 +42,7 @@ vi.mock('./orchestration-structured-worker-session', async (importOriginal) => ( ...(await importOriginal>()), sendStructuredWorkerPreamble: async (args: { preamble: string }) => { structuredPreambles.push(args.preamble) + return 'accepted' }, releaseStructuredWorkerSession: () => {}, discardStructuredWorkerSession: async () => {} diff --git a/src/main/runtime/rpc/methods/orchestration-worker-start-mode-selection.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-start-mode-selection.test.ts index 52cf3579016..a2beea9ea05 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-start-mode-selection.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-start-mode-selection.test.ts @@ -30,7 +30,7 @@ vi.mock('./orchestration/federation/federated-worker-start', () => ({ })) vi.mock('./orchestration-structured-worker-session', async (importOriginal) => ({ ...(await importOriginal>()), - sendStructuredWorkerPreamble: async () => {}, + sendStructuredWorkerPreamble: async () => 'accepted', releaseStructuredWorkerSession: () => {}, discardStructuredWorkerSession: async () => {} })) diff --git a/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts b/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts index ecb3270874e..bc2b9601ceb 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts @@ -5,6 +5,7 @@ import { dispatchPreambleSendOptions } from '../../../../orchestration/preamble' import { sendStructuredWorkerPreamble } from '../../orchestration-structured-worker-session' +import type { WorkerTurnStartObservation } from './worker-start-turn-observation' import type { createStructuredWorkerSessionForWorktree } from './worker-topology' type StructuredSession = Awaited> | null @@ -14,7 +15,8 @@ type StructuredSession = Awaited { +}): Promise<{ + prompt?: RuntimeTerminalSend['prompt'] + structuredTurnStart?: WorkerTurnStartObservation +}> { const { runtime, structuredSession, terminalHandle } = args const preamble = buildDispatchPreamble({ // Depth only. A worker is taught the same verbs whichever mode it runs in, so this must not @@ -45,19 +50,32 @@ export async function deliverWorkerDispatchPreamble(args: { cliCommand: runtime.getTerminalOrchestrationCliCommand(terminalHandle) }) if (structuredSession) { - await sendStructuredWorkerPreamble({ + const delivery = await sendStructuredWorkerPreamble({ host: structuredSession.host, sessionId: structuredSession.identity.sessionId, dispatchId: args.dispatchId, preamble }) - return undefined + return { + structuredTurnStart: + delivery === 'accepted' + ? { verdict: 'observed' } + : { + verdict: 'unobserved', + reason: + 'The dispatch preamble was accepted, but the agent had not started to take it. It ' + + 'is delivered when the agent starts; if the worker then reports, this Dispatch ' + + 'settles normally.' + } + } + } + return { + prompt: ( + await runtime.sendTerminalAgentPrompt( + terminalHandle, + preamble, + dispatchPreambleSendOptions(args.requestId) + ) + ).prompt } - return ( - await runtime.sendTerminalAgentPrompt( - terminalHandle, - preamble, - dispatchPreambleSendOptions(args.requestId) - ) - ).prompt } diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.test.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.test.ts new file mode 100644 index 00000000000..e39a06110dc --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('../../../../orchestration/preamble', () => ({ buildDispatchPreamble: () => 'preamble' })) +vi.mock('./worker-topology', async (importOriginal) => ({ + ...(await importOriginal>()), + monitorWorkerSetup: () => {} +})) + +const { deliverAndSettleWorkerStartReadiness } = await import('./worker-start-readiness-settlement') + +function settle(delivered: 'accepted' | undefined) { + const db = { + getWorkerDispatch: () => ({ state: 'starting' }), + markWorkerStartUnknown: vi.fn(() => ({ + stage: 'turn_start_unobserved', + residual_resources: '[]' + })), + markWorkerDispatchReady: vi.fn(() => ({ state: 'ready', stage: 'ready' })) + } + const host = { + deps: { store: { getRecord: () => ({ lease: { runtimeFence: 1 } }) } }, + send: async () => ({ + ok: true, + value: { clientMessageId: 'c1', submission: { dispatchState: 'pending', reason: null } } + }), + // undefined: the worker's agent was still starting when the wait ran out. + waitForSendSettlement: async () => + delivered + ? { value: { clientMessageId: 'c1', submission: { dispatchState: delivered } } } + : undefined + } + const args = { + runtime: { + getNestedWorkerMaxDepth: () => 3, + getTerminalOrchestrationCliCommand: () => 'orca' + }, + db, + run: { id: 'run_1' }, + task: { id: 't1', spec: 'do the thing' }, + dispatchId: 'd1', + dispatchDepth: 0, + structuredSession: { host, identity: { sessionId: 's1' } }, + terminalHandle: 'structured_worker_1', + coordinatorHandle: 'term_c', + dispatchCapability: 'capability', + devMode: undefined, + requestId: 'r1', + agent: 'claude', + setupReceipt: {}, + launchReceipt: {}, + mode: {}, + timeoutMs: 60_000, + effects: [], + terminalRevealWarning: undefined, + onStage: () => {} + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fakes implement exactly the runtime, db and host members this settlement reaches. + const receipt = deliverAndSettleWorkerStartReadiness(args as never) + return { db, receipt } +} + +describe('a structured worker whose agent outlasts the preamble wait', () => { + it('parks as start-unknown instead of failing the start, and never names a screen to read', async () => { + const { db, receipt } = settle(undefined) + + // Resolving, not throwing, is what keeps the worker's session: a throw tears it down. + await expect(receipt).resolves.toMatchObject({ + state: 'outcome_unknown', + turnStart: 'unobserved', + nextCommands: [ + 'orca orchestration worker-show --dispatch d1 --json', + 'orca orchestration worker-abandon --dispatch d1 --json' + ] + }) + expect(db.markWorkerStartUnknown).toHaveBeenCalledWith( + 'd1', + 'turn_start_unobserved', + expect.stringContaining('delivered when the agent starts'), + expect.anything() + ) + expect(db.markWorkerDispatchReady).not.toHaveBeenCalled() + }) + + it('is ready once the agent took the preamble within the wait', async () => { + const { db, receipt } = settle('accepted') + + await expect(receipt).resolves.toMatchObject({ state: 'ready', turnStart: 'observed' }) + expect(db.markWorkerStartUnknown).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.ts index a3c57a357d2..7e9118daffb 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.ts @@ -47,7 +47,7 @@ export async function deliverAndSettleWorkerStartReadiness(args: { const { runtime, db, run, task, structuredSession, terminalHandle, effects } = args args.onStage('dispatch_input') - const promptDelivery = await deliverWorkerDispatchPreamble({ + const delivery = await deliverWorkerDispatchPreamble({ runtime, structuredSession, terminalHandle, @@ -71,11 +71,11 @@ export async function deliverAndSettleWorkerStartReadiness(args: { // The write above was accepted without waiting on provider hooks; now demand the positive // evidence the receipt claims is observable. A worker whose turn never starts must not be // reported ready — a wedged agent and a working one looked identical before this gate. - // A structured preamble send is acknowledged by the provider or throws, so it is already - // positive evidence. - const turnStart: WorkerTurnStartObservation = structuredSession - ? { verdict: 'observed' } - : await observeWorkerTurnStart({ runtime, terminalHandle, prompt: promptDelivery }) + // A structured preamble send is its own evidence: acknowledged, or still held for its agent. + const promptDelivery = delivery.prompt + const turnStart: WorkerTurnStartObservation = + delivery.structuredTurnStart ?? + (await observeWorkerTurnStart({ runtime, terminalHandle, prompt: promptDelivery })) const deliveredPrompt = turnStart.prompt ?? promptDelivery monitorWorkerSetup({ runtime, @@ -98,7 +98,7 @@ export async function deliverAndSettleWorkerStartReadiness(args: { id: terminalHandle, state: 'turn_unobserved' }) - const reason = describeUnobservedWorkerTurnStart(args.agent) + const reason = turnStart.reason ?? describeUnobservedWorkerTurnStart(args.agent) const worker = db.markWorkerStartUnknown( args.dispatchId, 'turn_start_unobserved', @@ -122,7 +122,8 @@ export async function deliverAndSettleWorkerStartReadiness(args: { residualResources: JSON.parse(worker.residual_resources) as unknown[], nextCommands: [ `orca orchestration worker-show --dispatch ${args.dispatchId} --json`, - `orca terminal read --terminal ${terminalHandle} --screen`, + // A structured worker has no screen to read. + ...(structuredSession ? [] : [`orca terminal read --terminal ${terminalHandle} --screen`]), `orca orchestration worker-abandon --dispatch ${args.dispatchId} --json` ], ...(args.terminalRevealWarning ? { warning: args.terminalRevealWarning } : {}) diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-turn-observation.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-turn-observation.ts index d1c9e59adfa..cc648ee7dde 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-turn-observation.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-turn-observation.ts @@ -19,6 +19,8 @@ export type WorkerTurnStartVerdict = 'observed' | 'permission' | 'unsupported' | export type WorkerTurnStartObservation = { verdict: WorkerTurnStartVerdict prompt?: RuntimeTerminalPromptDelivery + /** Why an `unobserved` start is unknown, when the default PTY wording does not fit. */ + reason?: string } function classifyPromptDelivery(prompt: RuntimeTerminalPromptDelivery): WorkerTurnStartVerdict { diff --git a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts new file mode 100644 index 00000000000..9ae846df6fc --- /dev/null +++ b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts @@ -0,0 +1,84 @@ +// Which clients get a send answered at acceptance, and which have their reply held until the +// message is handed over: a client that cannot show a rejection after `pending` must not see one. + +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, + STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY +} from '../../../../shared/protocol-version' +import { DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES } from '../../../ipc/desktop-renderer-runtime-capabilities' +import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement' +import { + call, + clearStructuredHostStub, + hostCalls, + installStructuredHostStub, + SESSION, + sendParams, + STRUCTURED_CLIENT +} from './structured-agent-session-rpc.test-fixture' + +beforeEach(() => { + installStructuredHostStub() +}) + +afterEach(() => { + clearStructuredHostStub() +}) + +describe('agentSession.send reply timing', () => { + it('holds a pending reply until handover for a client that cannot show a later rejection', async () => { + hostCalls.send.mockResolvedValueOnce(pendingSendResult()) + hostCalls.waitForSendSettlement.mockResolvedValueOnce(undefined) + + await call('agentSession.send', sendParams(), STRUCTURED_CLIENT) + + expect(hostCalls.waitForSendSettlement).toHaveBeenCalledWith(SESSION, 'client-1', { + until: 'handed-over', + budgetMs: STRUCTURED_AGENT_SESSION_START_WAIT_MS + }) + }) + + it('answers at acceptance for the local desktop and paired desktop clients (W2)', async () => { + for (const clientCapabilities of [ + DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES, + // A paired desktop gains the structured surface as its own capability; the reply rule rides + // on the list it already sends. + [...ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY] + ]) { + hostCalls.send.mockResolvedValueOnce(pendingSendResult()) + const response = await call('agentSession.send', sendParams(), { + clientKind: 'runtime', + clientCapabilities: [...clientCapabilities] + }) + expect(response).toMatchObject({ + ok: true, + result: { value: { submission: { dispatchState: 'pending' } } } + }) + } + expect(hostCalls.waitForSendSettlement).not.toHaveBeenCalled() + }) +}) + +function pendingSendResult() { + return { + ok: true, + replayed: false, + fence: 1, + cursor: { epoch: 'epoch-a', sequence: 1 }, + value: { + clientMessageId: 'client-1', + submission: { + clientMessageId: 'client-1', + fence: 1, + payloadFingerprint: 'fingerprint', + dispatchState: 'pending' as const, + providerItemId: null, + reason: null, + submittedAt: 1, + resolvedAt: null, + handoverRecorded: true as const + } + } + } +} diff --git a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts index 8b948869990..5fde5e0f6e4 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts @@ -1,26 +1,43 @@ -import { AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import { + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, + AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY +} from '../../../../shared/protocol-version' import type { StructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-host' +import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement' import type { RpcContext } from '../core' import { requireStructuredHost, structuredCallerFor } from './structured-agent-session-gate' +/** + * A send answers once the host accepts it. A client that predates that answer cannot show a + * message rejected after it, so its reply is held until the message is handed over or rejected; + * one that predates pending replies at all waits, as before, for the provider's answer. + */ export async function sendStructuredAgentSessionForClient( params: Parameters[1], context: RpcContext ) { const host = requireStructuredHost(context) const result = await host.send(structuredCallerFor(context), params) + const capabilities = context.clientCapabilities ?? [] if ( !result.ok || result.value.submission.dispatchState !== 'pending' || context.clientKind === undefined || - context.clientCapabilities?.includes(AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY) + capabilities.includes(AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY) ) { return result } + // The start that used to run before the reply now runs after acceptance, so both waits cover it. const settled = await host.waitForSendSettlement( params.envelope.sessionId, result.value.clientMessageId, - context.signal + { + until: capabilities.includes(AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY) + ? 'handed-over' + : 'answered', + budgetMs: STRUCTURED_AGENT_SESSION_START_WAIT_MS, + ...(context.signal ? { signal: context.signal } : {}) + } ) return settled ? { ...result, ...settled } : result } diff --git a/src/main/runtime/rpc/methods/structured-agent-session.test.ts b/src/main/runtime/rpc/methods/structured-agent-session.test.ts index 38b55fd70eb..f97a458226a 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session.test.ts @@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { setStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' import { + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, RUNTIME_CAPABILITIES, RUNTIME_PROTOCOL_VERSION, @@ -12,6 +13,7 @@ import { STRUCTURED_AGENT_SESSION_REVEAL_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement' import { computeAgentSessionPayloadFingerprint } from '../../../../shared/agent-session-mutation-envelope' import { ALL_RPC_METHODS } from './index' import { STRUCTURED_AGENT_SESSION_METHODS } from './structured-agent-session' @@ -149,6 +151,8 @@ describe('capability gating', () => { it('advertises the capability without bumping the protocol version', () => { expect(RUNTIME_CAPABILITIES).toContain(STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY) expect(RUNTIME_CAPABILITIES).toContain(AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY) + // A client tells a host that accepts first, and admits a writer-free Stop before a turn, by it. + expect(RUNTIME_CAPABILITIES).toContain(AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY) expect(RUNTIME_CAPABILITIES).toContain(STRUCTURED_AGENT_SESSION_HOLD_RUNTIME_CAPABILITY) expect(RUNTIME_CAPABILITIES).toContain(STRUCTURED_AGENT_SESSION_REVEAL_RUNTIME_CAPABILITY) // Separate from the structured capability on purpose: a host can serve the rest of the @@ -252,11 +256,11 @@ describe('capability gating', () => { signal: controller.signal }) - expect(hostCalls.waitForSendSettlement).toHaveBeenCalledWith( - SESSION, - 'client-1', - controller.signal - ) + expect(hostCalls.waitForSendSettlement).toHaveBeenCalledWith(SESSION, 'client-1', { + until: 'answered', + budgetMs: STRUCTURED_AGENT_SESSION_START_WAIT_MS, + signal: controller.signal + }) expect(response).toMatchObject({ ok: true, result: { @@ -302,36 +306,6 @@ describe('capability gating', () => { }) }) - it('returns durable pending immediately to clients that understand admission', async () => { - hostCalls.send.mockResolvedValueOnce({ - ok: true, - replayed: false, - fence: 1, - cursor: { epoch: 'epoch-a', sequence: 1 }, - value: { - clientMessageId: 'client-1', - submission: { - clientMessageId: 'client-1', - fence: 1, - payloadFingerprint: 'fingerprint', - dispatchState: 'pending', - providerItemId: null, - reason: null, - submittedAt: 1, - resolvedAt: null - } - } - }) - - const response = await call('agentSession.send', sendParams(), STRUCTURED_CLIENT) - - expect(hostCalls.waitForSendSettlement).not.toHaveBeenCalled() - expect(response).toMatchObject({ - ok: true, - result: { value: { submission: { dispatchState: 'pending' } } } - }) - }) - it('requires the host structured-chat setting for mobile clients', async () => { const response = await call('agentSession.send', sendParams(), STRUCTURED_MOBILE_CLIENT, { getClientSettings: () => ({ experimentalStructuredNativeChat: false }) diff --git a/src/main/runtime/runtime-worktree-structured-agent-rows-liveness.test.ts b/src/main/runtime/runtime-worktree-structured-agent-rows-liveness.test.ts index b1e43a9025d..27d5ba48b29 100644 --- a/src/main/runtime/runtime-worktree-structured-agent-rows-liveness.test.ts +++ b/src/main/runtime/runtime-worktree-structured-agent-rows-liveness.test.ts @@ -89,7 +89,7 @@ async function awaitingApproval() { SESSION, { journal, - hasProviderChild: true, + child: { phase: 'ready' as const }, params: { location: { executionHostId: 'local' as const, diff --git a/src/main/runtime/structured-agent-session-integration-replay.test.ts b/src/main/runtime/structured-agent-session-integration-replay.test.ts index 4fa390be0fb..a4e2500ffc4 100644 --- a/src/main/runtime/structured-agent-session-integration-replay.test.ts +++ b/src/main/runtime/structured-agent-session-integration-replay.test.ts @@ -317,11 +317,14 @@ describe('a structured codex session over agentSession.*', () => { body } + const turnStarts = () => codex.live().calls.filter((entry) => entry.method === 'turn/start') await ok('agentSession.send', params) + // Accepted first; the delivery loop hands it over once. + await vi.waitFor(() => expect(turnStarts()).toHaveLength(1)) const replay = await call('agentSession.send', params) expect(replay).toMatchObject({ ok: true, result: { ok: true, replayed: true } }) - expect(codex.live().calls.filter((entry) => entry.method === 'turn/start')).toHaveLength(1) + expect(turnStarts()).toHaveLength(1) }) it('joins an acquired attach through journal bind before draining final rows', async () => { diff --git a/src/main/runtime/structured-agent-session-integration.test.ts b/src/main/runtime/structured-agent-session-integration.test.ts index 3f8adae4906..d81170ee075 100644 --- a/src/main/runtime/structured-agent-session-integration.test.ts +++ b/src/main/runtime/structured-agent-session-integration.test.ts @@ -218,6 +218,11 @@ function createIntentParams() { } let codex: CodexScript +/** Accepted first; the delivery loop hands the send over as `turn/start` after the reply. */ +const handedOverAs = (params: Record) => + vi.waitFor(() => + expect(codex.live().calls.at(-1)).toMatchObject({ method: 'turn/start', params }) + ) let root: string let dispatcher: RpcDispatcher let bootEnvironmentReads: number @@ -466,10 +471,7 @@ describe('a structured codex session over agentSession.*', () => { // send coalesced into a running turn is answered with that turn's id, so // which message landed where is knowable only from the echo. expect(sent.submission).toMatchObject({ dispatchState: 'pending', providerItemId: null }) - expect(codex.live().calls.at(-1)).toMatchObject({ - method: 'turn/start', - params: { threadId: THREAD, clientUserMessageId: sent.clientMessageId } - }) + await handedOverAs({ threadId: THREAD, clientUserMessageId: sent.clientMessageId }) codex.notify('turn/started', { turn: { id: TURN } }) // Codex echoes the message back carrying the `clientId` it was sent under, @@ -557,14 +559,11 @@ describe('a structured codex session over agentSession.*', () => { // "delivery unconfirmed" — it carries no identity yet, because the response // to a coalesced send names the running turn rather than this message. expect(sent.submission).toMatchObject({ dispatchState: 'pending', providerItemId: null }) - expect(codex.live().calls.at(-1)).toMatchObject({ - method: 'turn/start', - params: { - threadId: THREAD, - clientUserMessageId: sent.clientMessageId, - model: 'gpt-live', - effort: 'high' - } + await handedOverAs({ + threadId: THREAD, + clientUserMessageId: sent.clientMessageId, + model: 'gpt-live', + effort: 'high' }) // ── stream ────────────────────────────────────────────────────────────── diff --git a/src/main/runtime/structured-agent-session-runtime-exit.test.ts b/src/main/runtime/structured-agent-session-runtime-exit.test.ts index 7a3736b9009..0b2c8bd4f4b 100644 --- a/src/main/runtime/structured-agent-session-runtime-exit.test.ts +++ b/src/main/runtime/structured-agent-session-runtime-exit.test.ts @@ -128,7 +128,8 @@ describe('structured session runtime provider-exit wiring', () => { await expect( host.send({ callerKey: 'runtime-test' }, { envelope, body }) ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) - expect(turn).toBe(1) + // The send answers at acceptance; the delivery loop hands it to the reacquired child after. + await vi.waitFor(() => expect(turn).toBe(1)) }) it('does not reacquire when the production exit callback comes from a requested close', async () => { diff --git a/src/renderer/src/components/native-chat/NativeChatDeliveryRetry.tsx b/src/renderer/src/components/native-chat/NativeChatDeliveryRetry.tsx index 5a93657f943..e37016d646d 100644 --- a/src/renderer/src/components/native-chat/NativeChatDeliveryRetry.tsx +++ b/src/renderer/src/components/native-chat/NativeChatDeliveryRetry.tsx @@ -18,9 +18,13 @@ export function NativeChatDeliveryRetry({ retry: (clientMessageId: string) => void }): React.JSX.Element | null { // Why: read through the drain's own rule, so Retry can never name an entry other than the one - // the queue actually stopped on -- which is no longer always the head. + // the queue actually stopped on -- which is no longer always the head. A rejected entry holds + // nothing up, so it is offered only when the queue itself is not stopped. const admission = admitStructuredAgentSessionOutboxEntry(outbox, blockedClientMessageId) - const retryable = admission.state === 'blocked' ? admission.entry : null + const retryable = + admission.state === 'blocked' + ? admission.entry + : (outbox.find((entry) => entry.state === 'rejected') ?? null) if (!retryable) { return null } diff --git a/src/renderer/src/components/native-chat/structured-agent-session-outbox-dispatch.ts b/src/renderer/src/components/native-chat/structured-agent-session-outbox-dispatch.ts index 91e6149cb09..9f91b9f9b3b 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-outbox-dispatch.ts +++ b/src/renderer/src/components/native-chat/structured-agent-session-outbox-dispatch.ts @@ -57,6 +57,18 @@ export function readMountedStructuredAgentSessionOutbox( ) } +/** A send left dispatching when its owner changed goes out again, under the same id. */ +export function requeueInterruptedStructuredAgentSessionDispatches( + entries: StructuredAgentSessionOutboxEntry[], + fence: number | null +): StructuredAgentSessionOutboxEntry[] { + return entries.map((entry) => + entry.state === 'dispatching' && !hasInFlightLaunchDispatch(entry, fence) + ? { ...entry, state: 'queued' as const } + : entry + ) +} + export function dispatchStructuredAgentSessionOutboxEntry(args: { next: StructuredAgentSessionOutboxEntry persisted: readonly StructuredAgentSessionOutboxEntry[] diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-fence.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-fence.test.tsx new file mode 100644 index 00000000000..61b5ea95e72 --- /dev/null +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-fence.test.tsx @@ -0,0 +1,139 @@ +// @vitest-environment happy-dom + +// A moved fence is not a reason to send anything again on a host that records every send before it +// starts an agent. There, only a Retry or a new send goes out. An older host, which restarts the +// agent inside the send and refuses it unrecorded when that fails, keeps the resend on a new fence. + +import { act, renderHook, waitFor } from '@testing-library/react' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' + +const mocks = vi.hoisted(() => ({ call: vi.fn() })) + +vi.mock('@/runtime/structured-agent-session-client', () => ({ + callStructuredAgentSession: mocks.call +})) + +import { setLocalRuntimeCapabilitiesForTests } from '@/runtime/local-runtime-capabilities' +import { useStructuredAgentSessionOutbox } from './use-structured-agent-session-outbox' + +const LOCAL_TARGET = { kind: 'local' } as const + +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((next) => { + resolve = next + }) + return { promise, resolve } +} + +function pendingResult(clientMessageId: string) { + return { + ok: true, + replayed: false, + fence: 1, + cursor: { epoch: 'epoch-1', sequence: 1 }, + value: { + clientMessageId, + submission: { + clientMessageId, + fence: 1, + payloadFingerprint: 'fingerprint', + dispatchState: 'pending' as const, + handoverRecorded: true, + providerItemId: null, + reason: null, + submittedAt: 1, + resolvedAt: null + } + } + } +} + +function sentId(call: number): string { + const id: unknown = mocks.call.mock.calls[call]?.[2].envelope.clientOperationId + return String(id) +} + +function render() { + return renderHook( + ({ fence }) => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target: LOCAL_TARGET, + fence, + submissions: [] + }), + { initialProps: { fence: 1 } } + ) +} + +/** Long enough for any effect a fence change schedules to have sent. */ +async function settle(): Promise { + await act(() => new Promise((resolve) => setTimeout(resolve, 50))) +} + +describe('an outbox on a host that accepts a send before any agent has it', () => { + beforeEach(() => { + vi.clearAllMocks() + localStorage.clear() + setLocalRuntimeCapabilitiesForTests([AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY]) + }) + + it('neither resends nor drops the answer of a send in flight when the fence moves', async () => { + const answer = deferred>() + mocks.call.mockReturnValueOnce(answer.promise) + const { result, rerender } = render() + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(1)) + // A start or restart on the host moves the fence while the send is out. + rerender({ fence: 2 }) + rerender({ fence: 3 }) + await settle() + expect(mocks.call).toHaveBeenCalledTimes(1) + + await act(async () => answer.resolve(pendingResult(sentId(0)))) + // The answer lands: the entry is the host's now, not re-queued behind a moved fence. + expect(result.current.outbox).toMatchObject([{ state: 'dispatching' }]) + await settle() + expect(mocks.call).toHaveBeenCalledTimes(1) + }) + + it('keeps a blocked head blocked across a fence change; only Retry sends it', async () => { + mocks.call.mockRejectedValueOnce(new Error('send failed')).mockResolvedValue({ ok: true }) + const { result, rerender } = render() + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(result.current.blockedClientMessageId).not.toBeNull()) + rerender({ fence: 2 }) + await settle() + expect(mocks.call).toHaveBeenCalledTimes(1) + // The failure stays on the message; the fence change neither clears nor resends it. + expect(result.current.outbox[0]?.lastFailure).toEqual({ kind: 'failed' }) + + act(() => result.current.retry(result.current.outbox[0]!.clientMessageId)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + }) +}) + +describe('an outbox on an older host', () => { + beforeEach(() => { + vi.clearAllMocks() + localStorage.clear() + setLocalRuntimeCapabilitiesForTests([]) + }) + + it('still resends a send in flight when the fence moves, as the new owner may take it', async () => { + mocks.call.mockReturnValueOnce(new Promise(() => {})).mockReturnValue(new Promise(() => {})) + const { result, rerender } = render() + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(1)) + rerender({ fence: 2 }) + + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + expect(sentId(1)).toBe(sentId(0)) + }) +}) diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-rejection-cause.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-rejection-cause.test.tsx index b6774f46416..a8b9f01b20b 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-rejection-cause.test.tsx +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-rejection-cause.test.tsx @@ -1,7 +1,9 @@ // @vitest-environment happy-dom import { act, renderHook, waitFor } from '@testing-library/react' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentJournalSubmission } from '../../../../shared/agent-session-journal-types' +import { DISPATCH_REJECTED_CANCELLED } from '../../../../shared/structured-agent-session-dispatch-rejection' const mocks = vi.hoisted(() => ({ call: vi.fn() @@ -11,7 +13,7 @@ vi.mock('@/runtime/structured-agent-session-client', () => ({ callStructuredAgentSession: mocks.call })) -import type { AgentJournalSubmission } from '../../../../shared/agent-session-journal-types' +import { setLocalRuntimeCapabilitiesForTests } from '@/runtime/local-runtime-capabilities' import { useStructuredAgentSessionOutbox } from './use-structured-agent-session-outbox' import { agentSessionWriteNoticeEnglish } from '../../../../shared/agent-session-refusal-notice' import { structuredAgentSessionAttemptFailureParts } from '../../../../shared/structured-agent-session-send-disposition' @@ -28,6 +30,28 @@ function shownFailure(entry: StructuredAgentSessionOutboxEntry | undefined): str const REASON = 'The provider stopped before it finished starting: claude stream-json exited (code 1): claude: not signed in.' +function acceptedResultFor(clientMessageId: string) { + return { + ok: true, + replayed: false, + fence: 1, + cursor: { epoch: 'epoch-1', sequence: 2 }, + value: { + clientMessageId, + submission: { + clientMessageId, + fence: 1, + payloadFingerprint: 'fingerprint', + dispatchState: 'accepted', + providerItemId: `provider-${clientMessageId}`, + reason: null, + submittedAt: 1, + resolvedAt: 1 + } + } + } +} + function rejectedResultFor(clientMessageId: string) { return { ok: true, @@ -76,14 +100,241 @@ describe('a send the host rejected because the agent never started', () => { act(() => expect(result.current.send('hello')).toBe(true)) await waitFor(() => expect(shownFailure(result.current.outbox[0])).toBe(REASON)) - expect(result.current.outbox[0]?.state).toBe('queued') - expect(result.current.blockedClientMessageId).toBe(result.current.outbox[0]?.clientMessageId) + // Settled as not delivered: it waits for Retry and holds no later message up. + expect(result.current.error).toBeNull() + expect(result.current.outbox[0]?.state).toBe('rejected') + expect(result.current.blockedClientMessageId).toBeNull() + }) + + it('sends a new message past one the host could not start the agent for, without resending it', async () => { + const message = "Claude couldn't restart: Not logged in. Please run /login." + mocks.call.mockImplementation(async (_target, _method, params) => { + const request = params as { + envelope: { clientOperationId: string } + body: { blocks: { text?: string }[] } + } + return request.body.blocks[0]?.text === 'first' + ? { ok: false, refusal: { code: 'agent_session_owner_restart_failed', message } } + : acceptedResultFor(request.envelope.clientOperationId) + }) + const { result } = renderHook(() => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target: { kind: 'local' }, + fence: 1, + submissions: [] + }) + ) + + act(() => expect(result.current.send('first')).toBe(true)) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) + const rejectedId = result.current.outbox[0]!.clientMessageId + + // The user's next message is the retry of the start: it goes out on its own. + act(() => expect(result.current.send('second')).toBe(true)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + await act(() => new Promise((resolve) => setTimeout(resolve, 50))) + + const sent = mocks.call.mock.calls.map( + (call) => (call[2] as { body?: { blocks?: { text?: string }[] } })?.body?.blocks?.[0]?.text + ) + expect(sent).toEqual(['first', 'second']) + expect(result.current.outbox.map((entry) => [entry.clientMessageId, entry.state])).toEqual([ + [rejectedId, 'rejected'] + ]) + }) + + it('keeps a message the host accepted and then could not deliver, with its reason and Retry', async () => { + const reason = "Codex couldn't restart: spawn codex ENOENT." + mocks.call.mockImplementation( + async ( + _target: unknown, + _method: unknown, + params: { envelope: { clientOperationId: string } } + ) => pendingResultFor(params.envelope.clientOperationId) + ) + const target = { kind: 'local' } as const + const { result, rerender } = renderHook( + (props: { submissions: AgentJournalSubmission[] }) => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: props.submissions + }), + { initialProps: { submissions: NO_SUBMISSIONS } } + ) + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('dispatching')) + const id = result.current.outbox[0]!.clientMessageId + + rerender({ + submissions: [{ ...pendingResultFor(id).value.submission, dispatchState: 'rejected', reason }] + }) + + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) + expect(shownFailure(result.current.outbox[0])).toBe(reason) + expect(result.current.error).toBeNull() + expect(result.current.blockedClientMessageId).toBeNull() + + // Retry is a new message with the same text: a fresh id, sent once. + act(() => result.current.retry(id)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + const retried: { + envelope: { clientOperationId: string } + body: { blocks: { text?: string }[] } + } = mocks.call.mock.calls[1]![2] + expect(retried.envelope.clientOperationId).not.toBe(id) + expect(retried.body.blocks[0]?.text).toBe('hello') + }) + + it('says nothing when a Stop withdrew the message', async () => { + mocks.call.mockImplementation( + async ( + _target: unknown, + _method: unknown, + params: { envelope: { clientOperationId: string } } + ) => pendingResultFor(params.envelope.clientOperationId) + ) + const target = { kind: 'local' } as const + const { result, rerender } = renderHook( + (props: { submissions: AgentJournalSubmission[] }) => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: props.submissions + }), + { initialProps: { submissions: NO_SUBMISSIONS } } + ) + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('dispatching')) + const id = result.current.outbox[0]!.clientMessageId + + rerender({ + submissions: [ + { + ...pendingResultFor(id).value.submission, + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_CANCELLED + } + ] + }) + + await waitFor(() => expect(result.current.outbox).toEqual([])) + expect(result.current.error).toBeNull() + }) + + it('reads a message rejected while the chat was closed as not sent, and sends past it', async () => { + const reason = "Codex couldn't restart: spawn codex ENOENT." + mocks.call.mockImplementation( + async ( + _target: unknown, + _method: unknown, + params: { envelope: { clientOperationId: string } } + ) => pendingResultFor(params.envelope.clientOperationId) + ) + const target = { kind: 'local' } as const + const first = renderHook(() => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: NO_SUBMISSIONS + }) + ) + act(() => expect(first.result.current.send('hello')).toBe(true)) + await waitFor(() => expect(first.result.current.outbox[0]?.state).toBe('dispatching')) + const id = first.result.current.outbox[0]!.clientMessageId + first.unmount() + + // Reopened after the start failed, or after a quit settled the message as not sent. + const rejected = [ + { ...pendingResultFor(id).value.submission, dispatchState: 'rejected' as const, reason } + ] + const { result } = renderHook(() => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: rejected + }) + ) + + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) + expect(shownFailure(result.current.outbox[0])).toBe(reason) + act(() => expect(result.current.send('second')).toBe(true)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + }) + + it('keeps the rejection when the journal settles the message before the send answers', async () => { + const reason = "Codex couldn't restart: spawn codex ENOENT." + let answer: (value: unknown) => void = () => undefined + mocks.call.mockImplementationOnce( + () => + new Promise((resolve) => { + answer = resolve + }) + ) + const target = { kind: 'local' } as const + const { result, rerender } = renderHook( + (props: { submissions: AgentJournalSubmission[] }) => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: props.submissions + }), + { initialProps: { submissions: NO_SUBMISSIONS } } + ) + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('dispatching')) + const id = result.current.outbox[0]!.clientMessageId + + // A start refused at once: the rejection frame lands before the send's own `pending` answer. + rerender({ + submissions: [{ ...pendingResultFor(id).value.submission, dispatchState: 'rejected', reason }] + }) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) + await act(async () => answer(pendingResultFor(id))) + + expect(result.current.outbox[0]?.state).toBe('rejected') + expect(shownFailure(result.current.outbox[0])).toBe(reason) + expect(result.current.error).toBeNull() }) }) +const NO_SUBMISSIONS: AgentJournalSubmission[] = [] + +function pendingResultFor(clientMessageId: string) { + const submission: AgentJournalSubmission = { + clientMessageId, + fence: 1, + payloadFingerprint: 'fingerprint', + dispatchState: 'pending', + providerItemId: null, + reason: null, + submittedAt: 1, + resolvedAt: null, + handoverRecorded: true + } + return { + ok: true, + replayed: false, + fence: 1, + cursor: { epoch: 'epoch-1', sequence: 2 }, + value: { + clientMessageId, + submission + } + } +} + // Stable across renders, as a mounted pane's target is. const LOCAL_TARGET = { kind: 'local' } as const -const NO_SUBMISSIONS: AgentJournalSubmission[] = [] function submission( clientMessageId: string, @@ -101,10 +352,16 @@ function submission( } } +// An older host: it restarts the agent inside the send, so a new fence is its word to send again. describe('a send refused while its agent restarted', () => { beforeEach(() => { vi.clearAllMocks() localStorage.clear() + setLocalRuntimeCapabilitiesForTests([]) + }) + + afterEach(() => { + setLocalRuntimeCapabilitiesForTests(null) }) // The live order: the restart takes seconds, so the journal settles the resend before its reply. diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.test.tsx index 12c190291b9..04b25f93e8a 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.test.tsx +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.test.tsx @@ -248,7 +248,7 @@ describe('useStructuredAgentSessionOutbox', () => { ) act(() => expect(result.current.send('hello')).toBe(true)) - await waitFor(() => expect(result.current.outbox[0]?.state).toBe('queued')) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) const firstId = (mocks.call.mock.calls[0]![2] as { envelope: { clientOperationId: string } }) .envelope.clientOperationId const retryId = result.current.outbox[0]!.clientMessageId @@ -558,7 +558,9 @@ describe('useStructuredAgentSessionOutbox', () => { expect(mocks.call).toHaveBeenCalledOnce() expect(result.current.outbox).toHaveLength(1) - expect(result.current.blockedClientMessageId).toBe(result.current.outbox[0]?.clientMessageId) + // Never sent, and never re-sent on its own: it waits for Retry and holds nothing up. + expect(result.current.outbox[0]?.state).toBe('rejected') + expect(result.current.blockedClientMessageId).toBeNull() // Settled, not pending: the refused id never ran, so a Retry is a new operation. const sentId: unknown = mocks.call.mock.calls[0]![2].envelope.clientOperationId const retryId = result.current.outbox[0]!.clientMessageId @@ -775,8 +777,8 @@ describe('useStructuredAgentSessionOutbox', () => { // A refused write is answered, not doubted: the entry parks with its rejection rather than // under the "delivery is unconfirmed" banner. The disposition tests pin its words. await waitFor(() => expect(result.current.outbox[0]?.lastFailure?.kind).toBe('rejected')) - expect(result.current.outbox[0]?.state).toBe('queued') - expect(result.current.blockedClientMessageId).toBe(firstId) + expect(result.current.outbox[0]?.state).toBe('rejected') + expect(result.current.blockedClientMessageId).toBeNull() // Retry immediately, before the journal subscription can publish the rejected row. act(() => result.current.retry(firstId)) diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.ts index 17a9e2769f3..254c936cf63 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.ts @@ -7,15 +7,19 @@ import { reconcileStructuredAgentSessionOutbox, type StructuredAgentSessionOutboxEntry } from '../../../../shared/structured-agent-session-outbox' -import type { StructuredAgentSessionSendDisposition } from '../../../../shared/structured-agent-session-send-disposition' +import { + journalAnswersInFlightSend, + type StructuredAgentSessionSendDisposition +} from '../../../../shared/structured-agent-session-send-disposition' import type { RuntimeClientTarget } from '@/runtime/runtime-rpc-client' import { readOutbox, writeOutbox } from './structured-agent-session-outbox-storage' import { dispatchStructuredAgentSessionOutboxEntry, - hasInFlightLaunchDispatch, - readMountedStructuredAgentSessionOutbox + readMountedStructuredAgentSessionOutbox, + requeueInterruptedStructuredAgentSessionDispatches } from './structured-agent-session-outbox-dispatch' import { getStructuredAgentLaunchPromptDispatch } from '@/lib/structured-agent-session-launch-prompt' +import { useStructuredAgentSessionOutboxOwnerChange } from '@/runtime/structured-agent-session-accepted-send-capability' import { createBrowserUuid } from '@/lib/browser-uuid' export function structuredSessionOperationId(): string { @@ -25,8 +29,8 @@ export function structuredSessionOperationId(): string { const UNCONFIRMED_PROBE_BASE_DELAY_MS = 1_000 /** No attempt ceiling: a transport outage outlives any fixed budget, and giving up * restores the wedge this fixes. Growth caps the rate at one status query per 16s. - * A refusal that blocks the head still ends probing until a fence change or a manual - * Retry, because the entry leaves `unconfirmed` -- pre-existing, not closed here. */ + * A refusal that blocks the head still ends probing until a manual Retry (or, on an older + * host, a fence change), because the entry leaves `unconfirmed`. */ const UNCONFIRMED_PROBE_MAX_DELAY_MS = 16_000 export function useStructuredAgentSessionOutbox(args: { @@ -36,7 +40,8 @@ export function useStructuredAgentSessionOutbox(args: { submissions: readonly AgentJournalSubmission[] }) { const { fence, sessionId, submissions, target } = args - const targetKey = target.kind === 'local' ? 'local' : `environment:${target.environmentId}` + // What resends, unblocks and drops a send in flight besides a Retry or a new send; see the hook. + const owner = useStructuredAgentSessionOutboxOwnerChange(target, fence) const [outbox, setOutbox] = useState(() => readMountedStructuredAgentSessionOutbox(sessionId, fence, readOutbox) ) @@ -68,19 +73,15 @@ export function useStructuredAgentSessionOutbox(args: { blockedIdRef.current = null retryWithFreshClientMessageIdRef.current = null probeAttemptsRef.current = { id: null, attempts: 0 } - }, [fence, sessionId, targetKey]) + }, [owner.ownerChange, owner.targetKey, sessionId]) useEffect(() => { const sessionChanged = outboxSessionRef.current !== sessionId outboxSessionRef.current = sessionId const current = sessionChanged - ? readMountedStructuredAgentSessionOutbox(sessionId, fence, readOutbox) + ? readMountedStructuredAgentSessionOutbox(sessionId, owner.fenceRef.current, readOutbox) : outboxRef.current - const next = current.map((entry) => - entry.state === 'dispatching' && !hasInFlightLaunchDispatch(entry, fence) - ? { ...entry, state: 'queued' as const } - : entry - ) + const next = requeueInterruptedStructuredAgentSessionDispatches(current, owner.fenceRef.current) if ( sessionChanged || next.some((entry, index) => entry !== current[index]) || @@ -90,7 +91,7 @@ export function useStructuredAgentSessionOutbox(args: { setOutbox(next) writeOutbox(sessionId, next) } - }, [fence, sessionId, target]) + }, [owner.fenceRef, owner.ownerChange, sessionId, target]) useEffect(() => { const current = outboxRef.current @@ -103,7 +104,7 @@ export function useStructuredAgentSessionOutbox(args: { .map((submission) => submission.clientMessageId) ) const next = reconcileStructuredAgentSessionOutbox(current, submissions) - const admittedInFlight = inFlightIdRef.current !== null && hostOwns.has(inFlightIdRef.current) + const admittedInFlight = journalAnswersInFlightSend(submissions, inFlightIdRef.current) if ( admittedInFlight || next.some((entry, index) => entry !== current[index]) || @@ -240,7 +241,7 @@ export function useStructuredAgentSessionOutbox(args: { const probeSettled = probeId !== null && submissions.some((submission) => submission.clientMessageId === probeId) useEffect(() => { - if (probeId === null || probeSettled || fence === null) { + if (probeId === null || probeSettled || !owner.attached) { return } const attempts = probeAttemptsRef.current.id === probeId ? probeAttemptsRef.current.attempts : 0 @@ -257,7 +258,7 @@ export function useStructuredAgentSessionOutbox(args: { Math.min(UNCONFIRMED_PROBE_BASE_DELAY_MS * 2 ** attempts, UNCONFIRMED_PROBE_MAX_DELAY_MS) ) return () => clearTimeout(timer) - }, [fence, probeId, probeSettled, sessionId, targetKey]) + }, [owner.attached, owner.ownerChange, owner.targetKey, probeId, probeSettled, sessionId]) const send = useCallback( (text: string, attachments: readonly { path: string; previewUri: string }[] = []): boolean => { diff --git a/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.test.tsx b/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.test.tsx new file mode 100644 index 00000000000..acdfdbb0323 --- /dev/null +++ b/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.test.tsx @@ -0,0 +1,51 @@ +// @vitest-environment happy-dom + +import { renderHook, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY } from '../../../shared/protocol-version' + +const mocks = vi.hoisted(() => ({ supports: vi.fn() })) + +vi.mock('./runtime-rpc-client', () => ({ + runtimeEnvironmentSupportsCapability: mocks.supports +})) + +import { setLocalRuntimeCapabilitiesForTests } from './local-runtime-capabilities' +import { useStructuredAgentSessionHostAcceptsSend } from './structured-agent-session-accepted-send-capability' + +afterEach(() => { + setLocalRuntimeCapabilitiesForTests(null) + vi.clearAllMocks() +}) + +describe('whether a host accepts a send before any agent has it', () => { + it('reads the local host from its advertised capabilities', () => { + setLocalRuntimeCapabilitiesForTests([AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY]) + const { result } = renderHook(() => useStructuredAgentSessionHostAcceptsSend({ kind: 'local' })) + expect(result.current).toBe(true) + }) + + it('treats a local host that does not advertise it as an older one', () => { + setLocalRuntimeCapabilitiesForTests([]) + const { result } = renderHook(() => useStructuredAgentSessionHostAcceptsSend({ kind: 'local' })) + expect(result.current).toBe(false) + }) + + it('asks a remote host, and reads a failed probe as an older host', async () => { + mocks.supports.mockResolvedValueOnce(true).mockRejectedValueOnce(new Error('offline')) + const accepts = renderHook(() => + useStructuredAgentSessionHostAcceptsSend({ kind: 'environment', environmentId: 'env-1' }) + ) + await waitFor(() => expect(accepts.result.current).toBe(true)) + expect(mocks.supports).toHaveBeenCalledWith( + 'env-1', + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY + ) + + const offline = renderHook(() => + useStructuredAgentSessionHostAcceptsSend({ kind: 'environment', environmentId: 'env-2' }) + ) + await waitFor(() => expect(mocks.supports).toHaveBeenCalledTimes(2)) + expect(offline.result.current).toBe(false) + }) +}) diff --git a/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.ts b/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.ts new file mode 100644 index 00000000000..c6e130b8bd5 --- /dev/null +++ b/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.ts @@ -0,0 +1,87 @@ +import { useEffect, useLayoutEffect, useRef, useState, type RefObject } from 'react' +import { AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY } from '../../../shared/protocol-version' +import type { RuntimeClientTarget } from './runtime-client-target' +import { + ensureLocalRuntimeCapabilities, + readLocalRuntimeCapabilitiesOrUnknown +} from './local-runtime-capabilities' +import { runtimeEnvironmentSupportsCapability } from './runtime-rpc-client' + +function targetKey(target: RuntimeClientTarget): string { + return target.kind === 'local' ? 'local' : `environment:${target.environmentId}` +} + +/** + * Whether the host answers a send at acceptance and never refuses one because its agent could not + * start. Such a host records every send before it starts anything, so nothing about a moved fence + * calls for a resend. False until the host has said so: an older host is handled as it always was. + */ +export function useStructuredAgentSessionHostAcceptsSend(target: RuntimeClientTarget): boolean { + const key = targetKey(target) + const [answer, setAnswer] = useState<{ key: string; accepts: boolean }>(() => ({ + key, + accepts: + target.kind === 'local' && + (readLocalRuntimeCapabilitiesOrUnknown()?.includes( + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY + ) ?? + false) + })) + const environmentId = target.kind === 'environment' ? target.environmentId : null + useEffect(() => { + let cancelled = false + const probe = + environmentId === null + ? ensureLocalRuntimeCapabilities().then( + (capabilities) => + capabilities?.includes(AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY) ?? false + ) + : runtimeEnvironmentSupportsCapability( + environmentId, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY + ) + void probe + .catch(() => false) + .then((accepts) => { + if (!cancelled) { + setAnswer((current) => + current.key === key && current.accepts === accepts ? current : { key, accepts } + ) + } + }) + return () => { + cancelled = true + } + }, [environmentId, key]) + return answer.key === key && answer.accepts +} + +/** + * When an outbox treats its owner as changed: resending a send in flight under the same id, + * dropping that send's answer, and unblocking a refused head. An older host restarts the agent + * inside the send and refuses it, unrecorded, when that fails, so a new fence is its only word that + * another try may land. A host that accepts first records every send before it starts anything, + * so a moved fence means nothing there, and only a Retry or a new send goes out. + */ +export function useStructuredAgentSessionOutboxOwnerChange( + target: RuntimeClientTarget, + fence: number | null +): { + ownerChange: number | null + attached: boolean + fenceRef: RefObject + targetKey: string +} { + const acceptsSend = useStructuredAgentSessionHostAcceptsSend(target) + // Read by effects that run on an owner change, not on every fence move. + const fenceRef = useRef(fence) + useLayoutEffect(() => { + fenceRef.current = fence + }, [fence]) + return { + ownerChange: acceptsSend ? null : fence, + attached: fence !== null, + fenceRef, + targetKey: targetKey(target) + } +} diff --git a/src/shared/agent-session-journal-schemas.ts b/src/shared/agent-session-journal-schemas.ts index b7a80634bf8..04f7183eb3f 100644 --- a/src/shared/agent-session-journal-schemas.ts +++ b/src/shared/agent-session-journal-schemas.ts @@ -304,7 +304,9 @@ export const AgentJournalSubmissionSchema = z.object({ reason: z.string().nullable(), submittedAt: z.number(), resolvedAt: z.number().nullable(), - recovered: z.literal(true).optional() + recovered: z.literal(true).optional(), + handoverRecorded: z.literal(true).optional(), + handedOverAt: z.number().optional() }) export function isAdmissibleAgentJournalItemBody(value: unknown): value is AgentJournalItemBody { diff --git a/src/shared/agent-session-journal-types.ts b/src/shared/agent-session-journal-types.ts index f5bd250c0c9..0a6cb655cd8 100644 --- a/src/shared/agent-session-journal-types.ts +++ b/src/shared/agent-session-journal-types.ts @@ -364,6 +364,13 @@ export type AgentJournalSubmission = { /** Set when crash reconciliation resolved the dispatch, not the provider. A live * `unknown` is a send still outstanding; a recovered one outlived its writer. */ recovered?: true + /** The host accepted this send to hand over later; absent on sends dispatched as they were + * recorded (older hosts). With no `handedOverAt` yet, a pending one is still queued. */ + handoverRecorded?: true + /** When the host handed it to the provider (its `dispatch{pending}` row). */ + handedOverAt?: number + /** Host-only: the submission row's sequence, which tells which host process accepted it. */ + acceptedSequence?: number } /** Durable answer to "did my send land?", keyed by client message id. Only an diff --git a/src/shared/agent-session-mutation-envelope.ts b/src/shared/agent-session-mutation-envelope.ts index b591d82af2e..dd409d406d1 100644 --- a/src/shared/agent-session-mutation-envelope.ts +++ b/src/shared/agent-session-mutation-envelope.ts @@ -92,6 +92,9 @@ export function admitAgentSessionMutation(input: { /** Decision from the durable ledger, evaluated under `hostFingerprint`. */ ledger: AgentSessionOperationDecision lease: AgentSessionLease + /** A write to the conversation, not to the provider child: a send is accepted and a Stop + * withdraws queued messages whoever owns the child, so the lease does not admit them. */ + conversationWrite?: true }): AgentSessionMutationAdmission { const { envelope, lease, ledger } = input const mismatch = agentSessionFingerprintConflict(envelope, input.hostFingerprint) @@ -110,6 +113,9 @@ export function admitAgentSessionMutation(input: { if (ledger.decision === 'replay') { return { decision: 'replay', row: ledger.row } } + if (input.conversationWrite) { + return { decision: 'admit', row: ledger.row } + } const leaseRefusal = refuseUnlessWriterAdmitted(lease) if (leaseRefusal) { return { decision: 'refused', refusal: leaseRefusal } diff --git a/src/shared/agent-session-queued-submission.ts b/src/shared/agent-session-queued-submission.ts new file mode 100644 index 00000000000..6295897fd54 --- /dev/null +++ b/src/shared/agent-session-queued-submission.ts @@ -0,0 +1,13 @@ +import type { AgentJournalSubmission } from './agent-session-journal-types' + +/** Accepted by the host and not yet handed to the provider: provably unwritten, so every way out + * of this state is delivery or a rejection, never doubt. */ +export function isQueuedAgentJournalSubmission( + submission: Pick +): boolean { + return ( + submission.handoverRecorded === true && + submission.dispatchState === 'pending' && + submission.handedOverAt === undefined + ) +} diff --git a/src/shared/agent-session-wire.ts b/src/shared/agent-session-wire.ts index 7118a2237eb..f5ceb625770 100644 --- a/src/shared/agent-session-wire.ts +++ b/src/shared/agent-session-wire.ts @@ -295,7 +295,7 @@ export type AgentSessionAttachResult = { sessionId: string fence: number page: AgentSessionHistoryPage - /** Submissions the crash boundary settled as `unknown` while attaching. */ + /** Submissions a crash boundary left `unknown` that provider history could not decide. */ unconfirmedClientMessageIds: string[] /** The host-owned id of the tab showing this chat, when it has one. Absent from older hosts. */ tabId?: string diff --git a/src/shared/protocol-version.ts b/src/shared/protocol-version.ts index db9d7f1b544..dd992e32ea5 100644 --- a/src/shared/protocol-version.ts +++ b/src/shared/protocol-version.ts @@ -165,6 +165,11 @@ export const STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY = 'agent-session.struct // clients skip the host's bounded best-effort settlement observation. export const AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY = 'agent-session.pending-send-result.v1' as const +// Why: a send is now answered once the host accepts it, before any agent has it. A client without +// this cannot show a message rejected after that answer, so the host holds its reply until the +// message is handed over or rejected. +export const AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY = + 'agent-session.accepted-send.v1' as const // Why: paired clients advertise Claude-structured support so the host can gate its agent-specific // journal and lifecycle surfaces independently from Codex support. export const CLAUDE_STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY = @@ -302,6 +307,7 @@ export const NATIVE_REMOTE_RUNTIME_CLIENT_CAPABILITIES = [ export const ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES = [ ...NATIVE_REMOTE_RUNTIME_CLIENT_CAPABILITIES, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, BROWSER_CLIENT_HOST_RUNTIME_CAPABILITY, BROWSER_CLIENT_PAGE_METADATA_RUNTIME_CAPABILITY, // Why: only the renderer runs the retirement-proof ledger; CLI and mobile must keep full lists. @@ -372,6 +378,9 @@ export const RUNTIME_CAPABILITIES = [ AGENT_SESSION_KEYBOARD_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, + // The host side: it accepts a send before any agent has it, and a Stop with no writer before a + // turn starts, so a client may gate on either. + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_HOLD_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_REVEAL_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_RESUME_HISTORY_RUNTIME_CAPABILITY, diff --git a/src/shared/structured-agent-session-dispatch-rejection.ts b/src/shared/structured-agent-session-dispatch-rejection.ts index 14e197325d8..a88dad5b16e 100644 --- a/src/shared/structured-agent-session-dispatch-rejection.ts +++ b/src/shared/structured-agent-session-dispatch-rejection.ts @@ -29,6 +29,12 @@ export const DISPATCH_REJECTED_CODEX_QUEUE_FULL = 'codex structured dispatch que /** The provider confirmed a queued frame was withdrawn before execution. */ export const DISPATCH_REJECTED_CANCELLED = 'provider_cancelled_before_start' +/** Accepted by a host process that ended before handing it to any provider. */ +export const DISPATCH_REJECTED_HOST_RESTARTED = 'host_restarted_before_delivery' + +/** Accepted, then the provider was closed before the message was handed to it. */ +export const DISPATCH_REJECTED_PROVIDER_CLOSED = 'provider_closed_before_delivery' + export function dispatchWriteFailureReason(error: unknown): string { const detail = error instanceof Error ? error.message : String(error) return `${DISPATCH_REJECTED_WRITE_FAILED}: ${detail}` @@ -54,6 +60,8 @@ export function dispatchRejectionReasonIsInternal(reason: string | null | undefi dispatchRejectionWasTransportWriteFailure(reason) || reason === DISPATCH_REJECTED_QUEUE_FULL || reason === DISPATCH_REJECTED_CODEX_QUEUE_FULL || - reason === DISPATCH_REJECTED_CANCELLED + reason === DISPATCH_REJECTED_CANCELLED || + reason === DISPATCH_REJECTED_HOST_RESTARTED || + reason === DISPATCH_REJECTED_PROVIDER_CLOSED ) } diff --git a/src/shared/structured-agent-session-outbox.ts b/src/shared/structured-agent-session-outbox.ts index ef5d4cfd7d7..6c820921162 100644 --- a/src/shared/structured-agent-session-outbox.ts +++ b/src/shared/structured-agent-session-outbox.ts @@ -11,7 +11,13 @@ import type { import { structuredAgentSessionPayloadFingerprint } from './structured-agent-session-mutation' import { DISPATCH_REJECTED_CANCELLED } from './structured-agent-session-dispatch-rejection' -export type StructuredAgentSessionOutboxState = 'queued' | 'dispatching' | 'unconfirmed' +/** `rejected`: the host settled the send as not delivered. The drain never sends it again on its + * own and nothing queues behind it; only the user's Retry does. */ +export type StructuredAgentSessionOutboxState = + | 'queued' + | 'dispatching' + | 'unconfirmed' + | 'rejected' export type StructuredAgentSessionOutboxEntry = { clientMessageId: string @@ -126,10 +132,12 @@ export function requeueStructuredAgentSessionSendRefusal( ) { return { ...entry, state: 'queued' } } + // Only here is the refusal proof the message never landed: an earlier attempt under this id, or + // one whose delivery was in doubt, may have, so those stay queued behind the block. return { ...entry, clientMessageId: createOperationId(), - state: 'queued', + state: 'rejected', lastAttemptAt: null, retryAfterUnknownSubmittedAt: null } @@ -154,6 +162,21 @@ export function reconcileStructuredAgentSessionOutbox( if (submission?.dispatchState === 'pending') { return entry.state === 'dispatching' ? [entry] : [{ ...entry, state: 'dispatching' as const }] } + // Accepted, then not delivered — the agent never started, or its start was refused. The text + // and why stay here for the user's Retry, and nothing queues behind it. `unconfirmed` is how a + // remount reads an entry it left dispatching; the journal has since answered it. + if ( + submission?.dispatchState === 'rejected' && + (entry.state === 'dispatching' || entry.state === 'unconfirmed') + ) { + return [ + { + ...entry, + state: 'rejected' as const, + lastFailure: { kind: 'rejected' as const, reason: submission.reason } + } + ] + } if ( submission?.dispatchState === 'unknown' && entry.retryAfterUnknownSubmittedAt !== -1 && @@ -185,6 +208,10 @@ export function admitStructuredAgentSessionOutboxEntry( blockedClientMessageId: string | null ): StructuredAgentSessionOutboxAdmission { for (const entry of entries) { + // It can no longer land, so nothing it could be reordered around; it waits for Retry. + if (entry.state === 'rejected') { + continue + } if (entry.state === 'unconfirmed' || entry.clientMessageId === blockedClientMessageId) { return { state: 'blocked', entry } } @@ -214,7 +241,7 @@ export function parseStructuredAgentSessionOutboxEntry( !Array.isArray(body.blocks) || !Array.isArray(entry.previewUris) || !entry.previewUris.every((uri) => typeof uri === 'string') || - !['queued', 'dispatching', 'unconfirmed'].includes(entry.state ?? '') + !['queued', 'dispatching', 'unconfirmed', 'rejected'].includes(entry.state ?? '') ) { return null } diff --git a/src/shared/structured-agent-session-send-disposition.ts b/src/shared/structured-agent-session-send-disposition.ts index 72409b8e928..11ba9703b83 100644 --- a/src/shared/structured-agent-session-send-disposition.ts +++ b/src/shared/structured-agent-session-send-disposition.ts @@ -7,6 +7,7 @@ // the refs, the React state and the storage write, and nothing else decides an // entry's state. +import type { AgentJournalSubmission } from './agent-session-journal-types' import type { AgentSessionMutationResult, AgentSessionSendResult } from './agent-session-wire' import { agentSessionRefusalFailure, @@ -92,6 +93,18 @@ function refusedRedelivery( ) } +/** Whether the journal already answers a send still in flight, so its own reply adds nothing: the + * host holds the message, or rejected it — a later `pending` reply must not undo that. */ +export function journalAnswersInFlightSend( + submissions: readonly AgentJournalSubmission[], + clientMessageId: string | null +): boolean { + return submissions.some( + (submission) => + submission.clientMessageId === clientMessageId && submission.dispatchState !== 'unknown' + ) +} + /** * What to put on screen for a rejection. * @@ -166,12 +179,17 @@ export function disposeStructuredAgentSessionSendResult( ) : candidate ) + const refused = entries[refusedIndex] return { entries, error: null, // Read back by index rather than from the input: a refusal can rotate the id, and the // refused entry is not always the head now that an admitted one no longer holds the queue. - blockedClientMessageId: entries[refusedIndex]?.clientMessageId ?? null, + // A rejected one holds nothing: it can no longer land, and it keeps its own Retry. + blockedClientMessageId: + !refused || refused.state === 'rejected' + ? input.blockedClientMessageId + : refused.clientMessageId, retryWithFreshClientMessageId: null } } @@ -194,12 +212,12 @@ export function disposeStructuredAgentSessionSendResult( } if (submission.dispatchState === 'rejected') { return { - entries: replaceEntryState(input, 'queued', { + entries: replaceEntryState(input, 'rejected', { kind: 'rejected', reason: submission.reason }), error: null, - blockedClientMessageId: input.entry.clientMessageId, + blockedClientMessageId: input.blockedClientMessageId, retryWithFreshClientMessageId: input.entry.clientMessageId } } diff --git a/src/shared/structured-agent-session-unanswered-dispatch.ts b/src/shared/structured-agent-session-unanswered-dispatch.ts index 36c70a357cb..002db36cb57 100644 --- a/src/shared/structured-agent-session-unanswered-dispatch.ts +++ b/src/shared/structured-agent-session-unanswered-dispatch.ts @@ -1,4 +1,5 @@ import type { AgentJournalSubmission } from './agent-session-journal-types' +import { isQueuedAgentJournalSubmission } from './agent-session-queued-submission' /** One send the provider has neither opened a turn for nor refused; the rule is explained on * `hasUnansweredStructuredAgentSessionDispatch`, which asks it of every send. */ @@ -6,6 +7,10 @@ export function isUnansweredStructuredAgentSessionDispatch( submission: AgentJournalSubmission, currentFence?: number | null ): boolean { + if (isQueuedAgentJournalSubmission(submission)) { + // Accepted and still owed to whichever child the host starts next, whatever the fence. + return true + } return ( (currentFence == null || submission.fence >= currentFence) && (submission.dispatchState === 'pending' || diff --git a/tests/e2e/codex-child-approval-activity-row.unit.test.ts b/tests/e2e/codex-child-approval-activity-row.unit.test.ts index b9210bf79e8..71bb0fddce1 100644 --- a/tests/e2e/codex-child-approval-activity-row.unit.test.ts +++ b/tests/e2e/codex-child-approval-activity-row.unit.test.ts @@ -143,7 +143,9 @@ async function openHost() { journalDir: join(root, SESSION) }) const feed = new StructuredAgentSessionStatusFeed({ - sessions: new Map([[SESSION, indexedStatusFeedSession({ journal, hasProviderChild: true })]]), + sessions: new Map([ + [SESSION, indexedStatusFeedSession({ journal, child: { phase: 'ready' } })] + ]), getRecord: () => null, now: () => 1, readBackgroundTasks: () => ({ state: 'monitoring', tasks: [] }) diff --git a/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts b/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts index b038a558517..193216e3c44 100644 --- a/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts +++ b/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts @@ -22,6 +22,7 @@ import { AgentSessionRecordStore } from '../../../src/main/runtime/agent-session import { RuntimeSubscriptionRegistry } from '../../../src/main/runtime/runtime-subscription-registry' import type { AgentSessionSubscribeEvent } from '../../../src/shared/agent-session-wire' import { + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, AGENT_SESSION_QUESTION_ANSWERS_RUNTIME_CAPABILITY, AGENT_SESSION_REWIND_RUNTIME_CAPABILITY, @@ -619,6 +620,10 @@ describe('cross-version structured agent sessions', () => { let store: AgentSessionRecordStore let runtime: unknown + /** Holds a provider start open, so a reply's timing can be read against it. */ + let startGate: Promise = Promise.resolve() + let starts = 0 + /** Phase 2 owns provider processes; the adapter is the only stub here. */ function adapter(): StructuredAgentSessionAdapter { return { @@ -626,23 +631,27 @@ describe('cross-version structured agent sessions', () => { // `supportsLocation`, which this fake also lacks, so the client-supplied-location gate // refused for the fake's silence rather than for the location. supportsCreate: () => true, - acquire: async ({ fence }) => ({ - process: { - hostId: 'local', - pid: 4242, - processStartTimeMs: 1_700_000_000_000, - spawnToken: store.getRecord(SESSION)?.lease.reservedSpawnToken ?? 'spawn-a' - }, - link: { - linkId: `link-${fence}`, - handle: { provider: 'codex', threadId: THREAD }, - // A restarted host re-proves the thread it inherited; only the first - // owner of a session may claim to have created it. - origin: store.getRecord(SESSION)?.providerHandleChain.length ? 'resumed' : 'created', - mintedAtFence: fence, - observedAt: NOW + acquire: async ({ fence }) => { + starts += 1 + await startGate + return { + process: { + hostId: 'local', + pid: 4242, + processStartTimeMs: 1_700_000_000_000, + spawnToken: store.getRecord(SESSION)?.lease.reservedSpawnToken ?? 'spawn-a' + }, + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex', threadId: THREAD }, + // A restarted host re-proves the thread it inherited; only the first + // owner of a session may claim to have created it. + origin: store.getRecord(SESSION)?.providerHandleChain.length ? 'resumed' : 'created', + mintedAtFence: fence, + observedAt: NOW + } } - }), + }, dispatch: async () => ({ state: 'accepted', providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-1', ordinal: 1 } @@ -697,14 +706,18 @@ describe('cross-version structured agent sessions', () => { return reattached } - async function call(method: string, params: unknown): Promise { + async function call( + method: string, + params: unknown, + clientCapabilities: readonly string[] = [STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY] + ): Promise { return callBuild( current, method, params, { clientKind: 'runtime', - clientCapabilities: [STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY], + clientCapabilities, clientId: 'paired-device-1', connectionId: 'connection-1' }, @@ -723,6 +736,7 @@ describe('cross-version structured agent sessions', () => { beforeEach(async () => { resetOperationIds() + startGate = Promise.resolve() root = await mkdtemp(join(tmpdir(), 'orca-cross-version-agent-session-')) runtime = runtimeStub() await bootHost('a') @@ -776,5 +790,58 @@ describe('cross-version structured agent sessions', () => { fence: reattached.fence }) }) + + // A released client answers a send's `pending` as delivered-or-refused; it has no way to show + // a rejection that arrives after it. So it is answered once the message is handed over, while + // a client that advertises accepted sends is answered at acceptance, start or no start (W9). + it('holds the send reply of a released client until the handover, and answers a current one at once', async () => { + const released = [ + STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY, + AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY + ] + expect(baseline.capabilities).not.toContain(AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY) + const created = await answer('agentSession.create', createIntentParams()) + await bootHost('b') + let open = (): void => undefined + startGate = new Promise((resolve) => (open = resolve)) + + let answered = false + const releasedReply = call( + 'agentSession.send', + sendParams('released', created.fence), + released + ).finally(() => (answered = true)) + // The start that delivers it is under way, and the reply still waits for it. + const before = starts + await vi.waitFor(() => expect(starts).toBeGreaterThan(before)) + await new Promise((resolve) => setTimeout(resolve, 20)) + expect(answered).toBe(false) + open() + const [reply] = await releasedReply + // Handed over, whatever the provider has said since. + expect(reply).toMatchObject({ + ok: true, + result: { value: { submission: { handedOverAt: expect.any(Number) } } } + }) + + const restarted = await bootHost('c') + startGate = new Promise((resolve) => (open = resolve)) + const currentReply = await call('agentSession.send', sendParams('current', created.fence), [ + ...released, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY + ]) + expect(currentReply[0]).toMatchObject({ + ok: true, + result: { value: { submission: { dispatchState: 'pending', handoverRecorded: true } } } + }) + open() + await vi.waitFor(() => + expect( + restarted + .journalSnapshot(SESSION) + .submissions.every((row) => row.dispatchState !== 'pending' || row.handedOverAt) + ).toBe(true) + ) + }) }) })