fix(native-chat): a message is accepted, then delivered

A send to a chat with no running agent restarted the agent inside the send
call, before the message was recorded, so the client waited for the whole
start and a failed restart refused the message. Claude held prompts sent
during startup, and those could settle as "unconfirmed".

A send is now accepted inside the session's serialized queue: one ledger row
and one submission row marked handoverRecorded, published, answered pending.
A per-session delivery loop exists while a message is queued. It starts the
agent through the same serialized attach a hold uses, waits outside the queue
for a Claude child to prove its start, and hands the oldest queued message
over as its own serialized step, writing dispatch{pending} before the adapter
call. A start it needed and did not get writes one error-tone row and rejects
every queued message with the same words; a start Stop cancelled writes none.

Settlement follows from the rows. A queued message is provably unwritten, so a
close, an eviction or an exit rejects it. A handed-over message stays in doubt.
A queued row at or below the sequence a handle found when it opened was left
by an earlier process and is rejected at open, with no latch. Stop withdraws
queued messages with no writer lease and no fence. An attach failure keeps the
conversation open, and the attach adopts its journal. Owed work counts the
loop and queued rows.

A compaction or rewind found prepared when a conversation opens was started
under a child this process no longer has, so the open settles it rather than
leaving it to refuse every send until a view attaches. The open cursor is
scoped to its epoch, because sequences restart when an epoch is replaced.

Deleted: restart-before-admission, recordFailedRestart, the fence rebase,
Claude's startup gate, the attach's forget on failure and its own crash
boundary. Clients without agent-session.accepted-send.v1 get their reply held
until the handover; the desktop and paired desktop lists advertise it.
This commit is contained in:
Brennan Benson
2026-09-25 03:31:08 -07:00
parent 12dde3710b
commit c753e8ab9a
118 changed files with 3135 additions and 1512 deletions
@@ -4,7 +4,7 @@ import { retireClaudeDispatchWaiters } from './claude-structured-dispatch'
import type { ClaudeSession } from './claude-structured-session-state'
import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker'
import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog'
import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate'
import { createClaudeSessionStartup } from './claude-structured-session-startup-state'
export function sessionFor(send: Mock = vi.fn().mockResolvedValue(undefined)): ClaudeSession {
return {
@@ -30,7 +30,7 @@ export function sessionFor(send: Mock = vi.fn().mockResolvedValue(undefined)): C
capabilities: [],
events: undefined,
translator: null,
startup: { ...createClaudeSessionStartupGate(), state: 'proven' }
startup: { ...createClaudeSessionStartup(), state: 'proven' }
}
}
+4 -15
View File
@@ -35,10 +35,8 @@ import {
import { AgentSessionPreDispatchError } from '../native-chat/agent-session-wire/structured-agent-session-operation-settlement'
import {
claudeStartupFailureReason,
claudeStartupHoldsWrites,
failClaudeStartupGate,
holdClaudeStartupWrite
} from './claude-structured-session-startup-gate'
failClaudeStartup
} from './claude-structured-session-startup-state'
const MAX_ACTIVE_DISPATCH_WAITERS = 64
@@ -226,7 +224,7 @@ export function settleCancelledClaudeDispatchWaiters(
* Retired rather than dropped: their identities stay joinable, bounded by
* `MAX_RETIRED_DISPATCH_WAITERS`. */
export function retireClaudeDispatchWaiters(session: ClaudeSession): void {
failClaudeStartupGate(session, new Error('claude stream-json ended before startup completed'))
failClaudeStartup(session, new Error('claude stream-json ended before startup completed'))
for (const waiter of session.dispatchWaiters.splice(0)) {
retireWaiter(session, waiter)
waiter.resolve(null)
@@ -236,8 +234,7 @@ export function retireClaudeDispatchWaiters(session: ClaudeSession): void {
export async function dispatchClaudeTurn(
session: ClaudeSession,
input: { clientMessageId?: string; body: AgentJournalMessageItem; requestedAt?: number },
beforeDispatch?: () => Promise<void>,
onSettledLate?: ClaudeLateDispatchSettlement
beforeDispatch?: () => Promise<void>
): Promise<AgentSessionDispatchOutcome> {
let content: unknown[]
try {
@@ -276,14 +273,6 @@ export async function dispatchClaudeTurn(
parent_tool_use_id: null,
session_id: session.providerSessionId
}
if (claudeStartupHoldsWrites(session)) {
return holdClaudeStartupWrite(session, {
message,
arm,
...(beforeDispatch ? { beforeDispatch } : {}),
...(onSettledLate ? { settleLate: onSettledLate } : {})
})
}
const pending = { replay: beforeDispatch ? undefined : arm() }
const authorize = beforeDispatch
? async () => {
@@ -10,7 +10,7 @@ import {
} from './claude-agent-sdk-control-requests'
import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker'
import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog'
import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate'
import { createClaudeSessionStartup } from './claude-structured-session-startup-state'
import {
claudeStructuredSessionOptionsFrom,
observeClaudeFastModeFacts,
@@ -46,7 +46,7 @@ function sessionFor(setModel: ClaudeSession['connection']['setModel']): ClaudeSe
capabilities: [],
events: undefined,
translator: null,
startup: { ...createClaudeSessionStartupGate(), state: 'proven' }
startup: { ...createClaudeSessionStartup(), state: 'proven' }
}
}
@@ -11,11 +11,6 @@ import {
} from './claude-structured-control-actions'
import type { ClaudeLateDispatchSettlement } from './claude-structured-dispatch'
import type { ClaudeSession } from './claude-structured-session-state'
import {
claudeStartupHoldsWrites,
rejectClaudeStartupWrites
} from './claude-structured-session-startup-gate'
import { DISPATCH_REJECTED_CANCELLED } from '../../shared/structured-agent-session-dispatch-rejection'
/** Conservative user-facing window: below the 30s control deadline, trading
* residual slow-pump risk for ensuring delivery bookkeeping cannot block Stop indefinitely. */
@@ -103,14 +98,9 @@ export async function cancelClaudeStructuredTurn(input: {
const session = requireSession(sessions, request.sessionId)
const acquisitionGeneration = session.acquisitionGeneration
const prompt = request.prompt
// A held prompt was never written, so Stop withdraws it; the drain only writes what it still
// holds. Before startup lands nothing was written, so there is nothing to interrupt either.
let withdrewHeld = false
if (!prompt && claudeStartupHoldsWrites(session) && session.fence === request.fence) {
withdrewHeld = rejectClaudeStartupWrites(session, DISPATCH_REJECTED_CANCELLED)
if (session.startup.state === 'pending') {
return { cancelled: withdrewHeld }
}
// Before startup lands nothing was written, so there is nothing to interrupt.
if (!prompt && session.startup.state === 'pending') {
return { cancelled: false }
}
if (prompt && session.fence !== request.fence) {
return { cancelled: false }
@@ -193,7 +183,7 @@ export async function cancelClaudeStructuredTurn(input: {
} else if (claim) {
session.prompts.releaseClaim(claim)
}
return withdrewHeld ? { ...result, cancelled: true } : result
return result
} catch (error) {
if (claim && !interruptConfirmed) {
session.prompts.releaseClaim(claim)
@@ -70,7 +70,7 @@ function realAdapter(
const acquire = adapter.acquire
adapter.acquire = async (input) => {
const acquisition = await acquire(input)
await adapter.drainStartup(input.identity.sessionId)
await adapter.awaitStarted(input.identity.sessionId)
return acquisition
}
return adapter
@@ -248,30 +248,34 @@ export async function acquireClaudeSession({
event()
}
})
session.startup.settled = settleClaudeSessionStartup({
session,
facts: readClaudeStartupFacts({
connection,
initProof,
sessionId,
providerSessionId: launch.providerSessionId,
resumesTranscript: launch.resumesTranscript,
inputOptions: input.options,
requestTimeoutMs: deps.requestTimeoutMs,
emit
}),
isCurrent: () => sessions.get(sessionId) === session,
requestTimeoutMs: deps.requestTimeoutMs,
fault: (error) => callbacks.handleExit(sessionId, attempt, error),
onStarted: (options) =>
emit({
type: 'started',
// Whichever comes first: the start landing or faulting, or the child being ended.
session.startup.settled = Promise.race([
session.startup.settled,
settleClaudeSessionStartup({
session,
facts: readClaudeStartupFacts({
connection,
initProof,
sessionId,
fence: input.fence,
acquisitionGeneration: session.acquisitionGeneration,
...options
})
})
providerSessionId: launch.providerSessionId,
resumesTranscript: launch.resumesTranscript,
inputOptions: input.options,
requestTimeoutMs: deps.requestTimeoutMs,
emit
}),
isCurrent: () => sessions.get(sessionId) === session,
requestTimeoutMs: deps.requestTimeoutMs,
fault: (error) => callbacks.handleExit(sessionId, attempt, error),
onStarted: (options) =>
emit({
type: 'started',
sessionId,
fence: input.fence,
acquisitionGeneration: session.acquisitionGeneration,
...options
})
})
])
// A child whose exit already reached `handleExit` is not handed over as live: the create
// fails with the CLI's own diagnostic, as one that died before publish does.
if (sessions.get(sessionId) !== session) {
@@ -12,7 +12,7 @@ import { acquireClaudeSession } from './claude-structured-session-acquisition'
import { supportsClaudeStructuredLocation } from './claude-structured-location-support'
import { setClaudeStructuredSessionOption } from './claude-structured-options'
import { readClaudeStructuredSessionOptions } from './claude-structured-session-options'
import { claudeStartupSettledWithin } from './claude-structured-session-startup-gate'
import { claudeStartupSettledWithin } from './claude-structured-session-startup-state'
import { CLAUDE_DEFAULT_REQUEST_TIMEOUT_MS } from './claude-agent-sdk-control-requests'
import {
ClaudeAcquisitionRegistry,
@@ -114,8 +114,8 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda
* apart without guessing at wall-clock. */
drainObservedExits = (): Promise<void> => drainClaudeObservedExits(this.exits)
/** Resolves once a published session's startup has landed or faulted it. */
drainStartup = (sessionId: string): Promise<void> =>
/** Resolves once a published session's startup has landed, faulted, or been ended by a close. */
awaitStarted = (sessionId: string): Promise<void> =>
this.sessions.get(sessionId)?.startup.settled ?? Promise.resolve()
/** Restart reconciliation reads the transcript a resume replays; these maps track liveness. */
@@ -165,9 +165,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda
}
dispatch: StructuredAgentSessionAdapter['dispatch'] = (input) =>
dispatchClaudeTurn(this.session(input.sessionId), input, input.beforeDispatch, (settlement) =>
this.deps.onDispatchSettledLate?.({ sessionId: input.sessionId, ...settlement })
)
dispatchClaudeTurn(this.session(input.sessionId), input, input.beforeDispatch)
compact: NonNullable<StructuredAgentSessionAdapter['compact']> = (input) =>
compactClaudeSession(this.session(input.sessionId), this.compactions, input)
@@ -3,7 +3,7 @@ import {
claudeRootExitObserved,
settleClaudeExitedSession
} from './claude-structured-session-close'
import { failClaudeStartupGate } from './claude-structured-session-startup-gate'
import { failClaudeStartup } from './claude-structured-session-startup-state'
import type {
ClaudeAcquisitionAttempt,
ClaudeSession,
@@ -32,7 +32,7 @@ export function observeClaudeSessionExit(
return
}
lifecycle.sessions.delete(sessionId)
failClaudeStartupGate(session, error)
failClaudeStartup(session, error)
// Re-enter the provider's close ladder before publishing lifecycle recovery.
// An exit callback is root evidence only; the retained tree proof must run
// before the host releases and reacquires this exact child.
@@ -5,7 +5,7 @@ import type { ClaudeJournalTranslator } from './claude-structured-journal-transl
import type { ClaudeSession } from './claude-structured-session-state'
import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker'
import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog'
import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate'
import { createClaudeSessionStartup } from './claude-structured-session-startup-state'
/** The session as published at spawn: nothing the CLI reports at init is assumed yet. */
export function createClaudeSessionPublication(input: {
@@ -65,7 +65,7 @@ export function createClaudeSessionPublication(input: {
translator: input.translator,
events: input.events,
...(input.unbindReadingControl ? { unbindReadingControl: input.unbindReadingControl } : {}),
startup: createClaudeSessionStartupGate()
startup: createClaudeSessionStartup()
}
}
}
@@ -338,7 +338,7 @@ describe('ClaudeStructuredSessionAdapter close and exit recovery', () => {
spawnToken: 'spawn-9',
events: journalSink
})
await adapter.drainStartup('session-1')
await adapter.awaitStarted('session-1')
const first = claude.connections[0]
const oldPrompt = invokeCanUseTool(first, 'Bash', 'permission-retained', 'tool-retained')
const oldSession = (
@@ -1,172 +0,0 @@
// A Claude session is published once its child is spawned, before the CLI has answered
// initialize. Prompts sent in that window are held here and written, in order, once startup
// lands (init facts read and saved options restored), so a first turn never runs under
// defaults the restore was about to replace. A held prompt was never written, so a startup
// that fails rejects it rather than leaving its delivery in doubt.
import type { AgentSessionDispatchOutcome } from '../native-chat/agent-session-wire/structured-agent-session-adapter'
import { AgentSessionPreDispatchError } from '../native-chat/agent-session-wire/structured-agent-session-operation-settlement'
import { dispatchWriteFailureReason } from '../../shared/structured-agent-session-dispatch-rejection'
import { providerStartupFailureRejection } from '../native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement'
import { claudeUserMessageWasProvablyUnwritten } from './claude-agent-sdk-user-message-queue'
import {
forgetRetiredWaiter,
forgetWaiter,
retireWaiter
} from './claude-structured-dispatch-waiters'
import type {
ClaudeDispatchWaiter,
ClaudeLateDispatchOutcome,
ClaudeSession
} from './claude-structured-session-state'
type ClaudeStartupHeldWrite = {
waiter: ClaudeDispatchWaiter
message: Record<string, unknown>
settleLate?: (outcome: ClaudeLateDispatchOutcome) => void
}
export type ClaudeSessionStartupGate = {
state: 'pending' | 'proven' | 'failed'
held: ClaudeStartupHeldWrite[]
/** Held prompts are still being written; later prompts must queue behind them. */
draining: boolean
failure: Error | null
/** Resolves once startup has landed or faulted the session; never rejects. */
settled: Promise<void>
}
export function createClaudeSessionStartupGate(): ClaudeSessionStartupGate {
return { state: 'pending', held: [], draining: false, failure: null, settled: Promise.resolve() }
}
export function claudeStartupFailureReason(session: ClaudeSession): string | null {
return session.startup.state === 'failed'
? providerStartupFailureRejection(session.startup.failure ?? undefined)
: null
}
/** Resolves when startup lands or `timeoutMs` passes; a stuck start then refuses the write as before. */
export function claudeStartupSettledWithin(
session: ClaudeSession | undefined,
timeoutMs: number
): Promise<void> {
if (session?.startup.state !== 'pending') {
return Promise.resolve()
}
let timer: ReturnType<typeof setTimeout> | undefined
return Promise.race([
session.startup.settled,
new Promise<void>((resolve) => {
timer = setTimeout(resolve, timeoutMs)
})
]).finally(() => clearTimeout(timer))
}
export function claudeStartupHoldsWrites(session: ClaudeSession): boolean {
return session.startup.state === 'pending' || session.startup.draining
}
/** Admits a prompt while startup is pending; it is written when `openClaudeStartupGate` runs. */
export async function holdClaudeStartupWrite(
session: ClaudeSession,
input: {
message: Record<string, unknown>
arm: () => { waiter: ClaudeDispatchWaiter }
beforeDispatch?: () => Promise<void>
settleLate?: (outcome: ClaudeLateDispatchOutcome) => void
}
): Promise<AgentSessionDispatchOutcome> {
if (input.beforeDispatch) {
try {
await input.beforeDispatch()
} catch (error) {
if (error instanceof AgentSessionPreDispatchError) {
throw error
}
return { state: 'rejected', reason: dispatchWriteFailureReason(error) }
}
}
// Startup may have failed while the admission barrier ran.
const failed = claudeStartupFailureReason(session)
if (failed) {
return { state: 'rejected', reason: failed }
}
const { waiter } = input.arm()
session.startup.held.push({
waiter,
message: input.message,
...(input.settleLate ? { settleLate: input.settleLate } : {})
})
// Startup finished writing what it held while the barrier ran; nothing else would drain this.
if (!claudeStartupHoldsWrites(session)) {
void drainClaudeStartupWrites(session)
}
return { state: 'admitted' }
}
export async function openClaudeStartupGate(session: ClaudeSession): Promise<void> {
const gate = session.startup
if (gate.state !== 'pending') {
return
}
gate.state = 'proven'
await drainClaudeStartupWrites(session)
}
async function drainClaudeStartupWrites(session: ClaudeSession): Promise<void> {
const gate = session.startup
gate.draining = true
try {
for (let held = gate.held.shift(); held; held = gate.held.shift()) {
await writeHeld(session, held)
}
} finally {
gate.draining = false
}
}
async function writeHeld(session: ClaudeSession, held: ClaudeStartupHeldWrite): Promise<void> {
try {
await session.connection.send(held.message)
} catch (error) {
if (held.waiter.settledUuid) {
return
}
if (claudeUserMessageWasProvablyUnwritten(error)) {
rejectHeld(session, held, dispatchWriteFailureReason(error))
return
}
// Possibly written: only a replay or the child's exit can settle it now.
retireWaiter(session, held.waiter)
held.waiter.resolve(null)
}
}
function rejectHeld(session: ClaudeSession, held: ClaudeStartupHeldWrite, reason: string): void {
forgetWaiter(session, held.waiter)
forgetRetiredWaiter(session, held.waiter)
held.waiter.resolve(null)
if (held.waiter.clientMessageId) {
held.settleLate?.({ clientMessageId: held.waiter.clientMessageId, state: 'rejected', reason })
}
}
/** Rejects every held prompt with `reason`; true when any was held. */
export function rejectClaudeStartupWrites(session: ClaudeSession, reason: string): boolean {
const held = session.startup.held.splice(0)
for (const entry of held) {
rejectHeld(session, entry, reason)
}
return held.length > 0
}
/** Startup cannot land any more; nothing held was written, so all of it is rejected. */
export function failClaudeStartupGate(session: ClaudeSession, error: Error): void {
const gate = session.startup
if (gate.state === 'pending') {
gate.state = 'failed'
gate.failure = error
}
rejectClaudeStartupWrites(session, providerStartupFailureRejection(error))
}
@@ -0,0 +1,58 @@
// Where a Claude start stands. A session is published once its child is spawned, before the CLI
// has answered initialize. Nothing is written to it until startup lands (init facts read and saved
// options restored): the host's delivery loop waits on `settled` before it hands a message over,
// so a first turn never runs under defaults the restore was about to replace.
import { providerStartupFailureRejection } from '../native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement'
import type { ClaudeSession } from './claude-structured-session-state'
export type ClaudeSessionStartup = {
state: 'pending' | 'proven' | 'failed'
failure: Error | null
/** Resolves once startup has landed or faulted, or the child exited or was closed; never
* rejects. A close must end it: the delivery loop waits here, and a start Stop cut short
* would otherwise hold that loop forever. */
settled: Promise<void>
end: () => void
}
export function createClaudeSessionStartup(): ClaudeSessionStartup {
let end: () => void = () => undefined
const ended = new Promise<void>((resolve) => {
end = resolve
})
return { state: 'pending', failure: null, settled: ended, end }
}
export function claudeStartupFailureReason(session: ClaudeSession): string | null {
return session.startup.state === 'failed'
? providerStartupFailureRejection(session.startup.failure ?? undefined)
: null
}
/** Resolves when startup lands or `timeoutMs` passes; a stuck start then refuses the write as before. */
export function claudeStartupSettledWithin(
session: ClaudeSession | undefined,
timeoutMs: number
): Promise<void> {
if (session?.startup.state !== 'pending') {
return Promise.resolve()
}
let timer: ReturnType<typeof setTimeout> | undefined
return Promise.race([
session.startup.settled,
new Promise<void>((resolve) => {
timer = setTimeout(resolve, timeoutMs)
})
]).finally(() => clearTimeout(timer))
}
/** Startup cannot land any more: the child exited, was closed, or its start faulted. */
export function failClaudeStartup(session: ClaudeSession, error: Error): void {
const startup = session.startup
if (startup.state === 'pending') {
startup.state = 'failed'
startup.failure = error
}
startup.end()
}
@@ -55,7 +55,7 @@ describe('Claude structured session publishes before the CLI answers initialize'
expect(adapter.readCommands('session-1')).toBeUndefined()
await vi.advanceTimersByTimeAsync(SLOW_INIT_MS)
await adapter.drainStartup('session-1')
await adapter.awaitStarted('session-1')
expect(events.find((event) => event.type === 'options')).toMatchObject({
models: [{ value: 'claude-sonnet' }]
@@ -65,7 +65,7 @@ describe('Claude structured session publishes before the CLI answers initialize'
await adapter.closeAll()
})
it('reports `started` once saved options are restored, before any held prompt is written', async () => {
it('reports `started` once saved options are restored, having written no prompt of its own', async () => {
const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS, initModel: 'claude-opus-9' })
const { adapter, events } = startingAdapter(claude)
const order: string[] = []
@@ -74,11 +74,10 @@ describe('Claude structured session publishes before the CLI answers initialize'
return undefined
}
await adapter.acquire({ ...ACQUIRE, options: { model: 'opus' } })
await adapter.dispatch(PROMPT)
expect(events.some((event) => event.type === 'started')).toBe(false)
await vi.advanceTimersByTimeAsync(SLOW_INIT_MS)
await adapter.drainStartup('session-1')
await adapter.awaitStarted('session-1')
const startedAt = events.findIndex((event) => event.type === 'started')
expect(events[startedAt]).toEqual({
@@ -90,9 +89,9 @@ describe('Claude structured session publishes before the CLI answers initialize'
reportedOptions: expect.objectContaining({ model: 'opus' }),
restoreSkippedOptions: []
})
// The restore wrote the saved model before `started`, and the held prompt only after it.
// The restore wrote the saved model before `started`; no message waits inside the adapter.
expect(order).toEqual(['set_model'])
expect(claude.connections[0].sent).toHaveLength(1)
expect(claude.connections[0].sent).toEqual([])
expect(events.slice(0, startedAt).some((event) => event.type === 'options')).toBe(true)
await adapter.closeAll()
})
@@ -133,52 +132,37 @@ describe('Claude structured session publishes before the CLI answers initialize'
await adapter.closeAll()
})
it('holds a prompt sent before init and writes it once startup lands', async () => {
// The host's delivery loop waits here before it hands a message over, so the adapter no longer
// holds prompts of its own: nothing is written until startup lands because nothing is sent.
it('resolves awaitStarted only once startup lands', async () => {
const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS })
const { adapter } = startingAdapter(claude)
await adapter.acquire(ACQUIRE)
await expect(adapter.dispatch(PROMPT)).resolves.toEqual({ state: 'admitted' })
expect(claude.connections[0].sent).toEqual([])
await vi.advanceTimersByTimeAsync(SLOW_INIT_MS)
await adapter.drainStartup('session-1')
expect(claude.connections[0].sent).toHaveLength(1)
expect(claude.connections[0].sent[0]).toMatchObject({ type: 'user' })
await adapter.closeAll()
})
it('writes a prompt whose admission barrier was still running when startup landed', async () => {
const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS })
const { adapter } = startingAdapter(claude)
await adapter.acquire(ACQUIRE)
let passBarrier = (): void => {}
const barrier = new Promise<void>((resolve) => {
passBarrier = resolve
let started = false
const waited = adapter.awaitStarted('session-1').then(() => {
started = true
})
const dispatched = adapter.dispatch({ ...PROMPT, beforeDispatch: () => barrier })
await vi.advanceTimersByTimeAsync(SLOW_INIT_MS)
await adapter.drainStartup('session-1')
passBarrier()
await vi.advanceTimersByTimeAsync(SLOW_INIT_MS - 1)
expect(started).toBe(false)
await vi.advanceTimersByTimeAsync(1)
await waited
await expect(dispatched).resolves.toEqual({ state: 'admitted' })
expect(claude.connections[0].sent.filter((message) => message.type === 'user')).toHaveLength(1)
await expect(adapter.dispatch(PROMPT)).resolves.toEqual({ state: 'admitted' })
expect(claude.connections[0].sent).toEqual([expect.objectContaining({ type: 'user' })])
await adapter.closeAll()
})
it('ends the session with the exit reason when the CLI dies before init, and rejects held prompts', async () => {
it('ends the session with the exit reason when the CLI dies before init', async () => {
const claude = fakeClaude({
initDelayMs: SLOW_INIT_MS,
exitBeforeInit: 'claude stream-json exited (code 1): stderr says no'
})
const { adapter, events, late } = startingAdapter(claude)
const { adapter, events } = startingAdapter(claude)
await adapter.acquire(ACQUIRE)
await adapter.dispatch(PROMPT)
await vi.advanceTimersByTimeAsync(SLOW_INIT_MS)
await adapter.drainStartup('session-1')
await adapter.awaitStarted('session-1')
await adapter.drainObservedExits()
expect(events.find((event) => event.type === 'ended')).toMatchObject({
@@ -186,9 +170,6 @@ describe('Claude structured session publishes before the CLI answers initialize'
cause: 'unexpected-exit',
startupUnproven: true
})
expect(late).toEqual([
expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' })
])
expect(claude.connections[0].sent).toEqual([])
expect(claude.connections[0].closeCount).toBe(1)
})
@@ -203,7 +184,7 @@ describe('Claude structured session publishes before the CLI answers initialize'
await adapter.acquire(ACQUIRE)
await vi.advanceTimersByTimeAsync(SLOW_INIT_MS)
await adapter.drainStartup('session-1')
await adapter.awaitStarted('session-1')
await adapter.drainObservedExits()
// A failed start is released on the same evidence a failed create is; a proven-live
@@ -218,7 +199,7 @@ describe('Claude structured session publishes before the CLI answers initialize'
const claude = fakeClaude({ initAccount: { apiProvider: 'firstParty', tokenSource: 'none' } })
const { adapter, events } = startingAdapter(claude)
await adapter.acquire(ACQUIRE)
await adapter.drainStartup('session-1')
await adapter.awaitStarted('session-1')
await adapter.drainObservedExits()
expect(events.find((event) => event.type === 'ended')).toMatchObject({
@@ -227,82 +208,41 @@ describe('Claude structured session publishes before the CLI answers initialize'
})
})
it('closes a session stopped before init without faulting it or writing held prompts', async () => {
const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS })
const { adapter, events, late } = startingAdapter(claude)
// A Stop that closes a child still starting must end the wait the host's delivery loop is in,
// though initialize never answers; otherwise every later send joins a loop that never moves.
it('ends the wait on a start closed before init, without faulting it', async () => {
const claude = fakeClaude({ initDelayMs: 10 * SLOW_INIT_MS })
const { adapter, events } = startingAdapter(claude)
await adapter.acquire(ACQUIRE)
await adapter.dispatch(PROMPT)
let ended = false
const waited = adapter.awaitStarted('session-1').then(() => {
ended = true
})
await expect(adapter.closeSession('session-1')).resolves.toBe(true)
await vi.advanceTimersByTimeAsync(SLOW_INIT_MS)
await adapter.drainStartup('session-1')
await vi.advanceTimersByTimeAsync(0)
await waited
expect(ended).toBe(true)
const connection = claude.connections[0]
expect(connection.closeCount).toBe(1)
expect(connection.sent).toEqual([])
expect(connection.calls.map(({ subtype }) => subtype)).not.toContain('get_settings')
expect(late).toEqual([
expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' })
])
expect(events.some((event) => event.type === 'ended' && event.startupUnproven)).toBe(false)
expect(events.some((event) => event.type === 'started')).toBe(false)
})
it('withdraws a held prompt when the turn is cancelled before init', async () => {
it('interrupts nothing when Stop lands before init: nothing was written', async () => {
const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS })
const { adapter, late } = startingAdapter(claude)
const { adapter } = startingAdapter(claude)
await adapter.acquire(ACQUIRE)
await adapter.dispatch(PROMPT)
await expect(
adapter.cancelTurn({ sessionId: 'session-1', turnId: 'turn-1', fence: 7 })
).resolves.toEqual({ cancelled: true })
await vi.advanceTimersByTimeAsync(SLOW_INIT_MS)
await adapter.drainStartup('session-1')
).resolves.toEqual({ cancelled: false })
expect(claude.connections[0].calls.map(({ subtype }) => subtype)).not.toContain('interrupt')
expect(claude.connections[0].sent).toEqual([])
expect(late).toEqual([
expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' })
])
await adapter.closeAll()
})
it('withdraws the prompts still held when Stop lands while startup is writing them', async () => {
const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS })
const { adapter, late } = startingAdapter(claude)
await adapter.acquire(ACQUIRE)
const connection = claude.connections[0]
const send = connection.send
let landFirstWrite = (): void => {}
const firstWrite = new Promise<void>((resolve) => {
landFirstWrite = resolve
})
let writes = 0
connection.send = async (message, beforeDispatch) => {
writes += 1
if (writes === 1) {
await firstWrite
}
return send(message, beforeDispatch)
}
await adapter.dispatch(PROMPT)
await adapter.dispatch({ ...PROMPT, clientMessageId: 'client-2' })
await vi.advanceTimersByTimeAsync(SLOW_INIT_MS)
// Startup has landed and is writing the first held prompt.
expect(writes).toBe(1)
const cancelled = adapter.cancelTurn({ sessionId: 'session-1', turnId: 'turn-1', fence: 7 })
await vi.advanceTimersByTimeAsync(0)
landFirstWrite()
await vi.advanceTimersByTimeAsync(5_000)
await adapter.drainStartup('session-1')
expect(connection.sent.filter((message) => message.type === 'user')).toHaveLength(1)
expect(late).toContainEqual(
expect.objectContaining({ clientMessageId: 'client-2', state: 'rejected' })
)
// Stop withdrew something, so it answers as a cancel whatever the interrupt made of the turn.
await expect(cancelled).resolves.toEqual({ cancelled: true })
await adapter.closeAll()
})
})
@@ -26,10 +26,7 @@ import {
observeClaudeSettingsApplied,
readClaudeSettingsEffort
} from './claude-structured-session-options'
import {
failClaudeStartupGate,
openClaudeStartupGate
} from './claude-structured-session-startup-gate'
import { failClaudeStartup } from './claude-structured-session-startup-state'
import type { ClaudeSession, ClaudeStructuredSessionEvent } from './claude-structured-session-state'
export type ClaudeInitProof = {
@@ -163,8 +160,8 @@ function claudeStartedReportedOptions(
return persisted
}
/** Applies startup facts to the published session, restores saved options, then releases
* held prompts. Any failure faults the session so the user sees why it never started. */
/** Applies startup facts to the published session and restores saved options; only then does the
* session take input. Any failure faults the session so the user sees why it never started. */
export async function settleClaudeSessionStartup(input: {
session: ClaudeSession
facts: Promise<ClaudeStartupFacts>
@@ -179,7 +176,7 @@ export async function settleClaudeSessionStartup(input: {
if (input.isCurrent()) {
return false
}
failClaudeStartupGate(session, new Error('claude session closed before startup completed'))
failClaudeStartup(session, new Error('claude session closed before startup completed'))
return true
}
try {
@@ -198,13 +195,15 @@ export async function settleClaudeSessionStartup(input: {
),
restoreSkippedOptions: [...session.restoreSkippedOptions]
})
await openClaudeStartupGate(session)
if (session.startup.state === 'pending') {
session.startup.state = 'proven'
}
}
} catch (caught) {
const error = caught instanceof Error ? caught : new Error(String(caught))
// A close or exit that already ended startup owns how the session ends.
const endedElsewhere = session.startup.state !== 'pending'
failClaudeStartupGate(session, error)
failClaudeStartup(session, error)
if (!endedElsewhere && input.isCurrent()) {
input.fault(error)
}
@@ -23,7 +23,7 @@ import type {
} from '../../shared/agent-session-wire'
import type { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker'
import type { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog'
import type { ClaudeSessionStartupGate } from './claude-structured-session-startup-gate'
import type { ClaudeSessionStartup } from './claude-structured-session-startup-state'
export type ClaudeAuthDiagnostic = {
apiKeySourceConfigured: boolean
@@ -190,7 +190,7 @@ export type ClaudeSession = {
events: StructuredAgentSessionEventSink | undefined
unbindReadingControl?: () => void
/** Published at spawn; init facts, option restore and queued prompts land when startup does. */
startup: ClaudeSessionStartupGate
startup: ClaudeSessionStartup
}
export function mintClaudeAcquisitionGeneration(deps: ClaudeStructuredSessionAdapterDeps): string {
@@ -220,7 +220,7 @@ export function adapterFor(
const acquire = adapter.acquire
adapter.acquire = async (input) => {
const acquisition = await acquire(input)
await adapter.drainStartup(input.identity.sessionId)
await adapter.awaitStarted(input.identity.sessionId)
return acquisition
}
return adapter
@@ -1,5 +1,6 @@
import {
AGENT_LAUNCH_RUNTIME_CAPABILITY,
AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY,
AGENT_SESSION_BACKGROUND_TASK_ROW_STOP_CAPABILITY,
AGENT_SESSION_BACKGROUND_TASK_STOP_CAPABILITY,
AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY,
@@ -25,6 +26,7 @@ import {
export const DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES: readonly RuntimeCapability[] = [
AGENT_SESSION_BACKGROUND_TASK_STOP_CAPABILITY,
AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY,
AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY,
AGENT_SESSION_TURN_ITEM_CAPABILITY,
AGENT_SESSION_BACKGROUND_TASK_ROW_STOP_CAPABILITY,
STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY,
@@ -1,8 +1,11 @@
import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types'
import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission'
import { DISPATCH_DOUBT_HOST_RESTARTED } from './journal-dispatch-doubt-reasons'
import type { AgentSessionJournal } from './journal-store'
/** Settles every submission a process fact left unanswerable. Doubt is never
* proof of non-delivery, so nothing here ever becomes re-deliverable. */
* proof of non-delivery, so nothing here ever becomes re-deliverable. A queued
* submission was never handed over, so it is not in doubt and is left alone. */
export async function markJournalPendingSubmissionsUnknown(
journal: AgentSessionJournal,
fence: number,
@@ -12,8 +15,9 @@ export async function markJournalPendingSubmissionsUnknown(
.submissions()
.filter(
(entry) =>
entry.dispatchState === 'pending' ||
(entry.dispatchState === 'unknown' && entry.recovered !== true)
!isQueuedAgentJournalSubmission(entry) &&
(entry.dispatchState === 'pending' ||
(entry.dispatchState === 'unknown' && entry.recovered !== true))
)
for (const entry of unresolved) {
// An earlier reason already names a sharper fact than "the host restarted".
@@ -55,3 +59,25 @@ export async function rejectJournalPendingSubmissions(
}
return unwritten.map((entry) => entry.clientMessageId)
}
/** Rejects queued submissions — accepted, never handed over, so provably unwritten. */
export async function rejectJournalQueuedSubmissions(
journal: AgentSessionJournal,
fence: number,
reason: string,
which: (submission: AgentJournalSubmission) => boolean = () => true
): Promise<string[]> {
const queued = journal
.submissions()
.filter((entry) => isQueuedAgentJournalSubmission(entry) && which(entry))
for (const entry of queued) {
await journal.resolveDispatch({
clientMessageId: entry.clientMessageId,
state: 'rejected',
reason,
fence,
recovered: true
})
}
return queued.map((entry) => entry.clientMessageId)
}
@@ -254,7 +254,8 @@ function applySubmission(
providerItemId: null,
reason: null,
submittedAt: row.ts,
resolvedAt: null
resolvedAt: null,
...(row.handoverRecorded ? { handoverRecorded: true, acceptedSequence: row.seq } : {})
})
const itemId = agentJournalSubmissionKey(row.clientMessageId)
upsertItem(state, itemId, 0, journalRenderItem(itemId, 0, row.body, row))
@@ -277,6 +278,9 @@ function applyDispatch(
submission.providerItemId = row.providerItemId
submission.reason = row.reason
submission.resolvedAt = row.state === 'pending' ? null : row.ts
if (row.state === 'pending') {
submission.handedOverAt = row.ts
}
if (row.recovered) {
submission.recovered = row.recovered
} else {
@@ -18,6 +18,7 @@ import {
agentJournalItemKey,
agentJournalSubmissionKey
} from '../../../shared/agent-session-journal-item-key'
import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission'
import type { AgentSessionJournal } from './journal-store'
import { reconcileSubmissions, type ProviderHistoryWindow } from './journal-submission-reconciler'
@@ -42,7 +43,11 @@ function comparableSubmissions(journal: AgentSessionJournal): AgentJournalSubmis
const { items, submissions } = journal.snapshot()
const bodies = new Map(items.map((item) => [item.itemId, item.body]))
return submissions.filter((submission) => {
if (submission.dispatchState !== 'pending' && submission.dispatchState !== 'unknown') {
if (
(submission.dispatchState !== 'pending' && submission.dispatchState !== 'unknown') ||
// Never handed over, so provider history cannot hold it.
isQueuedAgentJournalSubmission(submission)
) {
return false
}
const body = bodies.get(agentJournalSubmissionKey(submission.clientMessageId))
@@ -65,6 +65,7 @@ export function journalSubmissionRowBuilder(
payloadFingerprint: string
body: AgentJournalMessageItem
fence: number
handoverRecorded?: true
}
): RowBuilder<JournalSubmissionRow> {
return (seq, ts) =>
@@ -267,6 +268,7 @@ export function buildJournalSubmissionRow(input: {
seq: number
fence: number
ts: number
handoverRecorded?: true
}): JournalSubmissionRow {
return {
kind: 'submission',
@@ -274,7 +276,8 @@ export function buildJournalSubmissionRow(input: {
payloadFingerprint: input.payloadFingerprint,
providerHandle: input.providerHandle,
body: input.body,
...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts)
...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts),
...(input.handoverRecorded ? { handoverRecorded: true } : {})
}
}
@@ -77,6 +77,9 @@ export type JournalSubmissionRow = JournalRowBase & {
payloadFingerprint: string
providerHandle: AgentSessionProviderHandle
body: AgentJournalMessageItem
/** Accepted to be handed over by a later `dispatch{pending}` row; absent on rows whose writer
* dispatched in the same step. Older readers keep the key and ignore it. */
handoverRecorded?: true
}
export type JournalDispatchRow = JournalRowBase & {
@@ -59,6 +59,8 @@ export type JournalSubmissionInput = {
payloadFingerprint: string
body: AgentJournalMessageItem
fence: number
/** The send is accepted now and handed over later, by a `dispatch{pending}` row. */
handoverRecorded?: true
}
export type JournalItemAppendInput = {
@@ -424,3 +424,31 @@ async function withJournalDatabase(
opened.db.close()
}
}
describe('what a handle found on disk when it opened', () => {
const submission = (clientMessageId: string) => ({
clientMessageId,
payloadFingerprint: 'fp',
body: { kind: 'message' as const, role: 'user' as const, blocks: [] },
fence: 1,
handoverRecorded: true as const
})
it('names rows an earlier handle wrote, and never a row of a later epoch', async () => {
const earlier = await open()
await earlier.appendItem(item(1), body('one'), { fence: 1 })
await earlier.appendSubmission(submission('earlier'))
await earlier.close()
const journal = await open()
const leftover = journal.submissions().find((entry) => entry.clientMessageId === 'earlier')
expect(journal.wroteBeforeOpen(leftover?.acceptedSequence)).toBe(true)
// Sequences restart with an epoch, so a row accepted after it can sit below the open cursor.
await journal.replaceEpochItems('handle_forked', 1, [])
await journal.appendSubmission(submission('later'))
const later = journal.submissions().find((entry) => entry.clientMessageId === 'later')
expect(later?.acceptedSequence).toBeLessThanOrEqual(2)
expect(journal.wroteBeforeOpen(later?.acceptedSequence)).toBe(false)
})
})
@@ -28,7 +28,8 @@ import { readJournalRowsAfterCursor, type JournalLoad } from './journal-open'
import { journalDatabaseFile } from './journal-paths'
import {
markJournalPendingSubmissionsUnknown,
rejectJournalPendingSubmissions
rejectJournalPendingSubmissions,
rejectJournalQueuedSubmissions
} from './journal-pending-submission-recovery'
import {
applyJournalRow,
@@ -75,6 +76,7 @@ export class AgentSessionJournal {
private state: JournalReducerState
private readOnly = false
private malformedRows = 0
private openedThrough: AgentJournalCursor = { epoch: '', sequence: 0 }
private database: OpenJournalDatabase | null = null
private onCommitted: (() => void) | null = null
private readonly queue: JournalWriteQueue
@@ -147,6 +149,16 @@ export class AgentSessionJournal {
return this.journalDir
}
/** Whether a row at this sequence was on disk when this handle opened, so an earlier handle
* wrote it. Sequences restart with each epoch, so a row of a later epoch never was. */
wroteBeforeOpen(sequence: number | undefined): boolean {
return (
sequence !== undefined &&
this.state.epoch === this.openedThrough.epoch &&
sequence <= this.openedThrough.sequence
)
}
/** What the last open's repair did. */
get repair(): { malformedRows: number } {
return { malformedRows: this.malformedRows }
@@ -157,6 +169,7 @@ export class AgentSessionJournal {
this.database = openJournalDatabase(this.dbPath)
try {
await this.restore()
this.openedThrough = this.cursor()
} catch (error) {
// Nothing else holds a reference to this connection, so a throw here is
// the leak site unless the store releases it itself — and a close that
@@ -310,6 +323,15 @@ export class AgentSessionJournal {
return rejectJournalPendingSubmissions(this, fence, reason)
}
/** Reject sends accepted but never handed over, optionally only those `which` names. */
async rejectQueuedSubmissions(
fence: number,
reason: string,
which?: (submission: AgentJournalSubmission) => boolean
): Promise<string[]> {
return rejectJournalQueuedSubmissions(this, fence, reason, which)
}
/** The escape hatch for corruption, an unreconcilable prefix, a forked handle,
* and an unreadable schema. It invalidates every cursor; clients reload. */
async rollEpoch(reason: AgentJournalEpochReason, fence: number): Promise<AgentJournalCursor> {
@@ -0,0 +1,611 @@
// A send is accepted, then delivered: the host answers once the message is recorded, and the
// session's delivery loop starts a provider child for it and hands it over. Against the real host,
// store and journal; each assertion reads what an open chat or the journal's next reader sees.
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest'
import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope'
import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types'
import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire'
import {
DISPATCH_REJECTED_CANCELLED,
DISPATCH_REJECTED_HOST_RESTARTED,
DISPATCH_REJECTED_PROVIDER_CLOSED
} from '../../../shared/structured-agent-session-dispatch-rejection'
import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store'
import { journalDirectoryFor } from '../agent-session-journal/journal-paths'
import { AgentSessionJournal } from '../agent-session-journal/journal-store'
import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory'
import {
AgentSessionPreSpawnError,
type StructuredAgentSessionAdapter
} from './structured-agent-session-adapter'
import { journalIdentityFor } from './structured-agent-session-attach'
import { attachParamsForRecord } from './structured-agent-session-conversation-open'
import { StructuredAgentSessionHost } from './structured-agent-session-host'
import { persistRewindRecord } from './structured-rewind-recovery'
import {
HOST_TEST_NOW as NOW,
HOST_TEST_SESSION as SESSION,
HOST_TEST_THREAD as THREAD,
hostTestAttachParams,
hostTestMessage,
hostTestOperationId,
resetHostTestOperationIds
} from './structured-agent-session-host-test-data'
const CALLER = { callerKey: 'client-1' }
function eventually(assertion: () => void | Promise<void>): Promise<void> {
return vi.waitFor(assertion, { timeout: 10_000 })
}
let root: string
let store: AgentSessionRecordStore
let host: StructuredAgentSessionHost
let acquire: Mock<StructuredAgentSessionAdapter['acquire']>
let dispatch: Mock<StructuredAgentSessionAdapter['dispatch']>
let adapterExtras: Partial<StructuredAgentSessionAdapter>
let releaseGraceMs: number
const spawnChild: StructuredAgentSessionAdapter['acquire'] = async ({ fence, spawnToken }) => ({
process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken },
acquisitionGeneration: `generation-${acquire.mock.calls.length}`,
link: {
linkId: `link-${fence}`,
handle: { provider: 'codex' as const, threadId: THREAD },
origin: store.getRecord(SESSION)?.providerHandleChain.length
? ('resumed' as const)
: ('created' as const),
mintedAtFence: fence,
observedAt: NOW
}
})
async function startHost(): Promise<void> {
host = new StructuredAgentSessionHost({
store,
adapter: {
acquire,
dispatch,
closeSession: vi.fn(async () => true),
releaseAcquisition: vi.fn(async () => true),
cancelTurn: vi.fn(async () => ({ cancelled: false })),
answerPrompt: vi.fn(async () => undefined),
setOption: vi.fn(async () => undefined),
...adapterExtras
},
journalRoot: root,
claimKeyId: 'key-1',
mintSpawnToken: () => `spawn-${acquire.mock.calls.length}`,
releaseGraceMs,
now: () => NOW
})
}
beforeEach(async () => {
root = await mkdtemp(join(tmpdir(), 'orca-accept-deliver-'))
resetHostTestOperationIds()
adapterExtras = {}
releaseGraceMs = 60_000
acquire = vi.fn(spawnChild)
dispatch = vi.fn(async () => ({
state: 'accepted' as const,
providerIdentity: {
provider: 'codex' as const,
threadId: THREAD,
turnId: `turn-${dispatch.mock.calls.length}`,
ordinal: dispatch.mock.calls.length
}
}))
store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' })
await startHost()
expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true })
})
afterEach(async () => {
await host.flushAllStreamedEvents()
await rm(root, { recursive: true, force: true })
})
function sendParams(text: string) {
const body = hostTestMessage(text)
return {
envelope: {
sessionId: SESSION,
clientOperationId: hostTestOperationId(),
expectedRuntimeFence: 1,
payloadFingerprint: computeAgentSessionPayloadFingerprint({
method: 'agentSession.send',
sessionId: SESSION,
fields: { body }
})
},
body
}
}
/** Accepted at once, whatever the child is doing; answers the message id. */
async function accept(text: string): Promise<string> {
const params = sendParams(text)
expect(await host.send(CALLER, params)).toMatchObject({
ok: true,
value: { submission: { dispatchState: 'pending', handoverRecorded: true } }
})
return params.envelope.clientOperationId
}
function stop() {
const turnId = 'turn-none'
return host.cancel(CALLER, {
envelope: {
sessionId: SESSION,
clientOperationId: hostTestOperationId(),
expectedRuntimeFence: null,
payloadFingerprint: computeAgentSessionPayloadFingerprint({
method: 'agentSession.cancel',
sessionId: SESSION,
fields: { turnId }
})
},
turnId
})
}
function submission(id: string): AgentJournalSubmission | undefined {
return host.journalSnapshot(SESSION).submissions.find((entry) => entry.clientMessageId === id)
}
/** Read back after the conversation was closed, through the same open any reader takes. */
async function reopened(id: string): Promise<AgentJournalSubmission | undefined> {
await host.revealSession(SESSION)
return submission(id)
}
function errorRows(): string[] {
return host
.journalSnapshot(SESSION)
.items.flatMap((item) =>
item.body.kind === 'status' && item.body.tone === 'error' ? [item.body.text] : []
)
}
let subscriptions = 0
function subscribe(): AgentSessionSubscribeEvent[] {
const events: AgentSessionSubscribeEvent[] = []
subscriptions += 1
// Cloned as received: a frame shares the journal's live objects, which later rows revise.
host.subscribe({
id: `sub-${subscriptions}`,
sessionId: SESSION,
emit: (event) => events.push(structuredClone(event))
})
return events
}
/** What an open chat was told about one message, in frame order. */
function framedStates(events: AgentSessionSubscribeEvent[], id: string): string[] {
return events.flatMap((event) =>
event.type === 'batch'
? event.batch.submissions
.filter((entry) => entry.clientMessageId === id)
.map((entry) =>
entry.dispatchState === 'pending' && entry.handedOverAt !== undefined
? 'handed-over'
: entry.dispatchState
)
: []
)
}
function deferred<T>() {
let resolve!: (value: T) => void
let reject!: (error: unknown) => void
const promise = new Promise<T>((next, fail) => {
resolve = next
reject = fail
})
return { promise, resolve, reject }
}
/** Rows written by an earlier host process that ended before handing them over. */
async function writeAsEarlierProcess(
write: (journal: AgentSessionJournal, fence: number) => Promise<void>
): Promise<void> {
await host.close(SESSION)
const record = store.getRecord(SESSION)!
const params = attachParamsForRecord(record, {
clientOperationId: 'earlier',
expectedRuntimeFence: record.lease.runtimeFence
})
const journal = await openAgentSessionJournal({
identity: journalIdentityFor(record, params),
journalDir: journalDirectoryFor(root, {
workspaceId: record.location.workspaceId,
sessionId: SESSION
})
})
await write(journal, record.lease.runtimeFence)
await journal.close()
}
function earlierSubmission(id: string, text: string, handoverRecorded?: true) {
const body = hostTestMessage(text)
return {
clientMessageId: id,
payloadFingerprint: computeAgentSessionPayloadFingerprint({
method: 'agentSession.send',
sessionId: SESSION,
fields: { body }
}),
body,
...(handoverRecorded ? { handoverRecorded } : {})
}
}
describe('a send is answered at acceptance', () => {
it('answers before the child starts, then an open chat sees the handover and the reply (W2)', async () => {
await host.close(SESSION)
const starting = deferred<void>()
acquire.mockImplementationOnce(async (input) => {
await starting.promise
return spawnChild(input)
})
const answering = deferred<void>()
const answer = dispatch.getMockImplementation()!
dispatch.mockImplementationOnce(async (input) => {
await answering.promise
return answer(input)
})
const id = await accept('hello')
const events = subscribe()
await eventually(() => expect(acquire).toHaveBeenCalledTimes(2))
expect(dispatch).not.toHaveBeenCalled()
starting.resolve()
await eventually(() => expect(framedStates(events, id)).toEqual(['handed-over']))
answering.resolve()
await eventually(() => expect(framedStates(events, id)).toEqual(['handed-over', 'accepted']))
})
it('accepts a second send while the first one starts the child, before handing either over (W6)', async () => {
await host.close(SESSION)
const starting = deferred<void>()
acquire.mockImplementationOnce(async (input) => {
await starting.promise
return spawnChild(input)
})
const first = await accept('first')
await eventually(() => expect(acquire).toHaveBeenCalledTimes(2))
const second = host.send(CALLER, sendParams('second'))
starting.resolve()
const secondResult = await second
if (!secondResult.ok) {
throw new Error('the second send was refused')
}
const secondId = secondResult.value.clientMessageId
await eventually(() => expect(submission(secondId)?.dispatchState).toBe('accepted'))
expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([first, secondId])
// The second was accepted while the start held the queue — before the first was handed over.
expect(submission(secondId)!.submittedAt).toBeLessThanOrEqual(submission(first)!.handedOverAt!)
const rows = host.journalSnapshot(SESSION).submissions
expect(rows.map((row) => row.clientMessageId)).toEqual([first, secondId])
})
})
describe('a start the chat needed and did not get', () => {
it('writes one error row and rejects every queued message with it; the next send starts (W3)', async () => {
await host.close(SESSION)
acquire.mockRejectedValueOnce(new Error('spawn codex ENOENT'))
const first = await accept('first')
const second = await accept('second')
await eventually(() => expect(submission(second)?.dispatchState).toBe('rejected'))
const rows = errorRows()
expect(rows).toHaveLength(1)
expect(rows[0]).toContain('spawn codex ENOENT')
expect(submission(first)).toMatchObject({ dispatchState: 'rejected', reason: rows[0] })
expect(submission(second)).toMatchObject({ dispatchState: 'rejected', reason: rows[0] })
const next = await accept('after the fix')
await eventually(() => expect(submission(next)?.dispatchState).toBe('accepted'))
expect(errorRows()).toHaveLength(1)
})
it.each([
[
'eligibility',
() => {
adapterExtras = { supportsLocation: () => false }
},
'cannot resume'
],
[
'spawn',
() => acquire.mockRejectedValueOnce(new Error('spawn codex ENOENT')),
'spawn codex ENOENT'
],
[
'auth',
() =>
acquire.mockRejectedValueOnce(
new AgentSessionPreSpawnError(new Error('Not logged in. Please run /login.'))
),
'Not logged in'
]
])('writes one row a live chat sees for a %s refusal (W14)', async (_source, arrange, cause) => {
await host.close(SESSION)
arrange()
await host.flushAllStreamedEvents()
await startHost()
const id = await accept('hello')
const events = subscribe()
await eventually(() => expect(submission(id)?.dispatchState).toBe('rejected'))
expect(errorRows()).toHaveLength(1)
expect(errorRows()[0]).toContain(cause)
const framedRows = events.flatMap((event) =>
event.type === 'batch' || event.type === 'snapshot'
? (event.type === 'batch' ? event.batch.items : event.page.items).filter(
(item) => item.body.kind === 'status' && item.body.tone === 'error'
)
: []
)
expect(framedRows.length).toBeGreaterThan(0)
})
it('names the start failure on queued messages when the attach fails after acquiring (W4′a)', async () => {
await host.close(SESSION)
const id = await accept('hello')
// The attach's own success record is the post-acquisition step that fails.
const record = vi.spyOn(store, 'recordOperationOutcome')
record.mockImplementation(async (input) => {
if (input.operationId !== id && input.outcome.status === 'succeeded') {
record.mockRestore()
throw new Error('record store write failed')
}
return AgentSessionRecordStore.prototype.recordOperationOutcome.call(store, input)
})
// Read through the open any reader takes, so a conversation the failure dropped is reopened
// and its message read as that reopen settles it.
let settled: AgentJournalSubmission | undefined
await eventually(async () => {
settled = await reopened(id)
expect(settled?.dispatchState).not.toBe('pending')
})
// The message names the start that failed, not a close or a restart it never met.
expect(settled).toMatchObject({
dispatchState: 'rejected',
reason: expect.stringContaining('record store write failed')
})
expect(errorRows()).toEqual([settled?.reason])
})
})
describe('what an earlier host process left behind', () => {
it('rejects a message it accepted and never handed over, as not sent (W4′b)', async () => {
await writeAsEarlierProcess(async (journal, fence) => {
await journal.appendSubmission({ ...earlierSubmission('queued', 'q', true), fence })
})
await host.revealSession(SESSION)
expect(submission('queued')).toMatchObject({
dispatchState: 'rejected',
reason: DISPATCH_REJECTED_HOST_RESTARTED
})
expect(dispatch).not.toHaveBeenCalled()
})
it('leaves a legacy pending message and a handed-over one in doubt, never re-sent (W4′c)', async () => {
const providerHistoryWindow = vi.fn(async () => null)
adapterExtras = { providerHistoryWindow }
await writeAsEarlierProcess(async (journal, fence) => {
await journal.appendSubmission({ ...earlierSubmission('legacy', 'l'), fence })
await journal.appendSubmission({ ...earlierSubmission('handed', 'h', true), fence })
await journal.resolveDispatch({ clientMessageId: 'handed', state: 'pending', fence })
})
await host.flushAllStreamedEvents()
await startHost()
await host.revealSession(SESSION)
expect(submission('legacy')).toMatchObject({ dispatchState: 'unknown', recovered: true })
expect(submission('handed')).toMatchObject({ dispatchState: 'unknown', recovered: true })
// Deciding them from provider history waits for a won lease (W4′d).
expect(providerHistoryWindow).not.toHaveBeenCalled()
expect(dispatch).not.toHaveBeenCalled()
})
})
describe('Stop withdraws what is queued', () => {
it('withdraws a crash leftover the open could not settle, ahead of any delivery step (W17a)', async () => {
await writeAsEarlierProcess(async (journal, fence) => {
await journal.appendSubmission({ ...earlierSubmission('leftover', 'l', true), fence })
})
const settle = vi.spyOn(AgentSessionJournal.prototype, 'rejectQueuedSubmissions')
settle.mockRejectedValueOnce(new Error('journal busy'))
expect(await stop()).toMatchObject({ ok: true })
settle.mockRestore()
expect(submission('leftover')).toMatchObject({
dispatchState: 'rejected',
reason: DISPATCH_REJECTED_CANCELLED
})
expect(acquire).toHaveBeenCalledTimes(1)
})
it('withdraws a message whose start holds the queue: nothing is handed over (W17b)', async () => {
await host.close(SESSION)
const starting = deferred<void>()
acquire.mockImplementationOnce(async (input) => {
await starting.promise
return spawnChild(input)
})
const id = await accept('hello')
await eventually(() => expect(acquire).toHaveBeenCalledTimes(2))
const stopped = stop()
starting.resolve()
expect(await stopped).toMatchObject({ ok: true })
await eventually(() => expect(submission(id)?.dispatchState).toBe('rejected'))
expect(submission(id)).toMatchObject({ reason: DISPATCH_REJECTED_CANCELLED })
expect(submission(id)?.handedOverAt).toBeUndefined()
expect(dispatch).not.toHaveBeenCalled()
})
it('stops a child still proving its start, and the delivery loop ends with it (W17c)', async () => {
const ended = deferred<void>()
const awaitStarted = vi.fn(() => ended.promise)
const closeSession = vi.fn(async () => {
ended.resolve()
return true
})
adapterExtras = { awaitStarted, closeSession }
await host.close(SESSION)
await startHost()
acquire.mockImplementationOnce(async (input) => ({
...(await spawnChild(input)),
providerChildPhase: 'starting' as const
}))
const id = await accept('hello')
await eventually(() => expect(awaitStarted).toHaveBeenCalled())
expect(await stop()).toMatchObject({ ok: true, value: { cancelled: true } })
expect(await reopened(id)).toMatchObject({
dispatchState: 'rejected',
reason: DISPATCH_REJECTED_CANCELLED
})
expect(closeSession).toHaveBeenCalled()
// A loop still waiting on that start would swallow this send; it starts a new child instead.
awaitStarted.mockImplementation(async () => undefined)
const next = await accept('after stop')
await eventually(() => expect(submission(next)?.dispatchState).toBe('accepted'))
expect(dispatch).toHaveBeenCalledTimes(1)
})
})
describe('an eviction between acceptance and handover', () => {
it('rejects the message as not sent, never leaves it in doubt (W24)', async () => {
const started = deferred<void>()
adapterExtras = { awaitStarted: () => started.promise }
await host.close(SESSION)
await startHost()
await host.hold(SESSION, 'surface-1')
const id = await accept('hello')
await eventually(() => expect(acquire).toHaveBeenCalledTimes(2))
// Between the delivery loop's start step and its handover step.
await host.close(SESSION)
started.resolve()
expect(await reopened(id)).toMatchObject({
dispatchState: 'rejected',
reason: DISPATCH_REJECTED_PROVIDER_CLOSED
})
expect(dispatch).not.toHaveBeenCalled()
})
it('rejects a queued message behind a handed-over one, which alone stays in doubt (W24)', async () => {
const second = deferred<void>()
const awaitStarted = vi.fn(async (): Promise<void> => undefined)
adapterExtras = { awaitStarted }
await host.close(SESSION)
await startHost()
await host.hold(SESSION, 'surface-1')
dispatch.mockResolvedValueOnce({ state: 'admitted' })
const handed = await accept('handed over')
await eventually(() => expect(submission(handed)?.handedOverAt).toBeDefined())
awaitStarted.mockImplementation(() => second.promise)
const queued = await accept('still queued')
await eventually(() => expect(awaitStarted).toHaveBeenCalledTimes(2))
await host.close(SESSION)
second.resolve()
expect(await reopened(queued)).toMatchObject({
dispatchState: 'rejected',
reason: DISPATCH_REJECTED_PROVIDER_CLOSED
})
expect(submission(handed)).toMatchObject({ dispatchState: 'unknown' })
expect(dispatch).toHaveBeenCalledTimes(1)
})
it('does not evict an idle child while a message is still queued for it (W24)', async () => {
releaseGraceMs = 0
const started = deferred<void>()
adapterExtras = { awaitStarted: () => started.promise }
await host.close(SESSION)
await startHost()
await host.hold(SESSION, 'surface-1')
const id = await accept('hello')
await eventually(() => expect(acquire).toHaveBeenCalledTimes(2))
host.release(SESSION, 'surface-1')
await new Promise((resolve) => setTimeout(resolve, 20))
started.resolve()
// Handed to the child it was queued for; the eviction may follow once nothing is owed.
await eventually(() => expect(dispatch).toHaveBeenCalledTimes(1))
expect(dispatch.mock.calls[0]?.[0].clientMessageId).toBe(id)
expect(acquire).toHaveBeenCalledTimes(2)
})
})
describe('a compaction or rewind an earlier child left prepared', () => {
async function leftPrepared(prepare: (fence: number) => Promise<unknown>): Promise<void> {
await host.close(SESSION)
await prepare(store.getRecord(SESSION)!.lease.runtimeFence)
// A new process: nothing is open and no view attaches.
await host.flushAllStreamedEvents()
await startHost()
}
it('settles an interrupted compaction at open, so a send is accepted and delivered (R16)', async () => {
await leftPrepared((fence) =>
store.setConversationCommand(SESSION, fence, {
command: 'compact',
runtimeFence: fence,
operationId: 'compact-op',
callerKey: 'client-1',
phase: 'prepared',
state: 'unknown'
})
)
const id = await accept('after the compaction')
await eventually(() => expect(submission(id)?.dispatchState).toBe('accepted'))
expect(store.getRecord(SESSION)?.conversationCommand).toMatchObject({
phase: 'committed',
state: 'unknown'
})
})
it('completes a rewind the provider already applied at open, so a send is accepted (R16)', async () => {
await leftPrepared((fence) =>
persistRewindRecord(store, SESSION, fence, {
operationId: 'rewind-op',
callerKey: 'client-1',
itemId: 'orca:rewound',
providerItemId: `codex:${THREAD}:turn-1:0`,
expectedEpoch: 'epoch-before',
phase: 'provider-succeeded',
hydrationVerified: true,
retained: []
})
)
const id = await accept('after the rewind')
await eventually(() => expect(submission(id)?.dispatchState).toBe('accepted'))
expect(store.getRecord(SESSION)?.rewind).toMatchObject({ phase: 'completed' })
})
})
@@ -122,6 +122,8 @@ export class StructuredAgentSessionAdapterRouter implements StructuredAgentSessi
awaitOptionWritable = (sessionId: string): Promise<void> =>
this.liveOwnerOrNull(sessionId)?.awaitOptionWritable?.(sessionId) ?? Promise.resolve()
awaitStarted = (sessionId: string): Promise<void> =>
this.liveOwnerOrNull(sessionId)?.awaitStarted?.(sessionId) ?? Promise.resolve()
readOptions = (input: { sessionId: string; fence: number }) => {
const reader = this.owner(input.sessionId).readOptions
@@ -278,6 +278,9 @@ export type StructuredAgentSessionAdapter = {
): Promise<void | Readonly<Record<string, string>>>
/** Resolves once a live session can take an option write, or after a bound; never rejects. */
awaitOptionWritable?(sessionId: string): Promise<void>
/** Resolves once a session published before it proved its start has proven it, failed, or been
* closed; at once for any other. Never rejects. */
awaitStarted?(sessionId: string): Promise<void>
readOptions?(input: { sessionId: string; fence: number }): Promise<AgentSessionOptionsResult>
/** Option keys skipped after a provider rejected their persisted restore value. */
readOptionRestoreFailures?(sessionId: string): readonly string[]
@@ -33,8 +33,9 @@ export type StructuredAgentSessionAttachContext = {
reconcileLeases: (sessionId: string) => Promise<AgentSessionWireRefusal | null>
serialize: <T>(sessionId: string, task: () => Promise<T>) => Promise<T>
now: () => number
/** Paired with `sessions.delete` by `forgetStructuredAgentSession`; a failed attach that only
* deleted would leave the store's row behind. */
/** Paired with `sessions.delete` by `forgetStructuredAgentSession`, which a close runs. */
forgetStatus: (sessionId: string) => void
publishStatus?: (sessionId: string) => void
/** The conversation's one open journal, opened when closed; see `conversation-open`. */
openConversation: (sessionId: string) => Promise<StructuredAgentSessionHostSession | null>
}
@@ -24,8 +24,6 @@ export async function settlePostAcquisitionAttachFailure(
? 'root-exit-observed'
: 'exit-proven'
}
// A failed close must not prevent durable failure settlement.
await Promise.resolve(input.onAttachFailed?.()).catch(() => undefined)
try {
await input.store.settleFailedPostAcquisitionAttachment({
sessionId: record.sessionId,
@@ -41,6 +41,7 @@ import {
type AgentSessionCreatePhaseRecorder
} from '../../observability/agent-session-instrumentation'
import type { ProviderHistoryWindow } from '../agent-session-journal/journal-submission-reconciler'
import type { AgentSessionJournal } from '../agent-session-journal/journal-store'
export type AttachFlowInput = {
store: AgentSessionRecordStore
@@ -66,8 +67,8 @@ export type AttachFlowInput = {
onAcquiring?: () => Promise<void> | void
/** Settles writes already captured by the superseded journal before opening another. */
beforeJournalOpen?: () => Promise<void> | void
/** Closes and removes partial publication after journal attachment fails. */
onAttachFailed?: () => Promise<void>
/** The host's open conversation, which the attach adopts rather than opening a second one. */
openConversation?: (sessionId: string) => Promise<AgentSessionJournal>
}
export async function performAttach(
@@ -208,6 +209,7 @@ export async function performAttach(
params,
journalRoot: input.journalRoot,
adapter: input.adapter,
...(input.openConversation ? { openConversation: input.openConversation } : {}),
providerHistoryWindow
})
await importAdoptedTranscript(params, attached, record, preparedTranscript.items)
@@ -24,9 +24,7 @@ import { refuseAgentSessionMutation } from './structured-agent-session-mutation-
import { retryPendingStructuredAgentSessionSettlement } from './structured-agent-session-settlement-retry'
import { settleStaleSessionStateOnAcquire } from './structured-agent-session-stale-turn-verdict'
import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context'
import { forgetStructuredAgentSession } from './structured-agent-session-host-lifetime'
import type { DeferredStructuredAgentSessionEventSink } from './structured-agent-session-event-sink'
import { agentSessionJournalCloseRetries } from '../agent-session-journal/journal-close-retry'
import type { AgentSessionJournal } from '../agent-session-journal/journal-store'
import {
addAgentSessionCreatePhaseAttributes,
@@ -162,12 +160,14 @@ async function runAttach(
params,
now: () => context.now(),
recordPhase,
// Site 9: this closes the PRIOR map entry it drops, never the provisional
// journal — it has no reference to that one. `onAttached` owns that.
onAttachFailed: async () => {
await forgetStructuredAgentSession(context, sessionId)
context.runtimeState.currentEventSink(sessionId)?.close()
context.runtimeState.discardEventSink(sessionId)
// A failed attach leaves the conversation open, with its queued messages, for the failure
// to be written into; only the attempt's own sink is discarded, in the `finally` below.
openConversation: async (id) => {
const conversation = await context.openConversation(id)
if (!conversation) {
throw new Error('agent_session_identity_required')
}
return conversation.journal
},
onAttached: async (attached, acquisitionGeneration, acquiredOwner, providerChildPhase) => {
const fence = context.deps.store.getRecord(sessionId)?.lease.runtimeFence ?? 0
@@ -177,37 +177,19 @@ async function runAttach(
const eventSink = acquiredOwner
? attemptSink
: (context.runtimeState.currentEventSink(sessionId) ?? attemptSink)
// Site 8: the provisional journal has no owner until the map takes it,
// and the barrier below throws by design.
try {
if (acquiredOwner) {
// Before the drain: the buffered events are the new child's, never a stale row's.
await settleStaleSessionStateOnAcquire({
journal: attached.journal,
sessionId,
fence,
acquisitionGeneration
})
}
await bindAndDrain(eventSink, attached.journal, fence, (activity) =>
context.subscribers.publish(sessionId, attached.journal, activity)
)
} catch (error) {
await agentSessionJournalCloseRetries.closeOrRetain(attached.journal)
throw error
}
// Site 10: a `set` over a live entry would orphan its handle — and a
// close that REJECTED did not release it. The replacement is therefore
// ABORTED rather than completed over a handle nothing can reach again:
// `previous` stays indexed, so teardown still owns it and can retry.
if (previous && previous.journal !== attached.journal) {
try {
await previous.journal.close()
} catch (error) {
await agentSessionJournalCloseRetries.closeOrRetain(attached.journal)
throw error
}
// The journal is the conversation's own open one, so a throw here leaves it indexed.
if (acquiredOwner) {
// Before the drain: the buffered events are the new child's, never a stale row's.
await settleStaleSessionStateOnAcquire({
journal: attached.journal,
sessionId,
fence,
acquisitionGeneration
})
}
await bindAndDrain(eventSink, attached.journal, fence, (activity) =>
context.subscribers.publish(sessionId, attached.journal, activity)
)
context.runtimeState.adoptEventSink(sessionId, eventSink)
attemptSinkAdopted = eventSink === attemptSink
context.sessions.set(sessionId, {
@@ -230,9 +212,7 @@ async function runAttach(
context.now
)
await recoverInterruptedCompaction(context.deps.store, sessionId, attached.journal, fence)
if (attached.recovery) {
context.subscribers.reset(sessionId, attached.journal, attached.recovery.reset)
} else if (previousFence !== undefined && previousFence !== fence) {
if (previousFence !== undefined && previousFence !== fence) {
context.subscribers.snapshot(sessionId, attached.journal)
} else {
context.subscribers.publish(sessionId, attached.journal)
@@ -241,12 +221,23 @@ async function runAttach(
}).finally(() => {
if (!attemptSinkAdopted) {
attemptSink.close()
followRecordFence(context, sessionId)
}
})
)
return attached
}
/** A failed attempt still moved the record's fence when it reserved; the conversation it leaves
* open writes its failure row and rejections at that fence, not the one it opened at. */
function followRecordFence(context: StructuredAgentSessionAttachContext, sessionId: string): void {
const conversation = context.sessions.get(sessionId)
const fence = context.deps.store.getRecord(sessionId)?.lease.runtimeFence
if (conversation && !conversation.hasProviderChild && fence !== undefined) {
conversation.fence = Math.max(conversation.fence, fence)
}
}
/** Binds the sink to the journal and waits for the barrier the host publishes
* behind. It throws by design when a sink barrier fails. */
async function bindAndDrain(
@@ -147,4 +147,36 @@ describe('attachJournal restart reconciliation', () => {
expect(attached.unconfirmedClientMessageIds).toEqual(['cm_1'])
expect(attached.journal.submissions()[0]?.dispatchState).toBe('unknown')
})
it('leaves a message the open conversation still has queued alone (W4′e)', async () => {
const journal = await journals.open({
identity: IDENTITY,
journalDir: journalDirectoryFor(root, {
workspaceId: IDENTITY.workspaceId,
sessionId: IDENTITY.sessionId
})
})
await journal.appendSubmission({
clientMessageId: 'queued',
payloadFingerprint: digestPayload('still queued'),
body: userMessage('still queued'),
fence: RECORD.lease.runtimeFence,
handoverRecorded: true
})
// History that holds nothing: absence would prove a handed-over message undelivered.
const { adapter, dispatch } = adapterWith(async () => window())
const attached = await attachJournal({
record: RECORD,
params: PARAMS,
journalRoot: root,
adapter,
openConversation: async () => journal
})
expect(attached.journal).toBe(journal)
expect(journal.submissions()[0]).toMatchObject({ dispatchState: 'pending' })
expect(journal.submissions()[0]?.handedOverAt).toBeUndefined()
expect(dispatch).not.toHaveBeenCalled()
})
})
@@ -36,15 +36,10 @@ import {
} from '../../../shared/agent-session-mutation-envelope'
import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store'
import { agentSessionProviderHandleChainHead } from '../../../shared/agent-session-provider-handle'
import { agentSessionJournalCloseRetries } from '../agent-session-journal/journal-close-retry'
import { journalDirectoryFor } from '../agent-session-journal/journal-paths'
import type { AgentSessionJournal } from '../agent-session-journal/journal-store'
import { reconcileJournalSubmissionsAgainstHistory } from '../agent-session-journal/journal-restart-reconciliation'
import type { ProviderHistoryWindow } from '../agent-session-journal/journal-submission-reconciler'
import {
openAgentSessionJournalWithRecovery,
type AgentSessionJournalRecovery
} from './agent-session-journal-recovery'
import { openStructuredAgentSessionConversationJournal } from './structured-agent-session-conversation-open'
import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter'
import { structuredAgentSessionRefusalMessage } from './structured-agent-session-refusal-message'
@@ -166,16 +161,13 @@ export function journalIdentityFor(
export type AttachedJournal = {
journal: AgentSessionJournal
recovery: AgentSessionJournalRecovery | null
/** Submissions still `unknown` after this open: the crash boundary settled
* them there and provider history could not decide them either. */
/** Submissions the crash boundary left `unknown` that provider history could not decide. */
unconfirmedClientMessageIds: string[]
}
/**
* Open the session's journal, recovering it when the stored one is unusable,
* settle every submission left in flight by a previous process, then let
* provider history decide the ones it can prove.
* The conversation's journal — opened, and its crash boundary settled, by the conversation's own
* open — with provider history deciding the submissions that boundary could only doubt.
*
* Why the reconciliation belongs HERE and nowhere else: this runs after the
* record store handed this host the lease and before `onAttached` starts a
@@ -183,57 +175,58 @@ export type AttachedJournal = {
* is read, and the window stays valid until the resume consumes it. Every other
* settlement site — a proven child exit — runs while the host
* may still start another child, and a read there could be overtaken before it
* is acted on. Orca still never re-sends: this decides state only.
* is acted on. Orca still never re-sends: this decides state only. A queued
* submission is left alone: it was never handed over, so history cannot hold it.
*/
export async function attachJournal(input: {
record: AgentSessionRecord
params: AgentSessionAttachParams
journalRoot: string
adapter: StructuredAgentSessionAdapter
/** The host's open conversation; omitted, the journal is opened for this caller alone. */
openConversation?: (sessionId: string) => Promise<AgentSessionJournal>
/** Provider history sampled before a new child is acquired. `null` means the
* adapter had no usable history; omit to read lazily for direct callers. */
providerHistoryWindow?: ProviderHistoryWindow | null
}): Promise<AttachedJournal> {
const identity = journalIdentityFor(input.record, input.params)
const fence = input.record.lease.runtimeFence
const historyFilePath = input.adapter.historyFilePath
? await input.adapter.historyFilePath({ identity })
: null
const opened = await openAgentSessionJournalWithRecovery({
const journal = input.openConversation
? await input.openConversation(input.record.sessionId)
: await openJournalForCaller(input)
const settled = await reconcileAgainstProviderHistory({
adapter: input.adapter,
identity,
journalDir: journalDirectoryFor(input.journalRoot, {
workspaceId: identity.workspaceId,
sessionId: identity.sessionId
}),
journal,
fence,
historyFilePath
accountHome: input.record.accountHome,
...(Object.hasOwn(input, 'providerHistoryWindow')
? { history: input.providerHistoryWindow }
: {})
})
try {
// That await is a WRITE. A failure in it leaves the journal with no caller
// holding a reference to close it.
const unconfirmed = await opened.journal.markPendingSubmissionsUnknown(fence)
const settled = await reconcileAgainstProviderHistory({
adapter: input.adapter,
identity,
journal: opened.journal,
fence,
accountHome: input.record.accountHome,
...(Object.hasOwn(input, 'providerHistoryWindow')
? { history: input.providerHistoryWindow }
: {})
})
return {
...opened,
unconfirmedClientMessageIds: unconfirmed.filter((id) => !settled.includes(id))
}
} catch (error) {
// A rejected close leaves the handle open, so the journal is retained for a
// later retry rather than dropped along with the only reference to it.
await agentSessionJournalCloseRetries.closeOrRetain(opened.journal)
throw error
return {
journal,
unconfirmedClientMessageIds: journal
.submissions()
.filter(
(entry) =>
entry.dispatchState === 'unknown' &&
entry.recovered === true &&
!settled.includes(entry.clientMessageId)
)
.map((entry) => entry.clientMessageId)
}
}
async function openJournalForCaller(input: {
record: AgentSessionRecord
journalRoot: string
adapter: StructuredAgentSessionAdapter
}): Promise<AgentSessionJournal> {
const opened = await openStructuredAgentSessionConversationJournal(input, input.record)
return opened.session.journal
}
/** Reading provider history is best effort: a provider that reports none, or a
* read that fails, leaves every submission exactly as the crash boundary wrote
* it. The journal writes the outcome implies are NOT caught here — a failed
@@ -178,40 +178,34 @@ describe('the registry', () => {
})
describe('the attach orchestration', () => {
it('ABORTS the map replacement when the previous journal will not close', async () => {
const previousDir = join(root, 'previous')
const provisionalDir = join(root, 'provisional')
const previous = flakyClose(
await journals.open({ identity: IDENTITY, journalDir: previousDir }),
1
)
const provisional = await journals.open({
identity: IDENTITY,
journalDir: provisionalDir
})
attachFlow.journal = provisional
const sessions = new Map([[SESSION, hostSession(previous)]])
// The attach adopts the conversation's one open journal; it never opens a second handle, so
// there is no replacement to abort and no provisional journal to close.
it('keeps the journal it adopted indexed and open when an attach succeeds', async () => {
const directory = join(root, 'adopted')
const journal = await journals.open({ identity: IDENTITY, journalDir: directory })
attachFlow.journal = journal
const sessions = new Map([[SESSION, hostSession(journal)]])
await attachStructuredAgentSession(attachContext(sessions), 'caller-1', attachParams)
expect(sessions.get(SESSION)?.journal).toBe(journal)
await expect(
attachStructuredAgentSession(attachContext(sessions), 'caller-1', attachParams)
).rejects.toThrow('close rejected')
// The live entry is UNTOUCHED: overwriting it would have left its handle
// open with nothing able to reach it again.
expect(sessions.get(SESSION)?.journal).toBe(previous)
// And the provisional journal is owned by the registry, not orphaned.
journal.appendItem(
{ provider: 'orca', clientMessageId: 'after-attach' },
{
kind: 'status',
text: 'still writable'
}
)
).resolves.toBeDefined()
expect(agentSessionJournalCloseRetries.pendingDirectories).toEqual([])
await expectNothingHoldsTheDirectory(provisionalDir)
})
it('retains the provisional journal when its own close rejects on the barrier path', async () => {
const provisionalDir = join(root, 'provisional-barrier')
const provisional = flakyClose(
await journals.open({ identity: IDENTITY, journalDir: provisionalDir }),
1
)
attachFlow.journal = provisional
const sessions = new Map<string, StructuredAgentSessionHostSession>()
it('leaves the conversation indexed and open when the sink barrier fails', async () => {
const directory = join(root, 'adopted-barrier')
const journal = await journals.open({ identity: IDENTITY, journalDir: directory })
attachFlow.journal = journal
const sessions = new Map([[SESSION, hostSession(journal)]])
const context = attachContext(sessions)
const failing = {
sink: {},
@@ -229,9 +223,19 @@ describe('the attach orchestration', () => {
'sink barrier failed'
)
expect(sessions.size).toBe(0)
// Retained rather than dropped, so teardown can still release the handle.
expect(agentSessionJournalCloseRetries.pendingDirectories).toEqual([provisionalDir])
// A failed attach does not end the conversation: its queued messages and the failure row
// are written into this same journal.
expect(sessions.get(SESSION)?.journal).toBe(journal)
await expect(
journal.appendItem(
{ provider: 'orca', clientMessageId: 'after-failure' },
{
kind: 'status',
text: 'still writable'
}
)
).resolves.toBeDefined()
expect(agentSessionJournalCloseRetries.pendingDirectories).toEqual([])
})
})
@@ -0,0 +1,153 @@
// The one way a conversation's journal becomes open on this host: for a send, for a reader, and
// for an attach that finds none open.
//
// It opens with recovery, so an unusable journal is rebuilt rather than refused, and it settles
// what an earlier host process left unanswered — the crash boundary. That needs no lease: a row
// queued there was accepted by a process that ended before handing it over, so it is provably
// unwritten; a handed-over row is only doubt, and provider history decides it later, under a won
// lease, in the attach. Nothing here starts a provider child.
import type { AgentJournalResetReason } from '../../../shared/agent-session-journal-types'
import { DISPATCH_REJECTED_HOST_RESTARTED } from '../../../shared/structured-agent-session-dispatch-rejection'
import { journalDirectoryFor } from '../agent-session-journal/journal-paths'
import type { AgentSessionJournal } from '../agent-session-journal/journal-store'
import { openAgentSessionJournalWithRecovery } from './agent-session-journal-recovery'
import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope'
import type { AgentSessionRecord } from '../../../shared/agent-session-record'
import {
attachFingerprintFields,
journalIdentityFor,
type AgentSessionAttachParams
} from './structured-agent-session-attach'
import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store'
import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter'
import type {
StructuredAgentSessionHostDeps,
StructuredAgentSessionHostSession
} from './structured-agent-session-host-types'
export type OpenedStructuredAgentSessionConversation = {
session: StructuredAgentSessionHostSession
/** Set when the journal was rebuilt on the way; readers reload from a snapshot. */
reset: AgentJournalResetReason | null
}
export type StructuredAgentSessionConversationOpenDeps = {
store: Pick<AgentSessionRecordStore, 'getRecord'>
adapter: Pick<StructuredAgentSessionAdapter, 'historyFilePath'>
journalRoot: string
onEventSinkError?: StructuredAgentSessionHostDeps['onEventSinkError']
}
export type StructuredAgentSessionConversationOpenContext = {
deps: StructuredAgentSessionConversationOpenDeps
sessions: Map<string, StructuredAgentSessionHostSession>
/** Indexes a conversation that just became open; the host publishes it and wakes delivery. */
adoptOpened: (
sessionId: string,
opened: OpenedStructuredAgentSessionConversation
) => Promise<void>
}
/** The open conversation, or null when this host has no record of it. For a caller inside the
* session's serialize, which is what makes "not open yet" exact. */
export async function openStructuredAgentSessionConversation(
context: StructuredAgentSessionConversationOpenContext,
sessionId: string
): Promise<StructuredAgentSessionHostSession | null> {
const open = context.sessions.get(sessionId)
if (open) {
return open
}
const record = context.deps.store.getRecord(sessionId)
if (!record) {
return null
}
const opened = await openStructuredAgentSessionConversationJournal(context.deps, record)
await context.adoptOpened(sessionId, opened)
return opened.session
}
/** The open itself, indexed by nobody yet: the caller adopts the result. */
export async function openStructuredAgentSessionConversationJournal(
deps: Omit<StructuredAgentSessionConversationOpenDeps, 'store'>,
record: AgentSessionRecord
): Promise<OpenedStructuredAgentSessionConversation> {
const { sessionId } = record
const fence = record.lease.runtimeFence
const params = attachParamsForRecord(record, {
clientOperationId: `read-restore:${sessionId}`,
expectedRuntimeFence: fence
})
const identity = journalIdentityFor(record, params)
const opened = await openAgentSessionJournalWithRecovery({
identity,
journalDir: journalDirectoryFor(deps.journalRoot, {
workspaceId: record.location.workspaceId,
sessionId
}),
fence,
historyFilePath: (await deps.adapter.historyFilePath?.({ identity })) ?? null
})
try {
await settleEarlierProcessSubmissions(opened.journal, fence)
} catch (error) {
// Best effort: the delivery loop rejects a leftover queued row itself.
deps.onEventSinkError?.({ sessionId, error })
}
return {
session: {
journal: opened.journal,
params,
fence,
hasProviderChild: false,
providerChildPhase: 'ready',
acquisitionGeneration: null
},
reset: opened.recovery?.reset ?? null
}
}
async function settleEarlierProcessSubmissions(
journal: AgentSessionJournal,
fence: number
): Promise<void> {
// A handle closes only with nothing queued, so a queued row found here is a leftover.
await journal.rejectQueuedSubmissions(fence, DISPATCH_REJECTED_HOST_RESTARTED, (submission) =>
journal.wroteBeforeOpen(submission.acceptedSequence)
)
await journal.markPendingSubmissionsUnknown(fence)
}
export function attachParamsForRecord(
record: AgentSessionRecord,
input: {
clientOperationId: string
expectedRuntimeFence: number
}
): AgentSessionAttachParams {
const params: AgentSessionAttachParams = {
envelope: {
sessionId: record.sessionId,
clientOperationId: input.clientOperationId,
expectedRuntimeFence: input.expectedRuntimeFence,
payloadFingerprint: ''
},
location: record.location,
provider: record.provider,
agent: record.provider,
accountHome: record.accountHome,
runtimeKind: 'native'
}
return {
...params,
envelope: {
...params.envelope,
payloadFingerprint: computeAgentSessionPayloadFingerprint({
method: 'agentSession.attach',
sessionId: record.sessionId,
fields: attachFingerprintFields(params)
})
}
}
}
@@ -190,6 +190,7 @@ describe('dead structured-session generation settlement', () => {
| 'submissions'
| 'markPendingSubmissionsUnknown'
| 'rejectPendingSubmissions'
| 'rejectQueuedSubmissions'
| 'appendLifecycleBatch'
> = {
snapshot: () => ({
@@ -203,6 +204,9 @@ describe('dead structured-session generation settlement', () => {
rejectPendingSubmissions: async () => {
throw new Error('journal_closed')
},
rejectQueuedSubmissions: async () => {
throw new Error('journal_closed')
},
appendLifecycleBatch: async () => {
throw new Error('journal_closed')
}
@@ -4,6 +4,7 @@ import type {
AgentJournalRenderItem
} from '../../../shared/agent-session-journal-types'
import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record'
import { DISPATCH_REJECTED_PROVIDER_CLOSED } from '../../../shared/structured-agent-session-dispatch-rejection'
import { partitionJournalLifecycleMutations } from '../agent-session-journal/journal-lifecycle-batch-partition'
import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders'
import {
@@ -55,6 +56,14 @@ export function providerStartupFailureOutcome(reason?: string): string {
: 'The provider stopped before it finished starting.'
}
/** Why a message accepted but not yet handed over was rejected when its child exited. */
export function providerExitBeforeDeliveryRejection(reason?: string): string {
const detail = exitReasonDetail(reason)
return detail
? `The provider stopped before this message was sent: ${detail}.`
: 'The provider stopped before this message was sent.'
}
/** Why a send a child that never started left unwritten was rejected. The child's own diagnostic is
* the cause the user can act on, so it is the reason, in the words the chat row uses. */
export function providerStartupFailureRejection(cause?: unknown): string {
@@ -72,13 +81,14 @@ function exitReasonDetail(reason: string | undefined): string | undefined {
type DeadGenerationSubmission = Pick<
ReturnType<AgentSessionJournal['submissions']>[number],
'clientMessageId' | 'dispatchState' | 'recovered'
'clientMessageId' | 'dispatchState' | 'recovered' | 'handoverRecorded' | 'handedOverAt'
>
export type DeadGenerationJournal = {
appendLifecycleBatch: AgentSessionJournal['appendLifecycleBatch']
markPendingSubmissionsUnknown: AgentSessionJournal['markPendingSubmissionsUnknown']
rejectPendingSubmissions: AgentSessionJournal['rejectPendingSubmissions']
rejectQueuedSubmissions: AgentSessionJournal['rejectQueuedSubmissions']
snapshot: () => Pick<ReturnType<AgentSessionJournal['snapshot']>, 'items'>
pendingSubmissions?: AgentSessionJournal['pendingSubmissions']
submissions?: () => DeadGenerationSubmission[]
@@ -152,9 +162,18 @@ export async function settleStructuredAgentSessionDeadGeneration(input: {
if (!showUnexpectedExitOutcome && !hasUnfinishedWork) {
return true
}
// A child that never proved its start accepted nothing — input is written only after it
// initializes — so every send it left unanswered is provably unwritten and is rejected with the
// child's own diagnostic. A proven child's unanswered sends stay in doubt.
// A queued message was never handed to this child, so it is provably unwritten. A child
// that never proved its start accepted nothing either — input is written only after it
// initializes — so every send it left unanswered is rejected with the child's own diagnostic.
// A proven child's handed-over sends stay in doubt.
await input.journal.rejectQueuedSubmissions(
input.fence,
input.exitedDuringStartup
? providerStartupFailureRejection(input.unexpectedExitReason)
: input.unexpectedExitReason === undefined
? DISPATCH_REJECTED_PROVIDER_CLOSED
: providerExitBeforeDeliveryRejection(input.unexpectedExitReason)
)
await (input.exitedDuringStartup
? input.journal.rejectPendingSubmissions(
input.fence,
@@ -247,7 +266,9 @@ function hasUnsettledSubmission(journal: DeadGenerationJournal): boolean {
return submissions
? submissions.some(
(submission) =>
submission.dispatchState === 'pending' ||
// A queued message is not work in progress: nothing has it yet.
(submission.dispatchState === 'pending' &&
!(submission.handoverRecorded && submission.handedOverAt === undefined)) ||
(submission.dispatchState === 'unknown' && submission.recovered !== true)
)
: (journal.pendingSubmissions?.().length ?? 0) > 0
@@ -0,0 +1,152 @@
// The one thing that starts a provider child for a send, and the one thing that hands a message
// to it.
//
// A send is accepted on its own serialized step and returns; this loop does the rest. It exists
// for a session exactly while a message is queued there — accepted, not yet handed over — and
// every step re-reads the journal to decide, so there is no loop state to disagree with it.
// Each step is its own serialized task. That is what lets a Stop that arrives while a start holds
// the queue withdraw the queued messages before the handover that would have written them.
import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire'
import { DISPATCH_REJECTED_HOST_RESTARTED } from '../../../shared/structured-agent-session-dispatch-rejection'
import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter'
import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-hold-resume'
import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types'
import {
oldestQueuedSubmission,
recordStructuredAgentSessionStartFailure
} from './structured-agent-session-start-failure-row'
import { handOverSubmission } from './structured-agent-session-turns'
export type StructuredAgentSessionDeliveryLoopDeps = {
sessions: ReadonlyMap<string, StructuredAgentSessionHostSession>
adapter: StructuredAgentSessionAdapter
serialize: <T>(sessionId: string, task: () => Promise<T>) => Promise<T>
/** Gives the session a provider child if it has none; for a caller inside `serialize`. */
ensureProviderChild: (sessionId: string) => Promise<StructuredAgentSessionResumeOutcome>
/** What the chat says when the session could not be made ready. */
startFailureText: (sessionId: string, cause: AgentSessionWireRefusal) => string
onError: (sessionId: string, error: unknown) => void
}
type Step = 'continue' | 'stop'
export class StructuredAgentSessionDeliveryLoop {
private readonly running = new Set<string>()
private disposed = false
constructor(private readonly deps: StructuredAgentSessionDeliveryLoopDeps) {}
isRunning(sessionId: string): boolean {
return this.running.has(sessionId)
}
/** Quit: no step after this one starts a child or hands a message over; what is still queued
* is left for the next open, which rejects it as never sent. */
dispose(): void {
this.disposed = true
}
/** From inside the session's serialize, after a message was accepted or the conversation
* opened. A loop already running re-reads the journal on its next step. */
wake(sessionId: string): void {
if (this.disposed || this.running.has(sessionId)) {
return
}
this.running.add(sessionId)
void this.run(sessionId)
}
private async run(sessionId: string): Promise<void> {
try {
for (;;) {
const ready = await this.deps.serialize(sessionId, () => this.prepare(sessionId))
if (ready === 'stop') {
return
}
if (!ready.ok) {
const text = this.deps.startFailureText(sessionId, ready.refusal)
await this.deps.serialize(sessionId, () => this.fail(sessionId, text))
return
}
// A child published before it proved its start takes no input yet; waited for outside
// the queue so a Stop can reach it meanwhile.
await this.deps.adapter.awaitStarted?.(sessionId)
if ((await this.deps.serialize(sessionId, () => this.handOver(sessionId))) === 'stop') {
return
}
}
} catch (error) {
this.deps.onError(sessionId, error)
const text = this.deps.startFailureText(sessionId, {
code: 'agent_session_owner_restart_failed',
message: error instanceof Error ? error.message : String(error)
})
await this.deps
.serialize(sessionId, () => this.fail(sessionId, text))
.catch((failure: unknown) => {
// Rows left queued are rejected by the next open, or by the next loop an accept wakes.
this.running.delete(sessionId)
this.deps.onError(sessionId, failure)
})
}
}
/** Settles what an earlier host process left queued, then makes the session ready. */
private async prepare(sessionId: string): Promise<StructuredAgentSessionResumeOutcome | 'stop'> {
const session = this.deps.sessions.get(sessionId)
if (!session || this.disposed) {
return this.stop(sessionId)
}
await session.journal.rejectQueuedSubmissions(
session.fence,
DISPATCH_REJECTED_HOST_RESTARTED,
// A handle closes only with nothing queued, so one an earlier handle wrote is a leftover.
(submission) => session.journal.wroteBeforeOpen(submission.acceptedSequence)
)
if (!oldestQueuedSubmission(session)) {
return this.stop(sessionId)
}
return this.deps.ensureProviderChild(sessionId)
}
private async handOver(sessionId: string): Promise<Step> {
const session = this.deps.sessions.get(sessionId)
if (!session || this.disposed) {
return this.stop(sessionId)
}
// Re-derived here, not carried from the start: the child may have gone since.
if (!session.hasProviderChild) {
return 'continue'
}
const next = oldestQueuedSubmission(session)
if (!next) {
return this.stop(sessionId)
}
await handOverSubmission(
{
sessionId,
journal: session.journal,
fence: session.fence,
adapter: this.deps.adapter,
providerChildPhase: () => this.deps.sessions.get(sessionId)?.providerChildPhase
},
next
)
return 'continue'
}
private async fail(sessionId: string, text: string): Promise<Step> {
const session = this.deps.sessions.get(sessionId)
if (session) {
await recordStructuredAgentSessionStartFailure(session, text)
}
return this.stop(sessionId)
}
/** Inside the serialized step that found nothing to do, so an accept after it wakes anew. */
private stop(sessionId: string): 'stop' {
this.running.delete(sessionId)
return 'stop'
}
}
@@ -21,11 +21,10 @@ import type { StructuredAgentSessionAttachContext } from './structured-agent-ses
import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types'
import { StructuredAgentSessionStatusFeed } from './structured-agent-session-status-feed'
// Everything before the journal is out of scope here; what matters is that the orchestration's
// own `onAttachFailed` runs, which is the real one.
// Everything before the journal is out of scope here; what matters is what the orchestration does
// when the attach throws after acquisition.
vi.mock('./structured-agent-session-attach-flow', () => ({
performAttach: async (input: { onAttachFailed?: () => Promise<void> }) => {
await input.onAttachFailed?.()
performAttach: async () => {
throw new Error('attach failed after acquisition')
}
}))
@@ -230,15 +229,19 @@ describe('a session that leaves the host without an explicit close', () => {
expect(server.getStatusSnapshot()).toEqual([expect.objectContaining({ prompt: 'other host' })])
})
it('leaves the agent-status store with it when an attach fails', async () => {
// A failed attach no longer drops the session: the conversation stays open for the failure to be
// written into, so its row stays with it and the later close forgets both together.
it('keeps the session and its status row together when an attach fails', async () => {
const { server, feed, sessions } = await workingSession()
const drop = vi.spyOn(server, 'dropStructuredStatus')
await expect(
attachStructuredAgentSession(attachContext(sessions, feed), 'caller-1', attachParams)
).rejects.toThrow('attach failed after acquisition')
expect(sessions.has(SESSION)).toBe(false)
expect(server.getStatusSnapshot()).toEqual([])
expect(sessions.has(SESSION)).toBe(true)
expect(drop).not.toHaveBeenCalled()
expect(server.getStatusSnapshot()).toHaveLength(1)
})
// The feed's own cache deliberately retains the projection for reload history; only the store
@@ -0,0 +1,91 @@
// The host's conversations: how one becomes open, and the delivery loop that hands its accepted
// messages to a provider child. Bundled because they share one invariant — a conversation open
// with a message queued has a delivery loop — and the open is where a loop for leftovers wakes.
import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission'
import type { AgentJournalResetReason } from '../../../shared/agent-session-journal-types'
import type { AgentSessionJournal } from '../agent-session-journal/journal-store'
import {
openStructuredAgentSessionConversation,
type OpenedStructuredAgentSessionConversation
} from './structured-agent-session-conversation-open'
import { StructuredAgentSessionDeliveryLoop } from './structured-agent-session-delivery-loop'
import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-hold-resume'
import type {
StructuredAgentSessionHostDeps,
StructuredAgentSessionHostSession
} from './structured-agent-session-host-types'
import { structuredAgentSessionStartFailureText } from './structured-agent-session-send-preparation'
import { settleInterruptedCompaction } from './structured-compaction-recovery'
import { recoverStructuredRewind } from './structured-rewind-recovery'
export type StructuredAgentSessionConversationDelivery = {
loop: StructuredAgentSessionDeliveryLoop
/** For a caller inside the session's serialize. */
open: (sessionId: string) => Promise<StructuredAgentSessionHostSession | null>
/** Indexes a conversation some other open produced, as `open` would have. */
adoptOpened: (
sessionId: string,
opened: OpenedStructuredAgentSessionConversation
) => Promise<void>
}
export function createStructuredAgentSessionConversationDelivery(input: {
deps: StructuredAgentSessionHostDeps
sessions: Map<string, StructuredAgentSessionHostSession>
serialize: <T>(sessionId: string, task: () => Promise<T>) => Promise<T>
ensureProviderChild: (sessionId: string) => Promise<StructuredAgentSessionResumeOutcome>
reset: (sessionId: string, journal: AgentSessionJournal, reset: AgentJournalResetReason) => void
publishRestored: (sessionId: string) => void
}): StructuredAgentSessionConversationDelivery {
const { deps, sessions } = input
const loop = new StructuredAgentSessionDeliveryLoop({
sessions,
adapter: deps.adapter,
serialize: input.serialize,
ensureProviderChild: input.ensureProviderChild,
startFailureText: (sessionId, cause) =>
structuredAgentSessionStartFailureText(deps.store.getRecord(sessionId), cause),
onError: (sessionId, error) => deps.onEventSinkError?.({ sessionId, error })
})
const adoptOpened = async (
sessionId: string,
opened: OpenedStructuredAgentSessionConversation
): Promise<void> => {
const { session, reset } = opened
sessions.set(sessionId, session)
if (reset) {
input.reset(sessionId, session.journal, reset)
}
input.publishRestored(sessionId)
await settleInterruptedCommands(deps, sessionId, session)
if (session.journal.submissions().some(isQueuedAgentJournalSubmission)) {
loop.wake(sessionId)
}
}
return {
loop,
adoptOpened,
open: (sessionId) =>
openStructuredAgentSessionConversation({ deps, sessions, adoptOpened }, sessionId)
}
}
/**
* A compaction or rewind found prepared when the conversation opens was started under a child
* this process no longer has — the open runs only when none is indexed — so nothing will finish
* it, and left alone it refuses every send until a view attaches. Settled here instead of by a
* start inside acceptance. A Codex rewind only its provider can prove stays for the attach.
*/
async function settleInterruptedCommands(
deps: StructuredAgentSessionHostDeps,
sessionId: string,
session: StructuredAgentSessionHostSession
): Promise<void> {
try {
await settleInterruptedCompaction(deps.store, sessionId, session.journal, session.fence)
await recoverStructuredRewind(deps.store, sessionId, session.journal, session.fence)
} catch (error) {
deps.onEventSinkError?.({ sessionId, error })
}
}
@@ -6,6 +6,8 @@
// do.
import { activeStructuredAgentSessionTurnId } from '../../../shared/structured-agent-session-projection'
import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission'
import { DISPATCH_REJECTED_PROVIDER_CLOSED } from '../../../shared/structured-agent-session-dispatch-rejection'
import {
evictStructuredAgentSession,
STRUCTURED_AGENT_SESSION_EVICTION_STEPS,
@@ -38,12 +40,18 @@ export type StructuredAgentSessionLifetimeContext = {
}
/** Dropping a session and dropping its status row are ONE operation: the store keeps the row until
* told, so a caller that only deletes strands a live-looking row no reader can ever decay. */
* told, so a caller that only deletes strands a live-looking row no reader can ever decay. A
* handle closes with nothing queued: what is still queued now will not be handed over. */
export async function forgetStructuredAgentSession(
context: StructuredAgentSessionLifetimeContext,
sessionId: string
): Promise<void> {
await context.sessions.get(sessionId)?.journal.close()
const session = context.sessions.get(sessionId)
await session?.journal
.rejectQueuedSubmissions(session.fence, DISPATCH_REJECTED_PROVIDER_CLOSED)
// Best effort: the next open rejects a leftover itself.
.catch((error: unknown) => context.deps.onEventSinkError?.({ sessionId, error }))
await session?.journal.close()
context.sessions.delete(sessionId)
context.forgetStatus(sessionId)
}
@@ -175,7 +183,8 @@ export async function evictOwnedStructuredAgentSessions(
* resume and a send's ensure-owner step are the same serialized attach with a different asker. */
export function createStructuredAgentSessionHolds(
attachContext: () => StructuredAgentSessionAttachContext,
close: (sessionId: string) => Promise<void>
close: (sessionId: string) => Promise<void>,
deliveryActive: (sessionId: string) => boolean
): StructuredAgentSessionHolds {
const context = attachContext()
return new StructuredAgentSessionHolds({
@@ -196,12 +205,15 @@ export function createStructuredAgentSessionHolds(
},
evict: close,
hasProviderChild: (sessionId) => hasProviderChild(context, sessionId),
// A send pending while the child is still starting is held for that start; evicting would
// refuse it. Any other pending send may wait on an echo that never comes, so eviction retires it.
// A message accepted and not yet handed over is owed to this child, and so is one pending while
// the child still starts. Any other pending send may wait on an echo that never comes, so
// eviction retires it.
hasOwedWork: (sessionId) => {
const session = context.sessions.get(sessionId)
return session
? activeStructuredAgentSessionTurnId(session.journal.snapshot().items) !== null ||
deliveryActive(sessionId) ||
session.journal.submissions().some(isQueuedAgentJournalSubmission) ||
(session.providerChildPhase === 'starting' &&
session.journal.pendingSubmissions().length > 0)
: false
@@ -3,8 +3,8 @@
//
// They share one shape — admit the envelope against the lease, run a plan, publish the journal — so
// they share one path here rather than five copies in the host. The host keeps attach, holds and
// teardown. A send is the one mutation that may need those first: it makes sure the session has
// an owner as a step of its own serialized admission, see `structured-agent-session-send-preparation`.
// teardown. A send and a Stop are conversation writes: they open the conversation and are admitted
// without the writer lease; the session's delivery loop starts the provider child a send needs.
import type {
AgentJournalItemIdentity,
@@ -21,14 +21,14 @@ import type {
AgentSessionThreadGoalChange,
AgentSessionThreadGoalResult
} from '../../../shared/agent-session-wire'
import type { StructuredAgentSessionHolds } from './structured-agent-session-holds'
import { DISPATCH_REJECTED_CANCELLED } from '../../../shared/structured-agent-session-dispatch-rejection'
import { threadGoalPlan } from './structured-agent-session-thread-goal'
import {
admitAndRunAgentSessionMutation,
type AgentSessionMutationRequest
} from './structured-agent-session-mutation-admission'
import {
prepareStructuredAgentSessionSend,
openConversationForWrite,
structuredAgentSessionSendBlock
} from './structured-agent-session-send-preparation'
import {
@@ -51,11 +51,12 @@ export type StructuredAgentSessionMutationContext = {
flushStreamedEvents: (sessionId: string) => Promise<void>
requireSession: (sessionId: string) => StructuredAgentSessionHostSession
serialize: <T>(sessionId: string, task: () => Promise<T>) => Promise<T>
/** A send that finds the owner gone brings it back through here, inside its own serialize. */
holds: Pick<StructuredAgentSessionHolds, 'ensureProviderChild'>
/** Makes a closed session's journal readable again, inside the caller's serialize, for a send
* the ledger answers without an owner. */
restoreReadable: (sessionId: string) => Promise<boolean>
/** The session's conversation, opened when closed; inside the caller's serialize. */
openConversation: (sessionId: string) => Promise<StructuredAgentSessionHostSession | null>
/** A message was accepted: the session's delivery loop hands it over. */
wakeDelivery: (sessionId: string) => void
/** Stops a provider child that has not proven its start; inside the caller's serialize. */
stopStartingChild: (sessionId: string) => Promise<void>
now: () => number
}
@@ -100,12 +101,19 @@ export function sendStructuredAgentSessionTurn(
params.envelope,
{
...plan,
run: (ctx) => {
run: async (ctx) => {
const blocked = structuredAgentSessionSendBlock(context.deps.store.getRecord(ctx.sessionId))
return blocked ? Promise.resolve(blocked) : plan.run(ctx)
if (blocked) {
return blocked
}
const accepted = await plan.run(ctx)
if (accepted.ok) {
context.wakeDelivery(ctx.sessionId)
}
return accepted
}
},
(ledger, record) => prepareStructuredAgentSessionSend(context, params.envelope, ledger, record)
() => openConversationForWrite(context.openConversation, params.envelope)
)
}
@@ -130,7 +138,35 @@ export function cancelStructuredAgentSessionTurn(
context.serialize(`compact-cancel:${sessionId}`, task)
}
: context
return mutate(cancellationContext, caller, params.envelope, cancelPlan(params))
const plan = cancelPlan(params)
if (params.scope || params.prompt) {
return mutate(cancellationContext, caller, params.envelope, plan)
}
return mutate(
cancellationContext,
caller,
params.envelope,
{
...plan,
run: async (ctx) => {
// Stop withdraws every queued message first, whatever the start or the child is doing.
const withdrawn = await ctx.journal.rejectQueuedSubmissions(
ctx.fence,
DISPATCH_REJECTED_CANCELLED
)
const session = context.sessions.get(ctx.sessionId)
if (session?.hasProviderChild && session.providerChildPhase === 'starting') {
// A start that may never land is the one thing here Stop has to end.
await context.stopStartingChild(ctx.sessionId)
return { ok: true, value: { turnId: params.turnId, cancelled: true } }
}
return session?.hasProviderChild
? plan.run(ctx)
: { ok: true, value: { turnId: params.turnId, cancelled: withdrawn.length > 0 } }
}
},
() => openConversationForWrite(context.openConversation, params.envelope)
)
}
export function respondToStructuredAgentSessionPrompt(
@@ -766,9 +766,19 @@ describe('subscribe', () => {
body
})
expect(result).toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } })
expect(dispatch).toHaveBeenCalledTimes(1)
expect(events.some((event) => event.type === 'batch')).toBe(true)
expect(result).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } })
// The failed transport does not stop the delivery loop either: the handover still lands and
// reaches the live subscriber.
await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1))
await vi.waitFor(() =>
expect(
events.some(
(event) =>
event.type === 'batch' &&
event.batch.submissions?.some((entry) => entry.dispatchState === 'accepted')
)
).toBe(true)
)
})
it('resets a subscriber whose epoch is gone', async () => {
@@ -52,6 +52,7 @@ import {
type StructuredAgentSessionRestartResume
} from './structured-agent-session-restart-resume-host'
import { structuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-wiring'
import { createStructuredAgentSessionConversationDelivery } from './structured-agent-session-host-delivery'
export type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types'
export class StructuredAgentSessionHost {
@@ -77,6 +78,9 @@ export class StructuredAgentSessionHost {
) => Promise<SessionWire.AgentSessionWireRefusal | null>
private readonly restore: ReturnType<typeof createStructuredAgentSessionHostRestore>
private readonly holds: StructuredAgentSessionHolds
private readonly conversationDelivery: ReturnType<
typeof createStructuredAgentSessionConversationDelivery
>
private readonly eventRecovery: StructuredAgentSessionEventRecovery
private readonly backgroundTasks: StructuredAgentSessionBackgroundTaskChannel
/** Public because the RPC surface addresses it directly; see the restart-resume collaborator. */
@@ -99,9 +103,18 @@ export class StructuredAgentSessionHost {
...(deps.probeOwners ? { probeMany: deps.probeOwners } : {}),
now: () => this.now()
})
this.conversationDelivery = createStructuredAgentSessionConversationDelivery({
deps,
sessions: this.sessions,
serialize: (sessionId, task) => this.serialize(sessionId, task),
ensureProviderChild: (sessionId) => this.holds.ensureProviderChild(sessionId),
reset: (...args) => this.subscribers.reset(...args),
publishRestored: this.clientDelivery.publishRestored
})
this.holds = createStructuredAgentSessionHolds(
() => this.attachContext(),
(sessionId) => this.close(sessionId)
(sessionId) => this.close(sessionId),
(sessionId) => this.conversationDelivery.loop.isRunning(sessionId)
)
this.restore = createStructuredAgentSessionHostRestore(deps, this.sessions, () => this.now(), {
reconcile: this.reconcileLeases,
@@ -110,10 +123,7 @@ export class StructuredAgentSessionHost {
hasSession: this.hasSession,
// Site 10: cannot overwrite a live entry — the restorer returns early on
// `hasSession` inside the same serialized step as this `set`.
onReadable: (sessionId, restored) => {
this.sessions.set(sessionId, restored)
this.clientDelivery.publishRestored(sessionId)
}
onReadable: this.conversationDelivery.adoptOpened
})
this.eventRecovery = new StructuredAgentSessionEventRecovery({
deps,
@@ -175,18 +185,21 @@ export class StructuredAgentSessionHost {
tasks: this.tasks,
reconcileLeases: (sessionId) => this.reconcileLeases(sessionId),
serialize: (sessionId, task) => this.serialize(sessionId, task),
publishStatus: this.clientDelivery.publishStatus
publishStatus: this.clientDelivery.publishStatus,
openConversation: this.conversationDelivery.open
}
}
/** Releases a session's resources without ending the conversation: the record and journal stay
* on disk, so the same session can be attached again. */
close(sessionId: string): Promise<void> {
return this.serialize(sessionId, async () => {
await evictHeldStructuredAgentSession(this.lifetimeContext(), sessionId)
this.clientDelivery.closeSession(sessionId)
// The holders now look at a session that is gone; a failed eviction throws above, keeping them.
this.holds.forget(sessionId)
})
return this.serialize(sessionId, () => this.closeUnderSerialize(sessionId))
}
private async closeUnderSerialize(sessionId: string): Promise<void> {
await evictHeldStructuredAgentSession(this.lifetimeContext(), sessionId)
this.clientDelivery.closeSession(sessionId)
// The holders now look at a session that is gone; a failed eviction throws above, keeping them.
this.holds.forget(sessionId)
}
supportsCreate = (location: AgentSessionExecutionLocation, agent: string): boolean =>
@@ -227,6 +240,7 @@ export class StructuredAgentSessionHost {
// Trigger inlined rather than imported: `AgentSessionResumeTrigger` in shared is the canonical
// type, and this file has no line budget left for the import.
async flushAllStreamedEvents(options?: { trigger?: 'quit' | 'update' }): Promise<void> {
this.conversationDelivery.loop.dispose()
await flushStructuredAgentSessionHost({
...this.lifetimeContext(),
holds: this.holds,
@@ -245,8 +259,9 @@ export class StructuredAgentSessionHost {
flushStreamedEvents: this.flushStreamedEvents,
requireSession: (sessionId) => this.requireSession(sessionId),
serialize: (sessionId, task) => this.serialize(sessionId, task),
holds: this.holds,
restoreReadable: (sessionId) => this.restore.restoreReadableUnderSerialize(sessionId),
openConversation: this.conversationDelivery.open,
wakeDelivery: (sessionId) => this.conversationDelivery.loop.wake(sessionId),
stopStartingChild: (sessionId) => this.closeUnderSerialize(sessionId),
now: () => this.now()
}
}
@@ -71,6 +71,18 @@ function journal(): AgentSessionJournal {
).sessions.get(SESSION)!.journal
}
/** A send is accepted first; this waits for the delivery loop to hand it to the provider. */
async function handedOver(clientMessageId: string): Promise<void> {
await vi.waitFor(() => {
expect(
journal()
.submissions()
.find((entry) => entry.clientMessageId === clientMessageId)?.handedOverAt
).toBeDefined()
expect(dispatch).toHaveBeenCalled()
})
}
beforeEach(async () => {
root = await mkdtemp(join(tmpdir(), 'orca-wire-late-settle-'))
resetHostTestOperationIds()
@@ -152,10 +164,13 @@ describe('settling a send the provider proves it received after the ack window',
finishDispatch({ state: 'unknown', reason: 'ack timeout' })
unsubscribe()
}
await expect(pending).resolves.toMatchObject({
ok: true,
value: { submission: { dispatchState: 'accepted' } }
})
await expect(pending).resolves.toMatchObject({ ok: true })
// The late `unknown` from the handover does not reopen the proven acceptance.
await vi.waitFor(() =>
expect(submissions()).toMatchObject([
{ clientMessageId: params.envelope.clientOperationId, dispatchState: 'accepted' }
])
)
await expect(host.send(CALLER, { ...params, retryUnknown: true })).resolves.toMatchObject({
ok: true,
value: { submission: { dispatchState: 'accepted' } }
@@ -167,6 +182,8 @@ describe('settling a send the provider proves it received after the ack window',
dispatch.mockResolvedValueOnce({ state: 'unknown', reason: 'ack timeout' })
const params = sendParams('received just before shutdown')
await host.send(CALLER, params)
await handedOver(params.envelope.clientOperationId)
await vi.waitFor(() => expect(submissions()).toMatchObject([{ dispatchState: 'unknown' }]))
let settlement: Promise<void> | undefined
closeSession.mockImplementationOnce(async () => {
settlement = host.settleLateDispatch({
@@ -188,8 +205,9 @@ describe('settling a send the provider proves it received after the ack window',
it('moves a durable unknown to accepted so nothing offers to send it again', async () => {
dispatch.mockRejectedValueOnce(new Error('socket closed'))
const params = sendParams('sent while a turn was running')
const first = await host.send(CALLER, params)
expect(first).toMatchObject({ ok: true, value: { submission: { dispatchState: 'unknown' } } })
await host.send(CALLER, params)
await handedOver(params.envelope.clientOperationId)
await vi.waitFor(() => expect(submissions()).toMatchObject([{ dispatchState: 'unknown' }]))
await host.settleLateDispatch({
sessionId: SESSION,
@@ -209,6 +227,7 @@ describe('settling a send the provider proves it received after the ack window',
dispatch.mockResolvedValueOnce({ state: 'admitted' })
const params = sendParams('queued behind the active turn')
await host.send(CALLER, params)
await handedOver(params.envelope.clientOperationId)
await host.settleLateDispatch({
sessionId: SESSION,
@@ -230,6 +249,7 @@ describe('settling a send the provider proves it received after the ack window',
dispatch.mockResolvedValueOnce({ state: 'admitted' })
const params = sendParams('settle from provider echo')
await host.send(CALLER, params)
await handedOver(params.envelope.clientOperationId)
vi.spyOn(journal(), 'resolveDispatch').mockRejectedValueOnce(
new Error('direct settlement write failed')
)
@@ -259,6 +279,7 @@ describe('settling a send the provider proves it received after the ack window',
it('leaves an already accepted send alone', async () => {
const params = sendParams('ordinary send')
await host.send(CALLER, params)
await vi.waitFor(() => expect(submissions()).toMatchObject([{ dispatchState: 'accepted' }]))
await host.settleLateDispatch({
sessionId: SESSION,
@@ -57,12 +57,20 @@ async function launchAndDeliver(): Promise<{
text: 'fix the failing test'
})
const sent = await send.mock.results[0]!.value
return {
messageId,
dispatchState: sent.ok
? sent.value.submission.dispatchState
: `refused:${sent.refusal.code}:${sent.refusal.message}`
if (!sent.ok) {
return { messageId, dispatchState: `refused:${sent.refusal.code}:${sent.refusal.message}` }
}
// Accepted first; the delivery loop hands it over, and that outcome is what reached the agent.
let dispatchState = sent.value.submission.dispatchState
await vi.waitFor(() => {
dispatchState =
host
.journalSnapshot(created.value.sessionId)
.submissions.find((entry) => entry.clientMessageId === sent.value.clientMessageId)
?.dispatchState ?? 'missing'
expect(dispatchState).not.toBe('pending')
})
return { messageId, dispatchState }
}
beforeEach(() => {
@@ -25,12 +25,18 @@ import {
const CALLER = { callerKey: 'client-1' }
/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */
function eventually(assertion: () => void): Promise<void> {
return vi.waitFor(assertion, { timeout: 10_000 })
}
let root: string
let store: AgentSessionRecordStore
let host: StructuredAgentSessionHost
let acquire: Mock<StructuredAgentSessionAdapter['acquire']>
let probe: Mock<() => Promise<AgentSessionOwnerProbe>>
let stopOwnerProcess: Mock<(pid: number, signal: 'SIGTERM' | 'SIGKILL') => void>
let dispatch: Mock<StructuredAgentSessionAdapter['dispatch']>
function openHost(): void {
host = new StructuredAgentSessionHost({
@@ -39,7 +45,7 @@ function openHost(): void {
acquire,
closeSession: vi.fn(async () => true),
releaseAcquisition: vi.fn(async () => true),
dispatch: vi.fn(async () => ({ state: 'admitted' as const })),
dispatch,
cancelTurn: vi.fn(async () => ({ cancelled: false })),
answerPrompt: vi.fn(async () => undefined),
setOption: vi.fn(async () => undefined)
@@ -65,6 +71,24 @@ async function persistFromOlderBuild(lease: Partial<PersistedAgentSessionLease>)
openHost()
}
/** A send is accepted at once; what became of it is the submission's state once the host's
* delivery settles it — handed over, or rejected with the reason the chat shows. */
async function delivered(text: string) {
const sent = await send(text)
expect(sent).toMatchObject({ ok: true })
const clientMessageId = sent.ok ? sent.value.clientMessageId : ''
const submission = () =>
host
.journalSnapshot(SESSION)
.submissions.find((candidate) => candidate.clientMessageId === clientMessageId)
await eventually(() =>
expect(
submission()?.dispatchState !== 'pending' || submission()?.handedOverAt !== undefined
).toBe(true)
)
return submission()
}
async function send(text: string) {
const body = hostTestMessage(text)
return host.send(CALLER, {
@@ -87,6 +111,7 @@ beforeEach(async () => {
resetHostTestOperationIds()
probe = vi.fn(async () => ({ outcome: 'pid-absent' as const }))
stopOwnerProcess = vi.fn()
dispatch = vi.fn(async () => ({ state: 'admitted' as const }))
acquire = vi.fn(async ({ fence, spawnToken }) => ({
process: { hostId: 'local', pid: 4242, processStartTimeMs: NOW - 1_000, spawnToken },
link: {
@@ -137,7 +162,8 @@ describe('a record an older build left mid terminal handoff', () => {
handoffOperationId: null,
settlementRetryRequired: undefined
})
expect(await send('after the upgrade')).toMatchObject({ ok: true })
expect(await delivered('after the upgrade')).toMatchObject({ dispatchState: 'pending' })
expect(dispatch).toHaveBeenCalledOnce()
expect(acquire).toHaveBeenCalledOnce()
expect(store.getRecord(SESSION)?.lease).toMatchObject({
runtimeKind: 'native',
@@ -169,7 +195,8 @@ describe('a record an older build left mid terminal handoff', () => {
handoffStage: null,
handoffOperationId: null
})
expect(await send('after the upgrade')).toMatchObject({ ok: true })
expect(await delivered('after the upgrade')).toMatchObject({ dispatchState: 'pending' })
expect(dispatch).toHaveBeenCalledOnce()
expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'live' })
})
@@ -188,7 +215,8 @@ describe('a record an older build left mid terminal handoff', () => {
handoffStage: null,
claimStatus: 'released'
})
expect(await send('after the upgrade')).toMatchObject({ ok: true })
expect(await delivered('after the upgrade')).toMatchObject({ dispatchState: 'pending' })
expect(dispatch).toHaveBeenCalledOnce()
expect(store.getRecord(SESSION)?.lease).toMatchObject({
runtimeKind: 'native',
claimStatus: 'live'
@@ -210,10 +238,16 @@ describe('a record an older build left mid terminal handoff', () => {
claimStatus: 'conflicted',
handoffStage: 'manual-recovery'
})
expect(await send('while the terminal still runs')).toMatchObject({
ok: false,
refusal: { code: 'agent_session_conflict' }
// Accepted, then rejected by the start that cannot take the lease: the chat says why.
expect(await delivered('while the terminal still runs')).toMatchObject({
dispatchState: 'rejected'
})
expect(
host
.journalSnapshot(SESSION)
.items.filter((item) => item.body.kind === 'status' && item.body.tone === 'error')
).toHaveLength(1)
expect(dispatch).not.toHaveBeenCalled()
expect(stopOwnerProcess).not.toHaveBeenCalled()
expect(acquire).not.toHaveBeenCalled()
@@ -222,7 +256,10 @@ describe('a record an older build left mid terminal handoff', () => {
await host.hold(SESSION, 'surface-1')
expect(stopOwnerProcess).not.toHaveBeenCalled()
expect(await send('after the terminal closed')).toMatchObject({ ok: true })
expect(await delivered('after the terminal closed')).toMatchObject({
dispatchState: 'pending'
})
expect(dispatch).toHaveBeenCalledOnce()
expect(store.getRecord(SESSION)?.lease).toMatchObject({
runtimeKind: 'native',
claimStatus: 'live',
@@ -107,7 +107,8 @@ export async function admitAndRunAgentSessionMutation<TValue>(
envelope,
hostFingerprint,
now: request.now(),
...(plan.operationIdScope ? { operationIdScope: plan.operationIdScope } : {})
...(plan.operationIdScope ? { operationIdScope: plan.operationIdScope } : {}),
...(plan.conversationWrite ? { conversationWrite: true } : {})
})
if (!admitted) {
return refuseAgentSessionMutation(AGENT_SESSION_NOT_ATTACHED)
@@ -141,7 +142,8 @@ export async function admitAndRunAgentSessionMutation<TValue>(
envelope,
hostFingerprint,
ledger: { decision: 'admit', row: admission.row },
lease: record.lease
lease: record.lease,
...(plan.conversationWrite ? { conversationWrite: true } : {})
})
if (rerun.decision === 'refused') {
return refuseAgentSessionMutation(rerun.refusal)
@@ -28,8 +28,9 @@ export type MutationPlan<TValue> = {
method: string
fields: Record<string, unknown>
operationIdScope?: 'global'
/** Admitted without the writer lease: see `admitAgentSessionMutation`. */
conversationWrite?: true
markUnknownBeforeRun?: boolean
beforeRun?: () => void
run: (ctx: AgentSessionTurnContext) => Promise<TurnOutcome<TValue>>
replay: (ctx: AgentSessionTurnContext, outcome: AgentSessionOperationOutcome) => TValue | null
rerunWhenReplayMissing?: (ctx: AgentSessionTurnContext) => boolean
@@ -49,19 +50,22 @@ export function sendPlan(params: {
return {
method: 'agentSession.send',
operationIdScope: 'global',
conversationWrite: true,
markUnknownBeforeRun: true,
// A control signal is not payload; it cannot alter durable replay.
fields: { body: params.body },
...(params.beforeRun ? { beforeRun: params.beforeRun } : {}),
recoverUnknownFromDurableState: true,
// `retryUnknown` is a compatibility-only client signal. A recorded send
// always replays and never reaches the provider twice.
run: (ctx) =>
performSend(ctx, {
run: (ctx) => {
// Asked at acceptance: a send accepted after this one is queued behind it.
params.beforeRun?.()
return performSend(ctx, {
clientMessageId,
payloadFingerprint: params.envelope.payloadFingerprint,
body: params.body
}),
})
},
replay: (ctx, outcome) => {
const submission = ctx.journal
.submissions()
@@ -100,6 +104,8 @@ export function cancelPlan(params: {
}): MutationPlan<AgentSessionCancelResult> {
return {
method: 'agentSession.cancel',
// Stop is a conversation write; a prompt or background-task cancel needs the live child.
...(params.scope || params.prompt ? {} : { conversationWrite: true as const }),
fields: {
turnId: params.turnId,
...(params.scope ? { scope: params.scope } : {}),
@@ -2,7 +2,6 @@ import { join } from 'node:path'
import { afterEach, expect, it, vi } from 'vitest'
import {
AgentSessionPreDispatchError,
AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS,
runSettledAgentSessionMutation
} from './structured-agent-session-operation-settlement'
import {
@@ -42,6 +41,9 @@ async function context(): Promise<AgentSessionTurnContext> {
}
}
/** Longer than any bookkeeping bound: a refusal must already be answered by then. */
const SETTLED_WAIT_MS = 2_000
afterEach(() => {
vi.useRealTimers()
vi.restoreAllMocks()
@@ -80,7 +82,7 @@ it('returns a pre-dispatch refusal without waiting on redundant uncertainty pers
})
try {
await refusing.promise
await vi.advanceTimersByTimeAsync(AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS)
await vi.advanceTimersByTimeAsync(SETTLED_WAIT_MS)
expect(returned).toBe(true)
expect(writes).toHaveBeenCalledOnce()
expect(hostTestState().dispatch).not.toHaveBeenCalled()
@@ -129,9 +131,10 @@ it.each([1, 2])(
}
)
// The pre-dispatch check judges only what the journal already holds; the send path never waits on
// the provider's stream barrier, so a sink that stalls or fails cannot delay or double a send.
it('dispatches without touching the event-stream barrier', async () => {
// A send's plan only accepts: it records the submission and never waits on the provider's stream
// barrier, so a sink that stalls or fails cannot delay or double a send. Handing it over is the
// delivery loop's.
it('accepts without touching the event-stream barrier or the provider', async () => {
const ctx = await context()
const { store } = hostTestState()
vi.spyOn(store, 'recordOperationOutcome').mockResolvedValue()
@@ -149,42 +152,33 @@ it('dispatches without touching the event-stream barrier', async () => {
})
expect(result).toMatchObject({ ok: true })
expect(beforeRun).toHaveBeenCalledOnce()
expect(hostTestState().dispatch).toHaveBeenCalledOnce()
expect(ctx.journal.submissions()[0]?.dispatchState).toBe('accepted')
expect(hostTestState().dispatch).not.toHaveBeenCalled()
expect(ctx.journal.submissions()[0]).toMatchObject({
dispatchState: 'pending',
handoverRecorded: true
})
expect(barrier).not.toHaveBeenCalled()
})
it('refuses a superseded send without waiting on a stalled refusal write, and never dispatches late', async () => {
it('refuses a superseded send at acceptance, recording and dispatching nothing', async () => {
const ctx = await context()
const { store } = hostTestState()
vi.spyOn(store, 'recordOperationOutcome').mockResolvedValue()
const pending = Promise.withResolvers<void>()
const refusing = Promise.withResolvers<void>()
vi.spyOn(ctx.journal, 'resolveDispatch').mockImplementationOnce(() => {
refusing.resolve()
return pending.promise.then(() => ctx.journal.cursor())
})
vi.spyOn(console, 'warn').mockImplementation(() => {})
const beforeRun = vi.fn(() => {
throw new AgentSessionPreDispatchError('agent_session_restart_work_superseded')
})
const body = hostTestMessage('Continue the interrupted work')
const operation = envelope('agentSession.send', { body })
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
const result = runSettledAgentSessionMutation({
const result = await runSettledAgentSessionMutation({
store,
operationCallerKey: 'test',
envelope: operation,
context: ctx,
plan: sendPlan({ envelope: operation, body, beforeRun })
}).catch((error: unknown) => error)
await refusing.promise
await vi.advanceTimersByTimeAsync(AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS)
expect(await result).toBeInstanceOf(AgentSessionPreDispatchError)
expect(hostTestState().dispatch).not.toHaveBeenCalled()
expect(ctx.journal.submissions()[0]?.dispatchState).toBe('pending')
pending.resolve()
await vi.advanceTimersByTimeAsync(0)
expect(result).toBeInstanceOf(AgentSessionPreDispatchError)
expect(ctx.journal.submissions()).toEqual([])
expect(hostTestState().dispatch).not.toHaveBeenCalled()
expect(vi.getTimerCount()).toBe(0)
})
@@ -11,8 +11,6 @@ export class AgentSessionPreDispatchError extends Error {
}
}
export const AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS = 2_000
export async function runSettledAgentSessionMutation<TValue>(input: {
store: AgentSessionRecordStore
operationCallerKey: string
@@ -33,10 +31,7 @@ export async function runSettledAgentSessionMutation<TValue>(input: {
if (input.plan.markUnknownBeforeRun) {
await settle({ status: 'unknown' })
}
outcome = await input.plan.run({
...input.context,
...(input.plan.beforeRun ? { beforeDispatch: input.plan.beforeRun } : {})
})
outcome = await input.plan.run(input.context)
await settle(
outcome.ok
? (input.plan.settledOutcome?.(outcome.value) ?? {
@@ -152,6 +152,11 @@ describe('structured session options and close', () => {
envelope: envelope('agentSession.send', { body }),
body
})
// Accepted, then handed over by the delivery loop; the status is read once it answered.
await vi.waitFor(() => expect(dispatchedModels).toEqual([DEFAULT_MODEL]))
await vi.waitFor(() =>
expect(host.journalSnapshot(SESSION).submissions[0]?.dispatchState).toBe('accepted')
)
const events: AgentSessionStatusEvent[] = []
host.subscribeStatus({ id: 'session-list', emit: (event) => events.push(event) })
expect(events).toEqual([
@@ -102,7 +102,7 @@ describe('a publish-first Claude create whose init is slow', () => {
expect(store.getRecord(SESSION)?.options?.model).toBeUndefined()
expect(lastPhase()).toBe('starting')
await adapter.drainStartup(SESSION)
await adapter.awaitStarted(SESSION)
await Promise.all(lifecycle)
expect(store.getRecord(SESSION)?.options?.model).toBe('claude-opus-9')
@@ -116,7 +116,7 @@ describe('a publish-first Claude create whose init is slow', () => {
).resolves.toMatchObject({ ok: true })
expect(store.getRecord(SESSION)?.options?.model).toBe('opus')
await adapter.drainStartup(SESSION)
await adapter.awaitStarted(SESSION)
await Promise.all(lifecycle)
expect(store.getRecord(SESSION)?.options?.model).toBe('opus')
@@ -126,7 +126,7 @@ describe('a publish-first Claude create whose init is slow', () => {
it('keeps the picked model across a resume whose new child starts on its own default', async () => {
const params = claudeParams()
await host.attach(CALLER, { ...params, options: { model: 'opus' } })
await adapter.drainStartup(SESSION)
await adapter.awaitStarted(SESSION)
await Promise.all(lifecycle)
await host.close(SESSION)
const releasedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0
@@ -138,7 +138,7 @@ describe('a publish-first Claude create whose init is slow', () => {
expect(store.getRecord(SESSION)?.options?.model).toBe('opus')
expect(lastPhase()).toBe('starting')
await adapter.drainStartup(SESSION)
await adapter.awaitStarted(SESSION)
await Promise.all(lifecycle)
expect(store.getRecord(SESSION)?.options?.model).toBe('opus')
@@ -48,6 +48,11 @@ const store = {
} as unknown as AgentSessionRecordStore
let journalRoot: string
const openDeps = () => ({
store,
journalRoot,
adapter: {}
})
const opened: AgentSessionJournal[] = []
async function writeRemnant(name: string): Promise<string> {
@@ -73,29 +78,29 @@ describe('a session whose journal is still the pre-SQLite format', () => {
it('is published, carrying the message that explains it', async () => {
const transcript = await writeRemnant('log.jsonl')
const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID)
const restored = await restoreStructuredAgentSessionRead(openDeps(), SESSION_ID)
expect(restored).not.toBeNull()
opened.push(restored!.journal)
const disclosed = restored!.journal
opened.push(restored!.session.journal)
const disclosed = restored!.session.journal
.snapshot()
.items.map((entry) => (entry.body.kind === 'status' ? entry.body.text : ''))
expect(disclosed.join('')).toContain(transcript)
// Publishing it costs no agent process; acquisition still waits for the user.
expect(restored!.hasProviderChild).toBe(false)
expect(restored!.session.hasProviderChild).toBe(false)
})
it('is published for a remnant whose log is gone', async () => {
await writeRemnant('snapshot.json')
const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID)
const restored = await restoreStructuredAgentSessionRead(openDeps(), SESSION_ID)
expect(restored).not.toBeNull()
opened.push(restored!.journal)
opened.push(restored!.session.journal)
})
it('still drops a session with neither a journal nor a remnant', async () => {
const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID)
const restored = await restoreStructuredAgentSessionRead(openDeps(), SESSION_ID)
expect(restored).toBeNull()
})
@@ -103,7 +108,7 @@ describe('a session whose journal is still the pre-SQLite format', () => {
it('still drops a session with no record', async () => {
await writeRemnant('log.jsonl')
const restored = await restoreStructuredAgentSessionRead(store, journalRoot, 'unknown-session')
const restored = await restoreStructuredAgentSessionRead(openDeps(), 'unknown-session')
expect(restored).toBeNull()
})
@@ -1,104 +1,32 @@
import type { AgentSessionRecord } from '../../../shared/agent-session-record'
import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store'
import { findJournalFileFormatRemnant } from '../agent-session-journal/journal-file-format-remnant'
import { loadJournal } from '../agent-session-journal/journal-open'
import { journalDirectoryFor } from '../agent-session-journal/journal-paths'
import type { AgentSessionJournal } from '../agent-session-journal/journal-store'
import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory'
import { existsSync } from 'node:fs'
import { journalDatabaseFile, journalDirectoryFor } from '../agent-session-journal/journal-paths'
import {
attachFingerprintFields,
journalIdentityFor,
type AgentSessionAttachParams
} from './structured-agent-session-attach'
import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope'
export type RestoredStructuredAgentSessionRead = {
journal: AgentSessionJournal
params: AgentSessionAttachParams
fence: number
hasProviderChild: false
providerChildPhase: 'ready'
acquisitionGeneration: null
}
openStructuredAgentSessionConversationJournal,
type OpenedStructuredAgentSessionConversation,
type StructuredAgentSessionConversationOpenDeps
} from './structured-agent-session-conversation-open'
/**
* A reader's open: the conversation's own open, for a session that has a journal to read. One
* with none — never written, or gone — stays unpublished rather than founding an empty one.
* Opening can still write: the crash boundary, and the row explaining an old-format history.
*/
export async function restoreStructuredAgentSessionRead(
store: AgentSessionRecordStore,
journalRoot: string,
deps: StructuredAgentSessionConversationOpenDeps,
sessionId: string
): Promise<RestoredStructuredAgentSessionRead | null> {
const record = store.getRecord(sessionId)
): Promise<OpenedStructuredAgentSessionConversation | null> {
const record = deps.store.getRecord(sessionId)
if (!record) {
return null
}
const params = attachParamsForRecord(record, {
clientOperationId: `read-restore:${record.sessionId}`,
expectedRuntimeFence: record.lease.runtimeFence
})
const journalDir = journalDirectoryFor(journalRoot, {
const journalDir = journalDirectoryFor(deps.journalRoot, {
workspaceId: record.location.workspaceId,
sessionId
})
const loaded = loadJournal(journalDir, sessionId)
if (loaded?.corrupt) {
// A session still in the pre-SQLite format has no `journal.db`; the open imports it.
if (!existsSync(journalDatabaseFile(journalDir)) && !findJournalFileFormatRemnant(journalDir)) {
return null
}
// A session still in the pre-SQLite format has no `journal.db` to load. Dropping
// it here leaves it unpublished, which is also what prunes its tab out of the
// saved workspace — so the chat disappears with nowhere to explain itself.
if (!loaded && !findJournalFileFormatRemnant(journalDir)) {
return null
}
const journal = await openAgentSessionJournal({
identity: journalIdentityFor(record, params),
journalDir,
// Omitted, not `null`: the store reads `null` as "replay already ran and
// found nothing" and founds a fresh epoch. In process the probe above is the
// previous statement, so the window is zero-width; this holds the line for a
// database another process creates in between.
...(loaded ? { loaded } : {})
})
// Read restore opens the journal and nothing else: no adapter call, so no
// provider child. Opening it can still write — a session whose history is in
// the old format founds its epoch and commits the row explaining that here.
return {
journal,
params,
fence: record.lease.runtimeFence,
hasProviderChild: false,
providerChildPhase: 'ready',
acquisitionGeneration: null
}
}
export function attachParamsForRecord(
record: AgentSessionRecord,
input: {
clientOperationId: string
expectedRuntimeFence: number
}
): AgentSessionAttachParams {
const params: AgentSessionAttachParams = {
envelope: {
sessionId: record.sessionId,
clientOperationId: input.clientOperationId,
expectedRuntimeFence: input.expectedRuntimeFence,
payloadFingerprint: ''
},
location: record.location,
provider: record.provider,
agent: record.provider,
accountHome: record.accountHome,
runtimeKind: 'native'
}
return {
...params,
envelope: {
...params.envelope,
payloadFingerprint: computeAgentSessionPayloadFingerprint({
method: 'agentSession.attach',
sessionId: record.sessionId,
fields: attachFingerprintFields(params)
})
}
}
return openStructuredAgentSessionConversationJournal(deps, record)
}
@@ -19,8 +19,11 @@ describe('StructuredAgentSessionReadableRestorer', () => {
(sessionId) => ({ sessionId }) as AgentSessionRecord
)
const restorer = new StructuredAgentSessionReadableRestorer({
store: { listRecords: () => records } as never,
journalRoot: '/tmp/journals',
openDeps: {
store: { getRecord: () => null, listRecords: () => records },
journalRoot: '/tmp/journals',
adapter: {}
},
supportsRecord: () => true,
reconcile: async () => null,
resolveRecovery: async () => undefined,
@@ -1,10 +1,7 @@
import type { AgentSessionRecord } from '../../../shared/agent-session-record'
import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire'
import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store'
import type { RestoredStructuredAgentSessionRead } from './structured-agent-session-read-restore'
import type { StructuredAgentSessionReadRestoreDeps } from './structured-agent-session-restart-restore'
import {
restoreOneStructuredAgentSessionRead,
restoreOneStructuredAgentSessionReadUnderSerialize,
restoreStructuredAgentSessionsOnRestart
} from './structured-agent-session-restart-restore'
@@ -12,19 +9,8 @@ export class StructuredAgentSessionReadableRestorer {
private restorePromise: Promise<void> | null = null
constructor(
private readonly input: {
store: AgentSessionRecordStore
journalRoot: string
private readonly input: StructuredAgentSessionReadRestoreDeps & {
supportsRecord: (record: AgentSessionRecord) => boolean
reconcile: (sessionId: string) => Promise<AgentSessionWireRefusal | null>
resolveRecovery: (sessionId: string) => Promise<unknown>
serialize: <T>(sessionId: string, task: () => Promise<T>) => Promise<T>
hasSession: (sessionId: string) => boolean
onReadable: (sessionId: string, restored: RestoredStructuredAgentSessionRead) => void
retrySettlement: (
sessionId: string,
params: RestoredStructuredAgentSessionRead['params']
) => Promise<boolean>
}
) {}
@@ -55,19 +41,8 @@ export class StructuredAgentSessionReadableRestorer {
return this.input.hasSession(sessionId)
}
/** `restoreOne` for a caller already inside the session's serialize. Reconciliation is skipped
* on purpose: a lease this host has not adjudicated is the attach's problem, and a replay
* needs only the journal. */
async restoreOneUnderSerialize(sessionId: string): Promise<boolean> {
if (!this.supports(sessionId)) {
return false
}
await restoreOneStructuredAgentSessionReadUnderSerialize(this.input, sessionId)
return this.input.hasSession(sessionId)
}
private supports(sessionId: string): boolean {
const record = this.input.store.getRecord(sessionId)
const record = this.input.openDeps.store.getRecord(sessionId)
return record !== null && this.input.supportsRecord(record)
}
@@ -75,7 +50,7 @@ export class StructuredAgentSessionReadableRestorer {
const targetOrder = sessionIds
? new Map(sessionIds.map((sessionId, index) => [sessionId, index]))
: null
const records = this.input.store
const records = this.input.openDeps.store
.listRecords()
.filter(
(record) =>
@@ -106,6 +106,7 @@ async function createHarness(options: { attached?: boolean } = {}) {
async function abandonHost(host: StructuredAgentSessionHost): Promise<void> {
host['runtimeState'].stopLeaseRenewal()
host['holds'].dispose()
host['conversationDelivery'].loop.dispose()
await Promise.all([...host['sessions'].values()].map((session) => session.journal.close()))
host['sessions'].clear()
}
@@ -225,16 +226,20 @@ const UNREACHABLE = new Set<Pair>([
// StructuredAgentSessionHost.mutate maps an absent record to AGENT_SESSION_NOT_ATTACHED.
'agentSession.setOption:agent_session_identity_required',
'agentSession.send:agent_session_identity_required',
// No structured-agent-session host branch emits agent_session_journal_unreadable.
// Only a send opens the conversation it writes to.
'agentSession.setOption:agent_session_journal_unreadable',
'agentSession.send:agent_session_journal_unreadable',
// Send reconstructs doubt from its global tombstone instead of refusing it.
'agentSession.send:agent_session_operation_unknown',
// Only a send restarts a lost owner.
'agentSession.setOption:agent_session_owner_restart_failed',
// A write names its target, not an owner generation; only an attach compares fences.
'agentSession.setOption:agent_session_checkpoint_stale',
'agentSession.send:agent_session_checkpoint_stale'
'agentSession.send:agent_session_checkpoint_stale',
// A send is a conversation write: admitted whoever owns the lease, and a start it needs that
// fails rejects the accepted message rather than refusing the call.
'agentSession.send:agent_session_conflict',
'agentSession.send:execution_owner_reconciling',
'agentSession.send:agent_session_owner_restart_failed'
])
describe('agentSessionRefusalOperationState host oracle', () => {
@@ -251,7 +256,7 @@ describe('agentSessionRefusalOperationState host oracle', () => {
expect(stale.setOption).toHaveBeenCalledTimes(1)
const conflict = await createHarness()
for (const method of ['agentSession.setOption', 'agentSession.send'] as const) {
for (const method of ['agentSession.setOption'] as const) {
await setLease(conflict, (current) => ({
...current,
lease: { ...current.lease, handoffStage: 'new-owner-proving' }
@@ -331,7 +336,7 @@ describe('agentSessionRefusalOperationState host oracle', () => {
record(await assertHostAgreement(unknown, optionUnknown, 'agent_session_operation_unknown'))
const reconciling = await createHarness()
for (const method of ['agentSession.setOption', 'agentSession.send'] as const) {
for (const method of ['agentSession.setOption'] as const) {
await setLease(reconciling, (current) => ({
...current,
lease: { ...current.lease, unreconciled: true }
@@ -348,16 +353,21 @@ describe('agentSessionRefusalOperationState host oracle', () => {
)
}
const unrecoverable = await createHarness()
await unrecoverable.host.close(SESSION)
unrecoverable.host.deps.adapter.acquire = async () => {
throw new Error('no provider thread to resume')
const unreadable = await createHarness()
await unreadable.host.close(SESSION)
unreadable.host.deps.adapter.historyFilePath = async () => {
throw new Error('transcript unreadable')
}
const unreadableSend = { method: 'agentSession.send' as const, operationId: operationId() }
record(
await assertHostAgreement(
unrecoverable,
{ method: 'agentSession.send', operationId: operationId() },
'agent_session_owner_restart_failed'
unreadable,
unreadableSend,
'agent_session_journal_unreadable',
async () => {
delete unreadable.host.deps.adapter.historyFilePath
return { harness: unreadable, spec: unreadableSend }
}
)
)
@@ -0,0 +1,63 @@
// The restart continuation is accepted like any send, so its verdict is its delivery: a cold start
// longer than the legacy client wait must not turn a continuation that went through into an
// "unconfirmed" one.
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types'
import { structuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-wiring'
import { StructuredAgentSessionSendSettlement } from './structured-agent-session-send-settlement'
const SESSION = 'session-1'
const MESSAGE = 'continuation-1'
let submission: AgentJournalSubmission
const journal = {
submissions: (): AgentJournalSubmission[] => [submission],
cursor: () => ({ epoch: 'epoch-1', sequence: 1 })
}
beforeEach(() => {
vi.useFakeTimers()
submission = {
clientMessageId: MESSAGE,
fence: 2,
payloadFingerprint: 'fingerprint',
dispatchState: 'pending',
providerItemId: null,
reason: null,
submittedAt: 1,
resolvedAt: null,
handoverRecorded: true
}
})
afterEach(() => {
vi.useRealTimers()
})
describe('the restart continuation waits for its delivery (W18)', () => {
it('reaches accepted after a 40 s cold start, with nothing filed as unconfirmed', async () => {
const settlement = new StructuredAgentSessionSendSettlement(() => journal)
const surfaces = structuredAgentSessionRestartResumeSurfaces(
{
revealSession: async () => ({ readable: true }),
hold: async () => undefined,
release: () => undefined,
send: async () => {
throw new Error('not used')
},
waitForSendSettlement: settlement.wait
},
() => 0
)
const verdict = surfaces.awaitSendSettlement(SESSION, MESSAGE)
await vi.advanceTimersByTimeAsync(40_000)
submission = { ...submission, dispatchState: 'accepted', providerItemId: 'item-1' }
settlement.publish(SESSION, journal)
await expect(verdict).resolves.toMatchObject({
value: { submission: { dispatchState: 'accepted' } }
})
})
})
@@ -6,7 +6,6 @@ import {
AGENT_SESSION_RESTART_NOT_CONNECTED_NOTE
} from '../../../shared/agent-session-restart-continuation'
import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key'
import { AgentSessionJournal } from '../agent-session-journal/journal-store'
import { latestStructuredAgentSessionUserItem } from '../../../shared/structured-agent-session-projection'
import { StructuredAgentSessionResumeAdmission } from './structured-agent-session-restart-resume-runner'
import {
@@ -39,17 +38,17 @@ it('files a continuation superseded by a replayed message, lists it and says so
await host.restartResume.list()
await host.hold(SESSION, 'pane')
const events = providerEvents(acquire)
const append = AgentSessionJournal.prototype.appendSubmission
vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission').mockImplementationOnce(
async function (this: AgentSessionJournal, input) {
const cursor = await append.call(this, input)
events.appendItem(
{ provider: 'codex', threadId: THREAD, turnId: 'replayed-turn', ordinal: 1 },
hostTestMessage('A queued notification the provider replayed')
)
return cursor
}
)
// The replay lands after the reattach and just before the continuation is accepted, which is
// where a send asks whether its offer still stands.
const send = host.send
vi.spyOn(host, 'send').mockImplementationOnce(async (caller, params) => {
events.appendItem(
{ provider: 'codex', threadId: THREAD, turnId: 'replayed-turn', ordinal: 1 },
hostTestMessage('A queued notification the provider replayed')
)
await host.flushStreamedEvents(SESSION)
return send(caller, params)
})
const result = await host.restartResume.continueAfterRestart([SESSION], 'modal')
@@ -10,7 +10,6 @@ import {
} from '../../runtime/agent-session-recovery-capsule'
import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store'
import { parseAgentSessionResumeMarker } from '../../../shared/agent-session-resume-marker'
import { AgentSessionJournal } from '../agent-session-journal/journal-store'
import { StructuredAgentSessionHost } from './structured-agent-session-host'
import { StructuredAgentSessionResumeAdmission } from './structured-agent-session-restart-resume-runner'
import {
@@ -52,6 +51,8 @@ export async function interruptedRestart(
previous.dispatch.mockResolvedValueOnce({ state: 'admitted' })
const body = hostTestMessage('Perform the original task')
await previous.host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body })
// Accepted first, handed over after: the work in flight is a send the provider took.
await vi.waitFor(() => expect(previous.dispatch).toHaveBeenCalledOnce())
} else if (work === 'children') {
events.appendItem(
{ provider: 'codex', threadId: THREAD, turnId: 'settled-turn', ordinal: 1 },
@@ -150,15 +151,17 @@ export async function supersededRefusal(userAnswers?: 'before' | 'after') {
if (!events) {
throw new Error('missing resumed provider event sink')
}
const append = AgentSessionJournal.prototype.appendSubmission
const writing = vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission')
writing.mockImplementationOnce(async function (this: AgentSessionJournal, input) {
const cursor = await append.call(this, input)
// The newer message lands after the reattach and just before the continuation is accepted,
// which is where a send asks whether its offer still stands.
const send = host.send
const writing = vi.spyOn(host, 'send')
writing.mockImplementationOnce(async (caller, params) => {
events.appendItem(
{ provider: 'codex', threadId: THREAD, turnId: 'newer-turn', ordinal: 1 },
hostTestMessage('A newer task from another client')
)
return cursor
await host.flushStreamedEvents(SESSION)
return send(caller, params)
})
const admit = StructuredAgentSessionResumeAdmission.prototype.run
const admitting = vi.spyOn(StructuredAgentSessionResumeAdmission.prototype, 'run')
@@ -178,9 +178,8 @@ it('refuses at send admission once the user has sent a newer message', async ()
{ outcome: 'refused', reason: 'agent_session_restart_work_superseded' }
])
expect(dispatch).not.toHaveBeenCalled()
expect(host.journalSnapshot(SESSION).submissions).toMatchObject([
{ dispatchState: 'rejected', reason: 'agent_session_restart_work_superseded' }
])
// Refused where it would have been accepted, so the chat records no continuation at all.
expect(host.journalSnapshot(SESSION).submissions).toEqual([])
host.release(SESSION, 'pane')
expect(host.isHeld(SESSION)).toBe(false)
})
@@ -249,6 +248,8 @@ it.each([false, true])(
hostTestMessage('A newer task from another client'),
{ lifecycle: true }
)
// Journaled before the continuation's acceptance asks whether its offer still stands.
await host.flushStreamedEvents(SESSION)
})
const result = await host.restartResume.continueAfterRestart([SESSION], 'modal')
@@ -257,8 +258,8 @@ it.each([false, true])(
{ outcome: 'refused', reason: 'agent_session_restart_work_superseded' }
])
expect(dispatch).not.toHaveBeenCalled()
expect(host.journalSnapshot(SESSION).submissions).toHaveLength(2)
expect(host.journalSnapshot(SESSION).submissions[1]?.dispatchState).toBe('rejected')
// Refused before acceptance: only the interrupted send is in the journal.
expect(host.journalSnapshot(SESSION).submissions).toHaveLength(1)
host.release(SESSION, 'pane')
expect(host.isHeld(SESSION)).toBe(false)
// The offer is spent, but the refusal is kept as a durable failure: a retry finds it, and the
@@ -560,7 +561,7 @@ it('fails closed on corrupt recovery storage while ordinary hold and send still
expect(
await host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body })
).toMatchObject({ ok: true })
expect(dispatch).toHaveBeenCalledTimes(1)
await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1))
// list; the action's read of offers and of failures; the post-action refresh of both.
expect(warning).toHaveBeenCalledTimes(5)
warning.mockRestore()
@@ -15,6 +15,12 @@ import {
restoreStructuredAgentSessionsOnRestart
} from './structured-agent-session-restart-restore'
const NO_OPEN_DEPS = {
store: { getRecord: () => null, listRecords: () => [] },
journalRoot: '/tmp/journals',
adapter: {}
}
describe('restart journal restoration', () => {
beforeEach(() => restoreRead.mockReset())
@@ -22,17 +28,20 @@ describe('restart journal restoration', () => {
const gate = Promise.withResolvers<void>()
let active = 0
let peak = 0
restoreRead.mockImplementation(async (_store, _root, sessionId: string) => {
restoreRead.mockImplementation(async (_deps, sessionId: string) => {
active += 1
peak = Math.max(peak, active)
await gate.promise
active -= 1
return {
journal: {},
params: { location: { workspaceId: 'workspace-1' }, provider: 'codex' },
fence: 1,
hasProviderChild: false,
sessionId
session: {
journal: {},
params: { location: { workspaceId: 'workspace-1' }, provider: 'codex' },
fence: 1,
hasProviderChild: false,
sessionId
},
reset: null
}
})
const records = Array.from(
@@ -41,8 +50,7 @@ describe('restart journal restoration', () => {
)
const restoration = restoreStructuredAgentSessionsOnRestart({
store: {} as never,
journalRoot: '/tmp/journals',
openDeps: NO_OPEN_DEPS,
records,
reconcile: async () => null,
resolveRecovery: async () => undefined,
@@ -82,17 +90,19 @@ describe('restart journal restoration', () => {
runtimeKind: 'native'
}
restoreRead.mockResolvedValue({
journal: {},
params,
fence: 4,
hasProviderChild: false,
acquisitionGeneration: null
session: {
journal: {},
params,
fence: 4,
hasProviderChild: false,
acquisitionGeneration: null
},
reset: null
})
await restoreOneStructuredAgentSessionRead(
{
store: {} as never,
journalRoot: '/tmp/journals',
openDeps: NO_OPEN_DEPS,
reconcile: async () => null,
resolveRecovery: async () => {
calls.push('resolveRecovery')
@@ -118,17 +128,19 @@ describe('restart journal restoration', () => {
it('does not rerun settlement retry when a second restore finds the session already open', async () => {
const retrySettlement = vi.fn(async () => true)
restoreRead.mockResolvedValue({
journal: {},
params: {},
fence: 4,
hasProviderChild: false,
acquisitionGeneration: null
session: {
journal: {},
params: {},
fence: 4,
hasProviderChild: false,
acquisitionGeneration: null
},
reset: null
})
await restoreOneStructuredAgentSessionRead(
{
store: {} as never,
journalRoot: '/tmp/journals',
openDeps: NO_OPEN_DEPS,
reconcile: async () => null,
resolveRecovery: async () => undefined,
serialize: async (_sessionId, task) => task(),
@@ -13,27 +13,30 @@
import type { AgentSessionRecord } from '../../../shared/agent-session-record'
import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire'
import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store'
import { mapWithConcurrency } from '../../../shared/map-with-concurrency'
import {
restoreStructuredAgentSessionRead,
type RestoredStructuredAgentSessionRead
} from './structured-agent-session-read-restore'
import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store'
import type {
OpenedStructuredAgentSessionConversation,
StructuredAgentSessionConversationOpenDeps
} from './structured-agent-session-conversation-open'
import type { AgentSessionAttachParams } from './structured-agent-session-attach'
import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore'
const JOURNAL_RESTORE_CONCURRENCY = 4
export type StructuredAgentSessionReadRestoreDeps = {
store: AgentSessionRecordStore
journalRoot: string
openDeps: StructuredAgentSessionConversationOpenDeps & {
store: Pick<AgentSessionRecordStore, 'getRecord' | 'listRecords'>
}
reconcile: (sessionId: string) => Promise<AgentSessionWireRefusal | null>
resolveRecovery: (sessionId: string) => Promise<unknown>
serialize: <T>(sessionId: string, task: () => Promise<T>) => Promise<T>
hasSession: (sessionId: string) => boolean
onReadable: (sessionId: string, restored: RestoredStructuredAgentSessionRead) => void
retrySettlement: (
onReadable: (
sessionId: string,
params: RestoredStructuredAgentSessionRead['params']
) => Promise<boolean>
opened: OpenedStructuredAgentSessionConversation
) => Promise<void> | void
retrySettlement: (sessionId: string, params: AgentSessionAttachParams) => Promise<boolean>
}
/**
@@ -62,7 +65,7 @@ export async function restoreOneStructuredAgentSessionRead(
export async function restoreOneStructuredAgentSessionReadUnderSerialize(
input: Pick<
StructuredAgentSessionReadRestoreDeps,
'store' | 'journalRoot' | 'hasSession' | 'onReadable' | 'retrySettlement'
'openDeps' | 'hasSession' | 'onReadable' | 'retrySettlement'
>,
sessionId: string
): Promise<void> {
@@ -70,16 +73,12 @@ export async function restoreOneStructuredAgentSessionReadUnderSerialize(
// A surface that took a hold mid-restore already attached this one.
return
}
const restored = await restoreStructuredAgentSessionRead(
input.store,
input.journalRoot,
sessionId
)
if (!restored) {
const opened = await restoreStructuredAgentSessionRead(input.openDeps, sessionId)
if (!opened) {
return
}
input.onReadable(sessionId, restored)
await input.retrySettlement(sessionId, restored.params)
await input.onReadable(sessionId, opened)
await input.retrySettlement(sessionId, opened.session.params)
}
export async function restoreStructuredAgentSessionsOnRestart(
@@ -11,7 +11,9 @@ import type {
AgentSessionMutationResult,
AgentSessionSendResult
} from '../../../shared/agent-session-wire'
import { MAX_TIMER_DELAY_MS } from '../../../shared/timer-delay'
import type { StructuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-host'
import type { SendSettlementWaitOptions } from './structured-agent-session-send-settlement'
/** The caller key the continuation sends under, so its writes are attributable to Orca itself. */
export const STRUCTURED_AGENT_SESSION_RESTART_CONTINUATION_CALLER =
@@ -34,7 +36,8 @@ type RestartResumeHostBindings = {
/** The host's existing settlement waiter; a send returns while its dispatch is still pending. */
waitForSendSettlement: (
sessionId: string,
clientMessageId: string
clientMessageId: string,
options: SendSettlementWaitOptions
) => Promise<{ value: AgentSessionSendResult } | undefined>
}
@@ -48,7 +51,10 @@ export function structuredAgentSessionRestartResumeSurfaces(
release: host.release,
send: (params) =>
host.send({ callerKey: STRUCTURED_AGENT_SESSION_RESTART_CONTINUATION_CALLER }, params),
awaitSendSettlement: host.waitForSendSettlement,
// Accepted like any send, so its verdict is its delivery, however long the start takes; the
// wait ends when the submission settles or the session closes.
awaitSendSettlement: (sessionId, clientMessageId) =>
host.waitForSendSettlement(sessionId, clientMessageId, { budgetMs: MAX_TIMER_DELAY_MS }),
onNoteFailed: () =>
console.warn('[structured-agent-session] restart continuation attribution failed'),
now
@@ -10,7 +10,7 @@
import type { AgentSessionRecord } from '../../../shared/agent-session-record'
import { randomUUID } from 'node:crypto'
import type { AgentSessionAttachParams } from './structured-agent-session-attach'
import { attachParamsForRecord } from './structured-agent-session-read-restore'
import { attachParamsForRecord } from './structured-agent-session-conversation-open'
export function isResumableStructuredAgentSessionRecord(record: AgentSessionRecord): boolean {
return (
@@ -46,23 +46,32 @@ function harness(
return task()
}
const restorer = new StructuredAgentSessionReadableRestorer({
store: {
getRecord: (sessionId: string) => records.find((r) => r.sessionId === sessionId) ?? null,
listRecords: () => records
} as never,
journalRoot: '/journals',
openDeps: {
store: {
getRecord: (sessionId: string) => records.find((r) => r.sessionId === sessionId) ?? null,
listRecords: () => records
},
journalRoot: '/journals',
adapter: {}
},
supportsRecord: options.supports ?? (() => true),
reconcile: async () => null,
resolveRecovery: async () => undefined,
serialize,
hasSession: (sessionId) => live.has(sessionId),
onReadable: (sessionId, restored) => live.set(sessionId, restored),
onReadable: (sessionId, restored) => {
live.set(sessionId, restored)
},
retrySettlement: async () => true
})
return { restorer, live, serializedIds }
}
const readable = { journal: {}, params: {}, fence: 1 } as never
const readable = {
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the restorer only indexes the session it is handed; no member of it is read here.
session: { journal: {}, params: {}, fence: 1 } as never,
reset: null
}
afterEach(() => {
vi.restoreAllMocks()
@@ -61,17 +61,14 @@ export function createStructuredAgentSessionHostRestore(
now: () => number,
wiring: Omit<
ConstructorParameters<typeof StructuredAgentSessionReadableRestorer>[0],
'store' | 'journalRoot' | 'supportsRecord' | 'retrySettlement'
'openDeps' | 'supportsRecord' | 'retrySettlement'
>
): {
restoreReadableSessions: (sessionIds?: readonly string[]) => Promise<void>
revealSession: (sessionId: string) => Promise<StructuredAgentSessionReveal>
/** One session, for a caller already inside its serialize. */
restoreReadableUnderSerialize: (sessionId: string) => Promise<boolean>
} {
const restorer = new StructuredAgentSessionReadableRestorer({
store: deps.store,
journalRoot: deps.journalRoot,
openDeps: deps,
supportsRecord: (record) => adapterSupportsRecord(deps.adapter, record),
retrySettlement: (sessionId, params) =>
retryPendingStructuredAgentSessionSettlement({ deps, sessions, sessionId, params, now }),
@@ -83,7 +80,6 @@ export function createStructuredAgentSessionHostRestore(
revealSession: (sessionId) =>
revealStructuredAgentSession(deps, sessionId, wiring.hasSession, (id) =>
restorer.restoreOne(id)
),
restoreReadableUnderSerialize: (sessionId) => restorer.restoreOneUnderSerialize(sessionId)
)
}
}
@@ -139,6 +139,14 @@ async function seed(acceptedSubmissions = false) {
}
})
).toMatchObject({ ok: true })
// Accepted into the conversation first; the delivery loop hands it over after.
await vi.waitFor(() =>
expect(
host
.journalSnapshot(HOST_TEST_SESSION)
.submissions.find((entry) => entry.clientMessageId === clientOperationId)?.dispatchState
).toBe('accepted')
)
if (i === 1) {
selectedItemId = agentJournalSubmissionKey(clientOperationId)
}
@@ -120,11 +120,16 @@ describe('structured send idempotency', () => {
await performSend(context, input)
const replay = await performSend(context, input)
// Acceptance records the one submission; the reused id answers with it and writes nothing.
// Handing it over is the delivery loop's, never a second accept's.
expect(replay).toMatchObject({
ok: true,
value: { clientMessageId: 'shared-send-id', submission: { dispatchState: 'accepted' } }
value: {
clientMessageId: 'shared-send-id',
submission: { dispatchState: 'pending', handoverRecorded: true }
}
})
expect(dispatch).toHaveBeenCalledOnce()
expect(dispatch).not.toHaveBeenCalled()
expect(journal.submissions()).toHaveLength(1)
})
})
@@ -1,11 +1,11 @@
// A send, or a hold, that finds the session's provider child gone, against the real host.
// A send, or a hold, that finds the session's provider child gone, against the real host. The
// send is accepted at once; its delivery restarts the child, or rejects it with the reason.
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest'
import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope'
import { agentSessionRefusalOperationState } from '../../../shared/agent-session-refusal-retry'
import type {
AgentSessionMutationEnvelope,
AgentSessionSubscribeEvent
@@ -24,6 +24,11 @@ import {
} from './structured-agent-session-host-test-data'
const CALLER = { callerKey: 'client-1' }
/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */
function eventually(assertion: () => void): Promise<void> {
return vi.waitFor(assertion, { timeout: 10_000 })
}
// Long enough that no release fires mid-test; whether one is pending is asserted directly.
const GRACE_MS = 60_000
@@ -105,15 +110,39 @@ function sendParams(text: string, operationId = hostTestOperationId()) {
return { envelope, body }
}
/** Every status row the chat shows, oldest first; none when the session is not even readable. */
function journalStatuses(): string[] {
if (!host.hasSession(SESSION)) {
return []
}
const history = host.history({ sessionId: SESSION, direction: 'tail' })
return history.ok
? history.page.items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : []))
: []
/** Accepted at once, before any owner exists for it; answers the message id. */
async function accept(params: ReturnType<typeof sendParams>): Promise<string> {
const result = await host.send(CALLER, params)
expect(result, JSON.stringify(result)).toMatchObject({
ok: true,
replayed: false,
value: { submission: { dispatchState: 'pending', handoverRecorded: true } }
})
return params.envelope.clientOperationId
}
function submission(clientMessageId: string) {
return host
.journalSnapshot(SESSION)
.submissions.find((entry) => entry.clientMessageId === clientMessageId)
}
/** The submission once delivery is done with it: handed over, or rejected unwritten. */
async function settled(clientMessageId: string) {
await eventually(() => {
const current = submission(clientMessageId)
expect(current?.dispatchState !== 'pending' || current.handedOverAt !== undefined).toBe(true)
})
return submission(clientMessageId)
}
/** The failure rows a start the chat needed left, oldest first. */
function errorStatuses(): string[] {
return host
.journalSnapshot(SESSION)
.items.flatMap((item) =>
item.body.kind === 'status' && item.body.tone === 'error' ? [item.body.text] : []
)
}
/** The child timed out or exited: its lease is handed back and the host holds no session. */
@@ -130,15 +159,14 @@ describe('a send with no live owner', () => {
it('restarts the owner once and delivers against it', async () => {
await loseOwner()
const result = await host.send(CALLER, sendParams('after the child died'))
await accept(sendParams('after the child died'))
expect(result).toMatchObject({ ok: true, replayed: false })
await eventually(() => expect(dispatch).toHaveBeenCalledOnce())
expect(acquire).toHaveBeenCalledOnce()
expect(dispatch).toHaveBeenCalledOnce()
expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live')
})
it('restarts the owner before the send is admitted, so the send is admitted once', async () => {
it('accepts the send before anything restarts, and the restart hands it over', async () => {
await loseOwner()
const order: string[] = []
const spawnChild = acquire.getMockImplementation()!
@@ -152,21 +180,18 @@ describe('a send with no live owner', () => {
return admit(args)
})
await expect(host.send(CALLER, sendParams('ensure first'))).resolves.toMatchObject({
ok: true,
replayed: false
})
await accept(sendParams('accept first'))
await eventually(() => expect(dispatch).toHaveBeenCalledOnce())
expect(order).toEqual(['acquire', 'admit'])
expect(order).toEqual(['admit', 'acquire'])
})
it('leaves a live owner alone', async () => {
acquire.mockClear()
await expect(host.send(CALLER, sendParams('owner is live'))).resolves.toMatchObject({
ok: true
})
await accept(sendParams('owner is live'))
await eventually(() => expect(dispatch).toHaveBeenCalledOnce())
expect(acquire).not.toHaveBeenCalled()
})
@@ -184,36 +209,35 @@ describe('a send with no live owner', () => {
}
}))
await host.send(CALLER, sendParams('into a cleared chat'))
await expect(host.send(CALLER, sendParams('into a cleared chat'))).resolves.toMatchObject({
ok: false,
refusal: { code: 'agent_session_operation_invalid' }
})
expect(acquire).not.toHaveBeenCalled()
})
it('renews the idle window on journal activity in an unheld session', async () => {
await loseOwner()
await expect(host.send(CALLER, sendParams('restart'))).resolves.toMatchObject({ ok: true })
await accept(sendParams('restart'))
await eventually(() => expect(dispatch).toHaveBeenCalledOnce())
const arm = vi.spyOn(host['holds']['clock'], 'arm')
await expect(host.send(CALLER, sendParams('more activity'))).resolves.toMatchObject({
ok: true
})
await accept(sendParams('more activity'))
expect(arm).toHaveBeenCalledWith(SESSION)
await eventually(() => expect(arm).toHaveBeenCalledWith(SESSION))
})
it('releases the restarted child on the usual clock only when no surface holds it', async () => {
await loseOwner()
await expect(host.send(CALLER, sendParams('nobody is watching'))).resolves.toMatchObject({
ok: true
})
expect(host['holds'].isReleasePending(SESSION)).toBe(true)
await accept(sendParams('nobody is watching'))
await eventually(() => expect(host['holds'].isReleasePending(SESSION)).toBe(true))
await host.close(SESSION)
// A reading surface that does not itself restart the agent.
await host.hold(SESSION, 'desktop-chat:1', { resume: false })
await expect(host.send(CALLER, sendParams('the chat is open'))).resolves.toMatchObject({
ok: true
})
await accept(sendParams('the chat is open'))
await eventually(() => expect(dispatch).toHaveBeenCalledTimes(2))
expect(host['holds'].isReleasePending(SESSION)).toBe(false)
})
@@ -231,15 +255,15 @@ describe('a send with no live owner', () => {
expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released')
acquire.mockClear()
await expect(host.send(CALLER, sendParams('after an exit'))).resolves.toMatchObject({
ok: true
})
await accept(sendParams('after an exit'))
await eventually(() => expect(dispatch).toHaveBeenCalledOnce())
expect(acquire).toHaveBeenCalledOnce()
})
it('restarts nothing for a resend the journal already answers', async () => {
const params = sendParams('sent once')
await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: false })
await accept(params)
await eventually(() => expect(dispatch).toHaveBeenCalledOnce())
// The child died during startup: the lease is handed back, the fence moves, and the session
// stays readable. The client resends against the new fence.
await host.handleAdapterEvent({
@@ -268,12 +292,9 @@ describe('a send with no live owner', () => {
expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released')
// Retry rotates the id: a genuinely new send restarts the owner once.
await expect(host.send(CALLER, sendParams('sent once'))).resolves.toMatchObject({
ok: true,
replayed: false
})
await accept(sendParams('sent once'))
await eventually(() => expect(dispatch).toHaveBeenCalledTimes(2))
expect(acquire).toHaveBeenCalledOnce()
expect(dispatch).toHaveBeenCalledTimes(2)
})
it('restarts nothing for a send the ledger holds but the journal never saw', async () => {
@@ -320,7 +341,7 @@ describe('a send with no live owner', () => {
expect(dispatch).not.toHaveBeenCalled()
})
it('admits a send that arrives after the restart has already claimed the lease', async () => {
it('accepts a send that arrives while a restart holds the queue, and hands both over in order', async () => {
await loseOwner()
const lostFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0
let claimed = () => {}
@@ -334,18 +355,22 @@ describe('a send with no live owner', () => {
return spawnChild!(input)
})
const first = host.send(CALLER, sendParams('first'))
const first = await accept(sendParams('first'))
await claim
expect(store.getRecord(SESSION)?.lease.runtimeFence).toBe(lostFence + 1)
// Written against the lost owner's fence, which admits it: a send is a conversation write.
const late = sendParams('second')
late.envelope.expectedRuntimeFence = lostFence
const second = host.send(CALLER, late)
release()
expect(await first).toMatchObject({ ok: true })
expect(await second).toMatchObject({ ok: true })
await eventually(() => expect(dispatch).toHaveBeenCalledTimes(2))
expect(acquire).toHaveBeenCalledOnce()
expect(dispatch).toHaveBeenCalledTimes(2)
expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([
first,
late.envelope.clientOperationId
])
})
it('shares one restart between concurrent sends', async () => {
@@ -358,8 +383,8 @@ describe('a send with no live owner', () => {
])
expect(results.map((result) => result.ok)).toEqual([true, true, true])
await eventually(() => expect(dispatch).toHaveBeenCalledTimes(3))
expect(acquire).toHaveBeenCalledOnce()
expect(dispatch).toHaveBeenCalledTimes(3)
})
it('shares one restart between a hold and a send that arrive in the same gap', async () => {
@@ -372,21 +397,22 @@ describe('a send with no live owner', () => {
expect(held).toMatchObject({ status: 'fulfilled' })
expect(sent).toMatchObject({ status: 'fulfilled', value: { ok: true } })
await eventually(() => expect(dispatch).toHaveBeenCalledOnce())
expect(acquire).toHaveBeenCalledOnce()
expect(dispatch).toHaveBeenCalledOnce()
expect(host['holds'].isHeld(SESSION)).toBe(true)
expect(host['holds'].isReleasePending(SESSION)).toBe(false)
})
it('replays into a closed session without spawning anything', async () => {
const params = sendParams('sent once')
await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: false })
await accept(params)
await eventually(() => expect(dispatch).toHaveBeenCalledOnce())
await loseOwner()
await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: true })
await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: true })
// The journal was made readable for the answer; the record's lease was left as it was.
// The conversation was opened for the answer; the record's lease was left as it was.
expect(host.hasSession(SESSION)).toBe(true)
expect(acquire).not.toHaveBeenCalled()
expect(dispatch).toHaveBeenCalledOnce()
@@ -394,104 +420,83 @@ describe('a send with no live owner', () => {
expect(host['holds'].isReleasePending(SESSION)).toBe(false)
})
it("refuses with the restart's own cause, in the answer and in the chat", async () => {
it("rejects the accepted message with the restart's own cause, and says so in the chat once", async () => {
await loseOwner()
acquire.mockRejectedValue(new Error('Not signed in. Run codex login'))
const params = sendParams('while signed out')
const result = await host.send(CALLER, params)
expect(result).toEqual({
ok: false,
refusal: {
code: 'agent_session_owner_restart_failed',
message: "Codex couldn't restart: Not signed in. Run codex login.",
// The failed attach proved its child gone: nothing runs for this session.
ownerVerdict: 'exited'
}
})
expect(dispatch).not.toHaveBeenCalled()
expect(hostErrors).not.toEqual([])
expect(agentSessionRefusalOperationState('agent_session_owner_restart_failed')).toBe(
'settled-rejected'
)
// Refused before admission: the ledger holds nothing a resend would replay.
expect(store.getOperationRow(CALLER.callerKey, params.envelope.clientOperationId)).toBeNull()
// The same status row a failed start leaves, so the reason outlives the error strip.
expect(journalStatuses()).toEqual([
const cause =
'The provider stopped before it finished starting: Not signed in. Run codex login.'
])
const id = await accept(params)
expect(await settled(id)).toMatchObject({ dispatchState: 'rejected', reason: cause })
expect(dispatch).not.toHaveBeenCalled()
// Accepted, so the ledger answers a resend with the rejection rather than a second attempt.
expect(
store.getOperationRow(CALLER.callerKey, params.envelope.clientOperationId)
).toMatchObject({ outcome: { status: 'succeeded' } })
// One row, in the error tone, so the reason outlives the error strip.
expect(errorStatuses()).toEqual([cause])
})
it('restarts again for a Retry of the refused send, under its own id or a new one', async () => {
it('restarts again for a Retry under a new id, and replays a resend of the same id', async () => {
await loseOwner()
acquire.mockRejectedValue(new Error('Not signed in'))
const params = sendParams('while signed out')
await expect(host.send(CALLER, params)).resolves.toMatchObject({
ok: false,
refusal: { code: 'agent_session_owner_restart_failed' }
})
await settled(await accept(params))
expect(acquire).toHaveBeenCalledTimes(1)
// A client that resends the same id gets another attempt, and the chat no second row.
// A client that resends the same id gets the recorded rejection, and the chat no second row.
await expect(host.send(CALLER, params)).resolves.toMatchObject({
ok: false,
refusal: { code: 'agent_session_owner_restart_failed' }
ok: true,
replayed: true,
value: { submission: { dispatchState: 'rejected' } }
})
expect(acquire).toHaveBeenCalledTimes(1)
expect(errorStatuses()).toHaveLength(1)
// The outbox's Retry rotates the id: a fresh attempt, with its own row.
expect(await settled(await accept(sendParams('while signed out')))).toMatchObject({
dispatchState: 'rejected'
})
expect(acquire).toHaveBeenCalledTimes(2)
expect(journalStatuses()).toHaveLength(1)
// The outbox's Retry rotates the id: also a fresh attempt.
await expect(host.send(CALLER, sendParams('while signed out'))).resolves.toMatchObject({
ok: false,
refusal: { code: 'agent_session_owner_restart_failed' }
})
expect(acquire).toHaveBeenCalledTimes(3)
expect(errorStatuses()).toHaveLength(2)
expect(dispatch).not.toHaveBeenCalled()
})
it('restarts and delivers a later send once the cause clears', async () => {
await loseOwner()
acquire.mockRejectedValueOnce(new Error('Not signed in'))
await expect(host.send(CALLER, sendParams('while signed out'))).resolves.toMatchObject({
ok: false,
refusal: { code: 'agent_session_owner_restart_failed' }
expect(await settled(await accept(sendParams('while signed out')))).toMatchObject({
dispatchState: 'rejected'
})
// The user signed in; nothing about the failed attempt is remembered.
await expect(host.send(CALLER, sendParams('signed in now'))).resolves.toMatchObject({
ok: true,
replayed: false
})
await accept(sendParams('signed in now'))
await eventually(() => expect(dispatch).toHaveBeenCalledOnce())
expect(acquire).toHaveBeenCalledTimes(2)
expect(dispatch).toHaveBeenCalledOnce()
expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live')
})
it('suggests a new chat only when this host has nothing to restart the chat from', async () => {
await loseOwner()
acquire.mockRejectedValue(new Error('Not signed in'))
const failed = await host.send(CALLER, sendParams('restart fails'))
expect(failed).toMatchObject({
ok: false,
refusal: { code: 'agent_session_owner_restart_failed' }
})
expect(failed.ok ? '' : failed.refusal.message).not.toMatch(/new chat/)
const failed = await settled(await accept(sendParams('restart fails')))
expect(failed).toMatchObject({ dispatchState: 'rejected' })
expect(failed?.reason).not.toMatch(/new chat/)
// The adapter cannot run this record where it lives: no retry would bring it back.
host.deps.adapter.supportsLocation = () => false
const unresumable = await host.send(CALLER, sendParams('cannot resume here'))
const unresumable = await settled(await accept(sendParams('cannot resume here')))
expect(unresumable).toMatchObject({
ok: false,
refusal: {
code: 'agent_session_owner_restart_failed',
message:
"Codex couldn't restart: This execution host cannot resume the requested structured agent session. Start a new chat to continue."
}
dispatchState: 'rejected',
reason:
"Codex couldn't restart: This execution host cannot resume the requested structured agent session. Start a new chat to continue."
})
})
it('runs the send as the lease stands when the restart met a lease someone else is settling', async () => {
it('rejects the message with the cause when the restart met a lease someone else is settling', async () => {
await loseOwner()
vi.spyOn(host['holds'], 'ensureProviderChild').mockResolvedValueOnce({
ok: false,
@@ -501,33 +506,30 @@ describe('a send with no live owner', () => {
}
})
const result = await host.send(CALLER, sendParams('owner being settled'))
const id = await accept(sendParams('owner being settled'))
// The ordinary lease check answers, retryably; nothing terminal and nothing in the chat.
expect(result).toMatchObject({
ok: false,
refusal: { code: 'agent_session_ownership_unknown' }
})
const cause = "Codex couldn't restart: Another runtime is still adjudicating this lease."
expect(await settled(id)).toMatchObject({ dispatchState: 'rejected', reason: cause })
expect(acquire).not.toHaveBeenCalled()
expect(journalStatuses()).toEqual([])
expect(errorStatuses()).toEqual([cause])
})
it('runs the send as the lease stands when the restart itself faults', async () => {
it('rejects the message, and reports the fault, when the restart itself faults', async () => {
await loseOwner()
vi.spyOn(host['holds'], 'ensureProviderChild').mockRejectedValueOnce(
new Error('spawn-token mint failed')
)
const result = await host.send(CALLER, sendParams('bookkeeping failed'))
const id = await accept(sendParams('bookkeeping failed'))
expect(result).toMatchObject({
ok: false,
refusal: { code: 'agent_session_ownership_unknown' }
expect(await settled(id)).toMatchObject({
dispatchState: 'rejected',
reason: "Codex couldn't restart: spawn-token mint failed."
})
expect(hostErrors).toContainEqual(
expect.objectContaining({ message: 'spawn-token mint failed' })
)
expect(journalStatuses()).toEqual([])
expect(errorStatuses()).toHaveLength(1)
})
it('keeps a second surface holder taken during an auto-restart, and starts nothing for it', async () => {
@@ -663,20 +665,22 @@ describe('a send with no live owner', () => {
})
})
it('leaves a lease it cannot adjudicate alone', async () => {
it('adjudicates a lease this host has not reconciled before its delivery resumes it', async () => {
await loseOwner()
await store.transitionHandoff(SESSION, (current) => ({
...current,
lease: { ...current.lease, unreconciled: true }
}))
const result = await host.send(CALLER, sendParams('owner unverifiable'))
// The send's start is the same serialized resume a hold runs, reconciliation first.
await accept(sendParams('owner unverified'))
expect(result).toMatchObject({
ok: false,
refusal: { code: 'agent_session_ownership_unknown' }
await eventually(() => expect(dispatch).toHaveBeenCalledOnce())
expect(acquire).toHaveBeenCalledOnce()
expect(store.getRecord(SESSION)?.lease).toMatchObject({
unreconciled: false,
claimStatus: 'live'
})
expect(acquire).not.toHaveBeenCalled()
})
})
@@ -691,18 +695,15 @@ describe('a write fenced to an owner the pane has not seen replaced', () => {
const params = sendParams('after the release')
params.envelope.expectedRuntimeFence = seenFence
expect(await host.send(CALLER, params)).toMatchObject({
ok: true,
replayed: false,
value: { submission: { dispatchState: 'accepted' } }
})
const id = await accept(params)
expect(await settled(id)).toMatchObject({ dispatchState: 'accepted' })
expect(acquire).toHaveBeenCalledOnce()
expect(dispatch).toHaveBeenCalledOnce()
const fence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0
expect(fence).toBeGreaterThan(seenFence + 1)
// The pane learns the fence the host is at, not the one it subscribed under.
expect(frames.at(-1)).toMatchObject({ fence })
await eventually(() => expect(frames.at(-1)).toMatchObject({ fence }))
})
it('admits a Stop and a send queued behind the cold start that replaced their owner', async () => {
@@ -718,7 +719,8 @@ describe('a write fenced to an owner the pane has not seen replaced', () => {
return spawnChild(input)
})
const first = host.send(CALLER, sendParams('starts the agent'))
const firstParams = sendParams('starts the agent')
const first = host.send(CALLER, firstParams)
await claim
const cancelFields = { turnId: 'turn-1' }
const stop = host.cancel(CALLER, {
@@ -742,7 +744,14 @@ describe('a write fenced to an owner the pane has not seen replaced', () => {
expect(await first).toMatchObject({ ok: true })
expect(await stop).toMatchObject({ ok: true, replayed: false })
expect(await second).toMatchObject({ ok: true, replayed: false })
// The Stop withdrew the message its start held; the one typed after it is delivered.
expect(await settled(late.envelope.clientOperationId)).toMatchObject({
dispatchState: 'accepted'
})
expect(submission(firstParams.envelope.clientOperationId)).toMatchObject({
dispatchState: 'rejected'
})
expect(acquire).toHaveBeenCalledOnce()
expect(dispatch).toHaveBeenCalledTimes(2)
expect(dispatch).toHaveBeenCalledTimes(1)
})
})
@@ -1,23 +1,8 @@
// What a send needs from the session before its lease is checked.
//
// A provider child that exits or fails to start hands its lease back. Before this, a send to that
// session was refused `agent_session_ownership_unknown` — which a client reads as "not admitted
// yet" and resends forever — and only a surface hold could ever make a new child. Now the send
// makes sure it has an owner as a step of its own serialized admission: a released lease where
// resume is allowed gets a child first; anything else runs as it is and meets the lease check.
// A restart that fails refuses with a code the client stops auto-retrying on, carrying the
// restart's own cause, and writes that cause into the chat the way a start that failed does, so
// the user sees why. A manual Retry or a new send is a fresh attempt: a refusal before admission
// leaves no ledger row behind.
//
// The ledger's answer comes first, so a send it already holds a row for restarts nothing:
// admission replays or refuses it whoever owns the session now, and a closed session is made
// readable for that, never given a child. Otherwise each resend of a message whose child died at
// startup would spawn another child that dies the same way.
//
// A child that has not proven its start is still the owner: the send is admitted against it and
// the adapter holds the message until startup lands, or rejects it with the child's own reason
// when the child dies first. The exit settlement writes that reason into the chat.
// What a send or a Stop needs from the session before the ledger places its row: the
// conversation open. Nothing here needs an owner — a send is accepted into the conversation and
// the delivery loop makes the session ready — so a refusal before acceptance is only one the
// conversation itself makes: a rewind or conversation command in doubt, a cleared conversation,
// or a journal that cannot be opened.
import type { AgentSessionRecord } from '../../../shared/agent-session-record'
import type {
@@ -25,45 +10,39 @@ import type {
AgentSessionWireRefusal
} from '../../../shared/agent-session-wire'
import type { AgentSessionWireRefusalCode } from '../../../shared/agent-session-wire-refusals'
import { boundJournalStatusText } from '../agent-session-journal/journal-prompt-body-bounds'
import { TUI_AGENT_DISPLAY_NAMES } from '../../../shared/tui-agent-display-names'
import {
ownerRestartFailedOutcome,
providerStartupFailureOutcome
} from './structured-agent-session-dead-generation-settlement'
import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types'
import type { StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations'
import type { AgentSessionMutationSessionPreparation } from './structured-agent-session-mutation-admission'
import { isResumableStructuredAgentSessionRecord } from './structured-agent-session-resume-eligibility'
import {
AGENT_SESSION_NOT_ATTACHED,
type AgentSessionMutationSessionPreparation
} from './structured-agent-session-mutation-admission'
import { rewindRefusal } from './structured-rewind-refusal'
/**
* What a refused resume means for the send that ran it. `transient`: the resume met a lease
* someone else is settling, which is not proof it cannot resume — the send runs as the lease
* stands and admission reports it. `failed`: the restart itself failed; the send answers with the
* cause and stops the client's retry loop, and the user may clear the cause and retry.
* `unresumable`: this host has nothing to restart the chat from — no record, or none it can run —
* so only a new chat continues. A new wire code does not compile until it is classified here.
* Whether a refused start leaves the chat anything to start again from. `unresumable`: this host
* has nothing to restart it from — no record, or none it can run — so only a new chat continues.
* A new wire code does not compile until it is classified here.
*/
const RESUME_REFUSAL_OUTCOME: Record<
AgentSessionWireRefusalCode,
'transient' | 'failed' | 'unresumable'
> = {
execution_owner_reconciling: 'transient',
agent_session_conflict: 'transient',
agent_session_checkpoint_stale: 'transient',
agent_session_ownership_unknown: 'transient',
agent_session_operation_capacity: 'transient',
structured_agent_session_unsupported: 'unresumable',
agent_session_operation_conflict: 'failed',
agent_session_operation_expired: 'failed',
agent_session_operation_invalid: 'failed',
agent_session_operation_unknown: 'failed',
agent_session_item_revision_stale: 'failed',
agent_session_already_resolved: 'failed',
agent_session_identity_required: 'unresumable',
agent_session_journal_unreadable: 'failed',
agent_session_owner_restart_failed: 'failed'
const START_REFUSAL_RESUMABLE: Record<AgentSessionWireRefusalCode, boolean> = {
execution_owner_reconciling: true,
agent_session_conflict: true,
agent_session_checkpoint_stale: true,
agent_session_ownership_unknown: true,
agent_session_operation_capacity: true,
structured_agent_session_unsupported: false,
agent_session_operation_conflict: true,
agent_session_operation_expired: true,
agent_session_operation_invalid: true,
agent_session_operation_unknown: true,
agent_session_item_revision_stale: true,
agent_session_already_resolved: true,
agent_session_identity_required: false,
agent_session_journal_unreadable: true,
agent_session_owner_restart_failed: true
}
/** Why the record refuses any send right now, whoever owns it; null when a send may run. */
@@ -93,133 +72,41 @@ export function structuredAgentSessionSendBlock(
return null
}
/** Whether this send is the one that must bring the owner back: no child, a lease handed back
* cleanly, and nothing on the record that refuses the send anyway. Live, unverifiable, still
* reserved, or handed off: that lease is not this send's to replace. */
export function structuredAgentSessionSendNeedsOwner(
session: StructuredAgentSessionHostSession | undefined,
record: AgentSessionRecord
): boolean {
return (
session?.hasProviderChild !== true &&
isResumableStructuredAgentSessionRecord(record) &&
structuredAgentSessionSendBlock(record) === null
)
}
type SendPreparationContext = Pick<
StructuredAgentSessionMutationContext,
'deps' | 'sessions' | 'holds' | 'restoreReadable'
>
export async function prepareStructuredAgentSessionSend(
context: SendPreparationContext,
envelope: AgentSessionMutationEnvelope,
ledger: 'admit' | 'replay',
record: AgentSessionRecord
/** The conversation a send or a Stop writes to, opened when this host holds it closed. */
export async function openConversationForWrite(
openConversation: (sessionId: string) => Promise<StructuredAgentSessionHostSession | null>,
envelope: AgentSessionMutationEnvelope
): Promise<AgentSessionMutationSessionPreparation> {
const { sessionId } = record
if (ledger !== 'admit') {
if (!context.sessions.has(sessionId)) {
await context.restoreReadable(sessionId)
try {
if (await openConversation(envelope.sessionId)) {
return { ok: true }
}
return { ok: true }
}
if (structuredAgentSessionSendNeedsOwner(context.sessions.get(sessionId), record)) {
const refusal = await restartOwnerForSend(context, envelope, record)
if (refusal) {
return { ok: false, refusal }
return { ok: false, refusal: AGENT_SESSION_NOT_ATTACHED }
} catch (error) {
return {
ok: false,
refusal: {
code: 'agent_session_journal_unreadable',
message: `The conversation could not be opened: ${
error instanceof Error ? error.message : String(error)
}`
}
}
}
return { ok: true }
}
/** One restart attempt. Answers with the refusal that ends the send, or null when the send goes
* on to admission — after a child, after a transient refusal, or after a fault in the restart's
* own bookkeeping, which is reported and never gates the user's action. */
async function restartOwnerForSend(
context: SendPreparationContext,
envelope: AgentSessionMutationEnvelope,
record: AgentSessionRecord
): Promise<AgentSessionWireRefusal | null> {
const { sessionId } = envelope
let resumed: Awaited<ReturnType<typeof context.holds.ensureProviderChild>>
try {
resumed = await context.holds.ensureProviderChild(sessionId)
} catch (error) {
context.deps.onEventSinkError?.({ sessionId, error })
return null
/** What the chat says, in its row and on every message it rejects, when the delivery loop could
* not make the session ready. A child that died starting reads as any start that died does. */
export function structuredAgentSessionStartFailureText(
record: AgentSessionRecord | null,
cause: AgentSessionWireRefusal
): string {
if (cause.ownerVerdict === 'exited') {
return providerStartupFailureOutcome(cause.message)
}
const outcome = resumed.ok ? null : RESUME_REFUSAL_OUTCOME[resumed.refusal.code]
if (resumed.ok || outcome === 'transient') {
return null
}
const refusal = ownerRestartFailedRefusal(record, resumed.refusal, outcome !== 'unresumable')
context.deps.onEventSinkError?.({
sessionId,
error: new Error(`${resumed.refusal.code}: ${resumed.refusal.message}`)
return ownerRestartFailedOutcome({
agentName: record ? TUI_AGENT_DISPLAY_NAMES[record.provider] : 'The agent',
reason: cause.message,
resumable: START_REFUSAL_RESUMABLE[cause.code]
})
// A restart whose child died starting leaves the row any start that died leaves, so the chat
// reads the same whether the send met that death before admission or after it.
await recordFailedRestart(
context,
envelope,
resumed.refusal.ownerVerdict === 'exited'
? providerStartupFailureOutcome(resumed.refusal.message)
: refusal.message
)
return refusal
}
/** The client stops on the code; the message carries the restart's own cause, and the verdict —
* when the failed attach proved its child gone — tells a client nothing runs for the session. */
function ownerRestartFailedRefusal(
record: AgentSessionRecord,
cause: AgentSessionWireRefusal,
resumable: boolean
): AgentSessionWireRefusal {
return {
code: 'agent_session_owner_restart_failed',
message: ownerRestartFailedOutcome({
agentName: TUI_AGENT_DISPLAY_NAMES[record.provider],
reason: cause.message,
resumable
}),
...(cause.ownerVerdict ? { ownerVerdict: cause.ownerVerdict } : {})
}
}
/** The same status row a start that failed leaves in the chat, so the reason outlives the error
* strip. The journal is made readable for it when the failed attach left none behind. Keyed by
* the send, not the clock: a resend of the same id that fails again adds no second row. */
async function recordFailedRestart(
context: SendPreparationContext,
envelope: AgentSessionMutationEnvelope,
text: string
): Promise<void> {
const { sessionId } = envelope
try {
if (!context.sessions.has(sessionId)) {
await context.restoreReadable(sessionId)
}
const session = context.sessions.get(sessionId)
if (!session) {
return
}
const settlementId = `failed-restart:${envelope.clientOperationId}`
await session.journal.appendLifecycleBatch({
settlementId,
fence: session.fence,
recovered: true,
mutations: [
{
kind: 'item',
identity: { provider: 'orca', clientMessageId: settlementId },
body: { kind: 'status', text: boundJournalStatusText(text) }
}
]
})
} catch (error) {
context.deps.onEventSinkError?.({ sessionId, error })
}
}
@@ -1,12 +1,11 @@
// A session that published and then lost its child before startup (not signed in, say) keeps a
// released lease and a chat the user can still type into. The send is the user asking for the
// child back: the host restarts it before admitting the write and delivers against the new owner,
// instead of parking the message behind a lease nothing would ever re-acquire.
// child back: the host accepts the message, and its delivery restarts the child and hands the
// message to the new owner, instead of parking it behind a lease nothing would ever re-acquire.
//
// A child is published before it has proven its start, and it owns the send from that moment: the
// message is admitted against it and the adapter holds it for the start. When the child exits
// first, the exit settlement rejects the message and writes the cause into the chat, once, and
// the next send is a fresh restart.
// message is handed to it. When the child exits first, the exit settlement rejects the message and
// writes the cause into the chat, once, and the next send is a fresh restart.
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
@@ -28,6 +27,11 @@ import {
} from './structured-agent-session-host-test-data'
const CALLER = { callerKey: 'client-1' }
/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */
function eventually(assertion: () => void): Promise<void> {
return vi.waitFor(assertion, { timeout: 10_000 })
}
const EXIT_REASON = 'Claude Code is not signed in. Sign in with the Claude CLI'
let root: string
@@ -53,7 +57,7 @@ function sendEnvelope(
}
}
/** A send is admitted against the child it meets, proven or not; the adapter holds the rest. */
/** A send is accepted at once and handed to the child delivery finds or starts. */
async function send(
text: string,
fence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0
@@ -63,9 +67,11 @@ async function send(
expect(sent, JSON.stringify(sent)).toMatchObject({
ok: true,
replayed: false,
value: { submission: { dispatchState: 'pending' } }
value: { submission: { dispatchState: 'pending', handoverRecorded: true } }
})
return sent.ok ? sent.value.clientMessageId : ''
const clientMessageId = sent.ok ? sent.value.clientMessageId : ''
await eventually(() => expect(submission(clientMessageId)?.handedOverAt).toBeDefined())
return clientMessageId
}
/** The child of the current acquisition, as the adapter would identify it in a lifecycle event. */
@@ -166,8 +172,7 @@ describe('a send into a published session whose child ended before startup', ()
await send('hello again', releasedFence)
// A send still fenced to the lost owner is admitted once against the restarted one, with no
// stale round trip.
// Accepted at the lost owner's fence and handed to the child delivery started, once.
expect(acquire).toHaveBeenCalledTimes(2)
expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live')
expect(dispatch).toHaveBeenCalledOnce()
@@ -207,7 +212,7 @@ describe('a send into a published session whose child ended before startup', ()
// One spawn per user action: nothing restarted it a second time.
expect(acquire).toHaveBeenCalledTimes(2)
// Retry is a fresh action: it restarts once and is admitted against the new child.
// Retry is a fresh action: it restarts once and is handed to the new child.
await send('signed in now')
expect(acquire).toHaveBeenCalledTimes(3)
expect(dispatch).toHaveBeenCalledTimes(2)
@@ -216,7 +221,7 @@ describe('a send into a published session whose child ended before startup', ()
})
describe('a send while the child of the first start is still proving itself', () => {
it('is admitted against the starting child, and nothing restarts it', async () => {
it('is handed to the starting child, and nothing restarts it', async () => {
await send('hello')
expect(dispatch).toHaveBeenCalledOnce()
@@ -61,7 +61,7 @@ describe('structured send settlement compatibility wait', () => {
it('removes an abandoned wait on transport cancellation', async () => {
const settlements = new StructuredAgentSessionSendSettlement(() => journal('pending'))
const controller = new AbortController()
const pending = settlements.wait('session-1', 'client-1', controller.signal)
const pending = settlements.wait('session-1', 'client-1', { signal: controller.signal })
controller.abort(new Error('transport closed'))
await expect(pending).rejects.toThrow('transport closed')
@@ -3,8 +3,12 @@ import type {
AgentJournalSubmission
} from '../../../shared/agent-session-journal-types'
import type { AgentSessionSendResult } from '../../../shared/agent-session-wire'
import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission'
import type { AgentSessionJournal } from '../agent-session-journal/journal-store'
/** What a settlement read needs of a journal. */
type SendSettlementJournal = Pick<AgentSessionJournal, 'submissions' | 'cursor'>
type SettledSend = {
cursor: AgentJournalCursor
value: AgentSessionSendResult
@@ -12,8 +16,19 @@ type SettledSend = {
type SendSettlement = SettledSend | 'pending' | 'missing'
/** What ends a wait: the provider's answer, or only the host handing the message over. */
export type SendSettlementPoint = 'answered' | 'handed-over'
export type SendSettlementWaitOptions = {
signal?: AbortSignal
/** How long to observe; unanswered by then resolves undefined. */
budgetMs?: number
until?: SendSettlementPoint
}
type SendSettlementWaiter = {
clientMessageId: string
until: SendSettlementPoint
resolve: (result: SettledSend | undefined) => void
reject: (error: Error) => void
timer: ReturnType<typeof setTimeout>
@@ -23,12 +38,15 @@ type SendSettlementWaiter = {
// Known legacy clients abandon the RPC after 15s without cancelling its socket dispatch.
const SEND_SETTLEMENT_WAIT_TIMEOUT_MS = 30_000
/** Long enough for a cold provider start; a longer one replays through the same wait. */
export const STRUCTURED_AGENT_SESSION_START_WAIT_MS = 120_000
const MAX_SEND_SETTLEMENT_WAITERS_PER_SESSION = 64
const MAX_SEND_SETTLEMENT_WAITERS = 1_024
function settledSend(
journal: AgentSessionJournal,
journal: SendSettlementJournal,
clientMessageId: string,
until: SendSettlementPoint,
submission: AgentJournalSubmission | undefined = journal
.submissions()
.find((candidate) => candidate.clientMessageId === clientMessageId)
@@ -36,9 +54,11 @@ function settledSend(
if (!submission) {
return 'missing'
}
return submission.dispatchState === 'pending'
? 'pending'
: { cursor: journal.cursor(), value: { clientMessageId, submission } }
const waiting =
until === 'handed-over'
? isQueuedAgentJournalSubmission(submission)
: submission.dispatchState === 'pending'
return waiting ? 'pending' : { cursor: journal.cursor(), value: { clientMessageId, submission } }
}
function abortError(signal: AbortSignal): Error {
@@ -52,17 +72,19 @@ export class StructuredAgentSessionSendSettlement {
private readonly waiters = new Map<string, Set<SendSettlementWaiter>>()
private waiterCount = 0
constructor(private readonly journalFor: (sessionId: string) => AgentSessionJournal) {}
constructor(private readonly journalFor: (sessionId: string) => SendSettlementJournal) {}
wait = (
sessionId: string,
clientMessageId: string,
signal?: AbortSignal
options: SendSettlementWaitOptions = {}
): Promise<SettledSend | undefined> => {
const { signal } = options
const until = options.until ?? 'answered'
if (signal?.aborted) {
return Promise.reject(abortError(signal))
}
const immediate = settledSend(this.journalFor(sessionId), clientMessageId)
const immediate = settledSend(this.journalFor(sessionId), clientMessageId, until)
if (immediate === 'missing') {
return Promise.reject(new Error('agent session send disappeared before settlement'))
}
@@ -79,12 +101,13 @@ export class StructuredAgentSessionSendSettlement {
return new Promise((resolve, reject) => {
const waiter: SendSettlementWaiter = {
clientMessageId,
until,
resolve,
reject,
timer: setTimeout(() => {
this.remove(sessionId, waiter)
resolve(undefined)
}, SEND_SETTLEMENT_WAIT_TIMEOUT_MS)
}, options.budgetMs ?? SEND_SETTLEMENT_WAIT_TIMEOUT_MS)
}
waiter.timer.unref?.()
const session = existingSession ?? new Set<SendSettlementWaiter>()
@@ -106,7 +129,7 @@ export class StructuredAgentSessionSendSettlement {
})
}
publish(sessionId: string, journal: AgentSessionJournal): void {
publish(sessionId: string, journal: SendSettlementJournal): void {
const waiters = this.waiters.get(sessionId)
if (!waiters) {
return
@@ -118,6 +141,7 @@ export class StructuredAgentSessionSendSettlement {
const result = settledSend(
journal,
waiter.clientMessageId,
waiter.until,
submissions.get(waiter.clientMessageId)
)
if (result !== 'pending') {
@@ -32,6 +32,28 @@ beforeEach(() => {
;({ store, host, dispatch } = hostTestState())
})
function hostJournal(): AgentSessionJournal {
return (
host as unknown as { sessions: Map<string, { journal: AgentSessionJournal }> }
).sessions.get(SESSION)!.journal
}
/** A send is accepted, then the session's delivery loop hands it over: wait for the handover's
* outcome, or with `handedOver`, only for the handover itself (an admitted send stays pending). */
async function delivered(clientMessageId: string, options: { handedOver?: true } = {}) {
let submission: ReturnType<AgentSessionJournal['submissions']>[number] | undefined
await vi.waitFor(() => {
submission = hostJournal()
.submissions()
.find((entry) => entry.clientMessageId === clientMessageId)
expect(submission?.handedOverAt).toBeDefined()
if (!options.handedOver) {
expect(submission?.dispatchState).not.toBe('pending')
}
})
return submission!
}
describe('send', () => {
it('writes the submission before dispatching and resolves it accepted', async () => {
await attach()
@@ -43,7 +65,15 @@ describe('send', () => {
if (!result.ok) {
throw new Error(`expected a send, got ${result.refusal.code}`)
}
expect(result.value.submission.dispatchState).toBe('accepted')
// Answered once accepted; the delivery loop hands it over after.
expect(result.value.submission).toMatchObject({
dispatchState: 'pending',
handoverRecorded: true
})
expect(result.value.submission.handedOverAt).toBeUndefined()
await expect(delivered(result.value.clientMessageId)).resolves.toMatchObject({
dispatchState: 'accepted'
})
expect(dispatch).toHaveBeenCalledTimes(1)
const page = host.history({ sessionId: SESSION, direction: 'tail' })
expect(page.ok && page.page.items).toHaveLength(1)
@@ -75,11 +105,11 @@ describe('send', () => {
await attach()
dispatch.mockRejectedValueOnce(new Error('socket closed'))
const body = hostTestMessage('add a retry')
const result = await host.send(CALLER, {
envelope: envelope('agentSession.send', { body }),
body
const params = { envelope: envelope('agentSession.send', { body }), body }
await host.send(CALLER, params)
await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({
dispatchState: 'unknown'
})
expect(result).toMatchObject({ ok: true, value: { submission: { dispatchState: 'unknown' } } })
})
it('replays a retried send from the journal without dispatching twice', async () => {
@@ -87,6 +117,7 @@ describe('send', () => {
const body = hostTestMessage('add a retry')
const params = { envelope: envelope('agentSession.send', { body }), body }
await host.send(CALLER, params)
await delivered(params.envelope.clientOperationId)
const retry = await host.send(CALLER, params)
expect(retry).toMatchObject({ ok: true, replayed: true })
expect(dispatch).toHaveBeenCalledTimes(1)
@@ -98,10 +129,9 @@ describe('send', () => {
const body = hostTestMessage('possibly delivered')
const params = { envelope: envelope('agentSession.send', { body }), body }
const first = await host.send(CALLER, params)
expect(first).toMatchObject({
ok: true,
value: { submission: { dispatchState: 'unknown' } }
await host.send(CALLER, params)
await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({
dispatchState: 'unknown'
})
// A thrown adapter call is indistinguishable from a lost reply, so Retry
// replays the recorded outcome.
@@ -129,6 +159,7 @@ describe('send', () => {
const params = { envelope: envelope('agentSession.send', { body }), body }
await host.send(CALLER, params)
await delivered(params.envelope.clientOperationId)
await expect(host.send(CALLER, { ...params, retryUnknown: true })).resolves.toMatchObject({
ok: true,
value: {
@@ -151,21 +182,18 @@ describe('send', () => {
const body = hostTestMessage('never written')
const params = { envelope: envelope('agentSession.send', { body }), body }
await expect(host.send(CALLER, params)).resolves.toMatchObject({
ok: true,
value: {
submission: { dispatchState: 'rejected', reason: 'provider_write_failed: broken pipe' }
}
await host.send(CALLER, params)
await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({
dispatchState: 'rejected',
reason: 'provider_write_failed: broken pipe'
})
// What the user's Retry does with a rejection: a fresh client message id,
// which is a first delivery by construction and cannot duplicate the frame
// that never left the process.
await expect(
host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body })
).resolves.toMatchObject({
ok: true,
replayed: false,
value: { submission: { dispatchState: 'accepted' } }
const rotated = { envelope: envelope('agentSession.send', { body }), body }
await expect(host.send(CALLER, rotated)).resolves.toMatchObject({ ok: true, replayed: false })
await expect(delivered(rotated.envelope.clientOperationId)).resolves.toMatchObject({
dispatchState: 'accepted'
})
expect(dispatch).toHaveBeenCalledTimes(2)
const state = host.history({ sessionId: SESSION, direction: 'tail' })
@@ -183,10 +211,9 @@ describe('send', () => {
const body = hostTestMessage('a message the provider may already hold')
const params = { envelope: envelope('agentSession.send', { body }), body }
const first = await host.send(CALLER, params)
expect(first).toMatchObject({
ok: true,
value: { submission: { dispatchState: 'unknown' } }
await host.send(CALLER, params)
await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({
dispatchState: 'unknown'
})
// No `unknown` is re-delivered under its own id, whatever its reason says,
// so Retry replays the recorded outcome instead of writing again.
@@ -203,6 +230,7 @@ describe('send', () => {
const body = hostTestMessage('settled for good')
const params = { envelope: envelope('agentSession.send', { body }), body }
await host.send(CALLER, params)
await delivered(params.envelope.clientOperationId)
const journal = (
host as unknown as { sessions: Map<string, { journal: AgentSessionJournal }> }
).sessions.get(SESSION)!.journal
@@ -223,7 +251,7 @@ describe('send', () => {
expect(journal.receiptFor(params.envelope.clientOperationId)).not.toBeNull()
})
it('leaves an admitted send pending and writes no dispatch row', async () => {
it('leaves an admitted send pending once handed over', async () => {
await attach()
dispatch.mockImplementationOnce(async () => ({ state: 'admitted' as const }))
const body = hostTestMessage('queued behind a running turn')
@@ -233,9 +261,9 @@ describe('send', () => {
ok: true,
value: { submission: { dispatchState: 'pending', reason: null, resolvedAt: null } }
})
const journal = (
host as unknown as { sessions: Map<string, { journal: AgentSessionJournal }> }
).sessions.get(SESSION)!.journal
await delivered(params.envelope.clientOperationId, { handedOver: true })
await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1))
const journal = hostJournal()
expect(journal.pendingSubmissions()).toHaveLength(1)
})
@@ -245,6 +273,8 @@ describe('send', () => {
const body = hostTestMessage('written, never acknowledged')
const params = { envelope: envelope('agentSession.send', { body }), body }
await host.send(CALLER, params)
await delivered(params.envelope.clientOperationId, { handedOver: true })
await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1))
const journal = (
host as unknown as { sessions: Map<string, { journal: AgentSessionJournal }> }
).sessions.get(SESSION)!.journal
@@ -281,6 +311,8 @@ describe('send', () => {
const params = { envelope: envelope('agentSession.send', { body }), body }
await expect(host.send(CALLER, params)).rejects.toThrow('operation settlement failed')
// The submission was recorded before the ledger write failed, so it is still delivered.
await delivered(params.envelope.clientOperationId)
await expect(host.send(CALLER, params)).resolves.toMatchObject({
ok: true,
replayed: true,
@@ -333,9 +365,8 @@ describe('send', () => {
await expect(host.send(CALLER, params)).rejects.toThrow('operation settlement failed')
settlement.mockRestore()
const journal = (
host as unknown as { sessions: Map<string, { journal: AgentSessionJournal }> }
).sessions.get(SESSION)!.journal
await delivered(params.envelope.clientOperationId)
const journal = hostJournal()
await journal.rollEpoch('schema_unreadable', store.getRecord(SESSION)?.lease.runtimeFence ?? 1)
expect(journal.submissions()).toHaveLength(0)
@@ -362,6 +393,7 @@ describe('send', () => {
const params = { envelope: envelope('agentSession.send', { body }), body }
await host.send(CALLER, params)
await delivered(params.envelope.clientOperationId)
expect(dispatch).toHaveBeenCalledTimes(1)
await store.recordOperationOutcome({
callerKey: CALLER.callerKey,
@@ -393,6 +425,8 @@ describe('send', () => {
const body = hostTestMessage('written, then the child died')
const params = { envelope: envelope('agentSession.send', { body }), body }
await host.send(CALLER, params)
await delivered(params.envelope.clientOperationId, { handedOver: true })
await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1))
const journal = (
host as unknown as { sessions: Map<string, { journal: AgentSessionJournal }> }
).sessions.get(SESSION)!.journal
@@ -411,21 +445,25 @@ describe('send', () => {
it('advances an explicit retry after a ledger-unknown send is reconciled in the journal', async () => {
await attach()
const journal = (
host as unknown as { sessions: Map<string, { journal: AgentSessionJournal }> }
).sessions.get(SESSION)!.journal
vi.spyOn(journal, 'resolveDispatch').mockRejectedValueOnce(new Error('journal resolve failed'))
const journal = hostJournal()
const resolve = journal.resolveDispatch.bind(journal)
// The handover row lands; the provider's answer, written after the adapter took the
// message, does not.
vi.spyOn(journal, 'resolveDispatch')
.mockImplementationOnce(resolve)
.mockRejectedValueOnce(new Error('journal resolve failed'))
const body = hostTestMessage('possibly delivered before persistence failed')
const params = { envelope: envelope('agentSession.send', { body }), body }
await expect(host.send(CALLER, params)).rejects.toThrow('journal resolve failed')
expect(journal.submissions()).toMatchObject([
{ clientMessageId: params.envelope.clientOperationId, dispatchState: 'unknown' }
])
await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true })
await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({
dispatchState: 'unknown'
})
// Acceptance is what the ledger answers for; delivery is the journal's to say.
expect(
store.listOperationRows().find((row) => row.operationId === params.envelope.clientOperationId)
?.outcome
).toEqual({ status: 'unknown' })
).toMatchObject({ status: 'succeeded' })
expect(dispatch).toHaveBeenCalledTimes(1)
await journal.markPendingSubmissionsUnknown(store.getRecord(SESSION)?.lease.runtimeFence ?? 1)
@@ -446,7 +484,7 @@ describe('send', () => {
expect(journal.submissions()).toHaveLength(1)
})
it('admits a send fenced to another generation, and replays it by id once the fence catches up', async () => {
it('admits a send fenced to another generation, delivers it once, and replays it by id', async () => {
const record = await attach()
const body = hostTestMessage('add a retry')
const params = {
@@ -457,10 +495,9 @@ describe('send', () => {
),
body
}
expect(await host.send(CALLER, params)).toMatchObject({
ok: true,
replayed: false,
value: { submission: { dispatchState: 'accepted' } }
expect(await host.send(CALLER, params)).toMatchObject({ ok: true, replayed: false })
await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({
dispatchState: 'accepted'
})
const retry = {
...params,
@@ -322,6 +322,8 @@ describe('settled attach retry', () => {
}
const first = await host.send(CALLER, unknownParams)
expect(first).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } })
// Handed over before the host dies: that is what makes the restart's answer doubt.
await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1))
await host.flushAllStreamedEvents()
store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' })
@@ -351,7 +353,7 @@ describe('settled attach retry', () => {
if (!sent.ok) {
throw new Error(`unexpected restored send refusal: ${sent.refusal.message}`)
}
expect(dispatch).toHaveBeenCalledTimes(2)
await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(2))
const restoredHistory = host.history({ sessionId: SESSION, direction: 'tail' })
if (!restoredHistory.ok) {
throw new Error(`unexpected restored history reset: ${restoredHistory.reset}`)
@@ -0,0 +1,48 @@
import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission'
import { boundJournalStatusText } from '../agent-session-journal/journal-prompt-body-bounds'
import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types'
/**
* A start the delivery loop needed and did not get: one error row in the chat, so the reason
* outlives any error strip, and every queued message rejected with the same words. Keyed by the
* oldest message it failed, so a replayed failure adds no second row. Writes nothing when nothing
* is still queued: a start whose messages Stop withdrew did not fail anyone.
*/
export async function recordStructuredAgentSessionStartFailure(
session: Pick<StructuredAgentSessionHostSession, 'journal' | 'fence'>,
text: string
): Promise<void> {
const oldest = oldestQueuedSubmission(session)
if (!oldest) {
return
}
const settlementId = `start-failure:${oldest.clientMessageId}`
await session.journal.appendLifecycleBatch({
settlementId,
fence: session.fence,
recovered: true,
mutations: [
{
kind: 'item',
identity: { provider: 'orca', clientMessageId: settlementId },
body: { kind: 'status', text: boundJournalStatusText(text), tone: 'error' }
}
]
})
await session.journal.rejectQueuedSubmissions(session.fence, text)
}
export function oldestQueuedSubmission(
session: Pick<StructuredAgentSessionHostSession, 'journal'>
): ReturnType<StructuredAgentSessionHostSession['journal']['submissions']>[number] | undefined {
let oldest: ReturnType<typeof oldestQueuedSubmission>
for (const submission of session.journal.submissions()) {
if (
isQueuedAgentJournalSubmission(submission) &&
(oldest === undefined || (submission.acceptedSequence ?? 0) < (oldest.acceptedSequence ?? 0))
) {
oldest = submission
}
}
return oldest
}
@@ -1,7 +1,8 @@
// A Claude chat is published before its CLI answers initialize, and a message sent in that window
// is held until it does. Switching away from the chat starts the release clock; the clock must
// treat that held message as work still owed, exactly as it treats a running turn, or it evicts
// the session and refuses a message the user already sent.
// is accepted and stays queued until it does; the delivery loop hands it over once startup lands.
// Switching away from the chat starts the release clock; the clock must treat that queued message
// as work still owed, exactly as it treats a running turn, or it evicts the session and rejects a
// message the user already sent.
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
@@ -62,7 +63,7 @@ beforeEach(async () => {
lifecycle.push(host.handleAdapterEvent(mapped))
}
},
// As the runtime wires it: a held prompt's outcome reaches the journal out of band.
// As the runtime wires it: an admitted prompt's outcome reaches the journal out of band.
onDispatchSettledLate: (settlement) => void host.settleLateDispatch(settlement),
// Initialize answers only when the test says so.
openConnection: async (launch, handlers) => {
@@ -136,13 +137,21 @@ function dispatchState(clientMessageId: string): string | undefined {
.submissions.find((entry) => entry.clientMessageId === clientMessageId)?.dispatchState
}
/** The delivery loop hands a message over on its own serialized steps after startup lands; this
* yields to them without advancing the (possibly faked) release clock. */
async function untilSent(connection: { sent: unknown[] }): Promise<void> {
for (let turn = 0; turn < 2000 && connection.sent.length === 0; turn += 1) {
await new Promise((resolve) => setImmediate(resolve))
}
}
/** Long enough for several grace windows to elapse, so "not evicted" means the clock declined. */
function waitOutSeveralGraceWindows(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, GRACE_MS * 20))
}
describe('a chat left while its Claude CLI is still starting', () => {
it('keeps the session for a message it is holding, and delivers it once startup lands', async () => {
it('keeps the session for a message still queued, and delivers it once startup lands', async () => {
await attachStarting()
const held = await send('sent while starting')
@@ -154,9 +163,12 @@ describe('a chat left while its Claude CLI is still starting', () => {
expect(dispatchState(held)).toBe('pending')
landInit()
await adapter.drainStartup(SESSION)
await adapter.awaitStarted(SESSION)
expect(claude.connections[0].sent).toEqual([expect.objectContaining({ type: 'user' })])
await vi.waitFor(
() => expect(claude.connections[0].sent).toEqual([expect.objectContaining({ type: 'user' })]),
{ timeout: 3000 }
)
await vi.waitFor(() => expect(dispatchState(held)).toBe('accepted'))
})
@@ -175,8 +187,9 @@ describe('a chat left while its Claude CLI is still starting', () => {
// Startup lands just before the clock's next tick.
landInit()
await adapter.drainStartup(SESSION)
await adapter.awaitStarted(SESSION)
await Promise.all(lifecycle)
await untilSent(connection)
expect(connection.sent).toEqual([expect.objectContaining({ type: 'user' })])
await vi.advanceTimersByTimeAsync(GRACE_MS - 1)
@@ -192,8 +205,9 @@ describe('a chat left while its Claude CLI is still starting', () => {
it('is released after the grace once its turn has finished', async () => {
await attachStarting()
landInit()
await adapter.drainStartup(SESSION)
await send('answered', 'accepted')
await adapter.awaitStarted(SESSION)
const answered = await send('answered')
await vi.waitFor(() => expect(dispatchState(answered)).toBe('accepted'))
claude.connections[0].handlers.onMessage?.({
type: 'result',
subtype: 'success',
@@ -27,7 +27,8 @@ function startedSession(): StructuredAgentSessionUnexpectedExitSession & {
snapshot: () => ({ items: [] }),
appendLifecycleBatch: vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })),
markPendingSubmissionsUnknown: vi.fn(async () => []),
rejectPendingSubmissions: vi.fn(async () => [])
rejectPendingSubmissions: vi.fn(async () => []),
rejectQueuedSubmissions: vi.fn(async () => [])
}
}
}
@@ -261,6 +261,8 @@ describe('a chat that closes', () => {
if (!result.ok) {
throw new Error('send was refused')
}
// Handed over first: a message still queued at close is rejected as never sent instead.
await vi.waitFor(() => expect(dispatch).toHaveBeenCalled())
const settlement = host.waitForSendSettlement(SESSION, result.value.clientMessageId)
await host.close(SESSION)
@@ -459,6 +461,12 @@ describe('a session evicted and opened again', () => {
})
})
function submissionState(clientMessageId: string): string | undefined {
return host
.journalSnapshot(SESSION)
.submissions.find((entry) => entry.clientMessageId === clientMessageId)?.dispatchState
}
describe('an unexpected provider exit', () => {
it('publishes terminal settlement to a waiting older client', async () => {
await attach()
@@ -475,6 +483,8 @@ describe('an unexpected provider exit', () => {
if (!result.ok) {
throw new Error('send was refused')
}
// Accepted first; the exit must meet a message the provider was handed.
await vi.waitFor(() => expect(dispatch).toHaveBeenCalledOnce())
const settlement = host.waitForSendSettlement(SESSION, result.value.clientMessageId)
const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0
@@ -536,12 +546,14 @@ describe('an unexpected provider exit', () => {
await host.hold(SESSION, SURFACE)
dispatch.mockRejectedValueOnce(new Error('provider delivery became unknown'))
const unknownBody = hostTestMessage('message with unknown delivery')
const unknownEnvelope = envelope('agentSession.send', { body: unknownBody })
await expect(
host.send(CALLER, {
envelope: envelope('agentSession.send', { body: unknownBody }),
body: unknownBody
})
).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'unknown' } } })
host.send(CALLER, { envelope: unknownEnvelope, body: unknownBody })
).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } })
// Accepted, then handed over by the delivery loop, where the thrown dispatch becomes doubt.
await vi.waitFor(() =>
expect(submissionState(unknownEnvelope.clientOperationId)).toBe('unknown')
)
const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0
await host.handleAdapterEvent({
@@ -570,9 +582,12 @@ describe('an unexpected provider exit', () => {
providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-next', ordinal: 1 }
})
const body = hostTestMessage('a distinct next message')
await expect(
host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body })
).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } })
const nextEnvelope = envelope('agentSession.send', { body })
await expect(host.send(CALLER, { envelope: nextEnvelope, body })).resolves.toMatchObject({
ok: true,
value: { submission: { dispatchState: 'pending' } }
})
await vi.waitFor(() => expect(submissionState(nextEnvelope.clientOperationId)).toBe('accepted'))
expect(dispatch).toHaveBeenCalledTimes(2)
})
@@ -637,8 +652,11 @@ describe('an unexpected provider exit', () => {
}
await expect(host.send(CALLER, unknownParams)).resolves.toMatchObject({
ok: true,
value: { submission: { dispatchState: 'unknown' } }
value: { submission: { dispatchState: 'pending' } }
})
await vi.waitFor(() =>
expect(submissionState(unknownParams.envelope.clientOperationId)).toBe('unknown')
)
const runtimeState = (
host as unknown as {
runtimeState: { lifecycleBarrier: () => Promise<{ ok: false; error: Error }> }
@@ -682,9 +700,12 @@ describe('an unexpected provider exit', () => {
providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-next', ordinal: 1 }
})
const body = hostTestMessage('a distinct next message after failed-barrier recovery')
await expect(
host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body })
).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } })
const nextEnvelope = envelope('agentSession.send', { body })
await expect(host.send(CALLER, { envelope: nextEnvelope, body })).resolves.toMatchObject({
ok: true,
value: { submission: { dispatchState: 'pending' } }
})
await vi.waitFor(() => expect(submissionState(nextEnvelope.clientOperationId)).toBe('accepted'))
expect(dispatch).toHaveBeenCalledTimes(2)
})
@@ -25,11 +25,7 @@ import type {
} from './structured-agent-session-adapter'
import { providerStartupFailureRejection } from './structured-agent-session-dead-generation-settlement'
import { validatePendingPrompt } from './structured-agent-session-prompt-state'
import { withTimeout } from '../../../shared/promise-timeout-fallback'
import {
AgentSessionPreDispatchError,
AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS
} from './structured-agent-session-operation-settlement'
import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key'
export { performSetOption } from './structured-agent-session-turns-options'
export { performPrompt } from './structured-agent-session-turns-prompt'
@@ -47,8 +43,6 @@ export type AgentSessionTurnContext = {
publish: () => void
/** Drains provider lifecycle already accepted by the execution host. */
flushStreamedEvents: () => Promise<void>
/** Re-derives authorization after submission persistence, immediately before provider dispatch. */
beforeDispatch?: () => void
/** What the host holds about the child this dispatch is for, read at the moment it is needed. */
providerChildPhase?: () => StructuredAgentSessionProviderChildPhase | undefined
now: () => number
@@ -67,10 +61,10 @@ function invalid(message: string): { ok: false; refusal: AgentSessionWireRefusal
* accepted nothing (input is written only after it initializes), so a dispatch it could not
* take is provably unwritten and is rejected with the cause the adapter gave. */
async function dispatchSafely(
ctx: AgentSessionTurnContext,
ctx: AgentSessionHandoverContext,
clientMessageId: string,
body: AgentJournalMessageItem,
requestedAt: number | undefined
requestedAt: number
): Promise<AgentSessionDispatchOutcome> {
try {
return await ctx.adapter.dispatch({
@@ -78,13 +72,9 @@ async function dispatchSafely(
clientMessageId,
body,
fence: ctx.fence,
...(ctx.beforeDispatch ? { beforeDispatch: async () => ctx.beforeDispatch?.() } : {}),
...(requestedAt === undefined ? {} : { requestedAt })
requestedAt
})
} catch (error) {
if (error instanceof AgentSessionPreDispatchError) {
throw error
}
if (ctx.providerChildPhase?.() === 'starting') {
return { state: 'rejected', reason: providerStartupFailureRejection(error) }
}
@@ -111,6 +101,8 @@ async function appendStatus(
* the whole content of the word — and one message reached the model five times
* when this was a judgement call instead of an invariant. A distinct send after
* a terminal rejection uses a fresh id, which is a first delivery.
*
* Accepting only records the message; the session's delivery loop hands it over.
*/
export async function performSend(
ctx: AgentSessionTurnContext,
@@ -133,79 +125,10 @@ export async function performSend(
}
}
try {
await ctx.journal.appendSubmission({ ...input, fence: ctx.fence })
await ctx.journal.appendSubmission({ ...input, fence: ctx.fence, handoverRecorded: true })
} catch {
return invalid('The message could not be recorded and was not sent.')
}
// The row just written is the send's instant on the host clock; the turn this
// dispatch opens records it so the live counter never re-anchors at turn-open.
const requestedAt = ctx.journal
.submissions()
.find((entry) => entry.clientMessageId === input.clientMessageId)?.submittedAt
const outcome = await dispatchSafely(ctx, input.clientMessageId, input.body, requestedAt).catch(
async (error: unknown) => {
if (error instanceof AgentSessionPreDispatchError) {
const recorded = await withTimeout(
ctx.journal
.resolveDispatch({
clientMessageId: input.clientMessageId,
state: 'rejected',
reason: error.message,
fence: ctx.fence
})
.then(() => true),
AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS,
false
)
if (!recorded) {
console.warn('[structured-agent-session] pre-dispatch refusal persistence failed')
}
}
throw error
}
)
// An admission needs no dispatch row: the submission is already pending.
if (outcome.state === 'admitted') {
return {
ok: true,
value: {
clientMessageId: input.clientMessageId,
submission: requireSubmission(ctx, input.clientMessageId)
}
}
}
try {
await ctx.journal.resolveDispatch(
outcome.state === 'accepted'
? {
clientMessageId: input.clientMessageId,
state: 'accepted',
providerIdentity: outcome.providerIdentity,
fence: ctx.fence
}
: {
clientMessageId: input.clientMessageId,
state: outcome.state,
reason: outcome.reason,
fence: ctx.fence
}
)
} catch (error) {
// A failed resolution must not strand a pending row; an unknown result is
// explicitly replayable.
try {
await ctx.journal.resolveDispatch({
clientMessageId: input.clientMessageId,
state: 'unknown',
reason: DISPATCH_DOUBT_PERSISTENCE_FAILED,
fence: ctx.fence
})
} catch {
// Nothing further to record; the pending row is settled on the next attach.
}
throw error
}
return {
ok: true,
value: {
@@ -215,6 +138,66 @@ export async function performSend(
}
}
export type AgentSessionHandoverContext = Pick<
AgentSessionTurnContext,
'sessionId' | 'journal' | 'fence' | 'adapter' | 'providerChildPhase'
>
/**
* Hands one queued submission to the provider. The `dispatch{pending}` row goes first: a crash
* after it leaves a message in doubt, never one that reads as queued and so provably unwritten.
*/
export async function handOverSubmission(
ctx: AgentSessionHandoverContext,
submission: AgentJournalSubmission
): Promise<void> {
const { clientMessageId } = submission
const body = ctx.journal.itemBody(agentJournalSubmissionKey(clientMessageId))
if (body?.kind !== 'message') {
await ctx.journal.resolveDispatch({
clientMessageId,
state: 'rejected',
reason: 'The message could not be read back and was not sent.',
fence: ctx.fence
})
return
}
await ctx.journal.resolveDispatch({ clientMessageId, state: 'pending', fence: ctx.fence })
// The row written at acceptance is the send's instant on the host clock; the turn this
// dispatch opens records it so the live counter never re-anchors at turn-open.
const outcome = await dispatchSafely(ctx, clientMessageId, body, submission.submittedAt)
// An admission needs no dispatch row: the submission is already pending.
if (outcome.state === 'admitted') {
return
}
try {
await ctx.journal.resolveDispatch(
outcome.state === 'accepted'
? {
clientMessageId,
state: 'accepted',
providerIdentity: outcome.providerIdentity,
fence: ctx.fence
}
: { clientMessageId, state: outcome.state, reason: outcome.reason, fence: ctx.fence }
)
} catch (error) {
// A failed resolution must not strand a pending row; an unknown result is
// explicitly replayable.
try {
await ctx.journal.resolveDispatch({
clientMessageId,
state: 'unknown',
reason: DISPATCH_DOUBT_PERSISTENCE_FAILED,
fence: ctx.fence
})
} catch {
// Nothing further to record; the pending row is settled on the next open.
}
throw error
}
}
function requireSubmission(
ctx: AgentSessionTurnContext,
clientMessageId: string
@@ -136,7 +136,8 @@ describe('provider-exit recovery tickets', () => {
items: [lifecycleItem('turn-1', 1, { state: 'running', startedAt: 1_000 })]
}),
appendLifecycleBatch,
markPendingSubmissionsUnknown: vi.fn(async () => [])
markPendingSubmissionsUnknown: vi.fn(async () => []),
rejectQueuedSubmissions: vi.fn(async () => [])
}
} as unknown as StructuredAgentSessionHostSession
@@ -204,7 +205,8 @@ describe('provider-exit recovery tickets', () => {
journal: {
snapshot: () => ({ items }),
appendLifecycleBatch,
markPendingSubmissionsUnknown: vi.fn(async () => [])
markPendingSubmissionsUnknown: vi.fn(async () => []),
rejectQueuedSubmissions: vi.fn(async () => [])
}
} as unknown as StructuredAgentSessionHostSession
const store = {
@@ -312,7 +314,8 @@ describe('provider-exit recovery tickets', () => {
snapshot: () => ({ items }),
appendLifecycleBatch,
markPendingSubmissionsUnknown: vi.fn(async () => []),
rejectPendingSubmissions: vi.fn(async () => [])
rejectPendingSubmissions: vi.fn(async () => []),
rejectQueuedSubmissions: vi.fn(async () => [])
}
}
@@ -370,6 +373,7 @@ describe('provider-exit recovery tickets', () => {
appendLifecycleBatch: vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })),
markPendingSubmissionsUnknown,
rejectPendingSubmissions: vi.fn(async () => []),
rejectQueuedSubmissions: vi.fn(async () => []),
submissions: () => [{ clientMessageId: 'client-1', dispatchState: 'pending' }]
}
}
@@ -416,6 +420,7 @@ describe('provider-exit recovery tickets', () => {
journal: {
markPendingSubmissionsUnknown: vi.fn(async () => []),
rejectPendingSubmissions: vi.fn(async () => []),
rejectQueuedSubmissions: vi.fn(async () => []),
snapshot: () => ({
items: [lifecycleItem('turn-failing', 1, { state: 'running', startedAt: 1 })]
}),
@@ -17,6 +17,7 @@ import {
agentSessionProviderHandleRoot
} from '../../../shared/agent-session-provider-handle'
import { latestStructuredAgentSessionUserItem } from '../../../shared/structured-agent-session-projection'
import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission'
import type { AgentSessionRecord } from '../../../shared/agent-session-record'
import type {
AgentSessionResumeMarker,
@@ -54,6 +55,8 @@ function pendingSubmissionInFlight(
if (
submission &&
submission.recovered !== true &&
// A queued message reached no agent, so there is no work of its to resume.
!isQueuedAgentJournalSubmission(submission) &&
(submission.dispatchState === 'pending' || submission.dispatchState === 'unknown')
) {
return submission
@@ -9,12 +9,21 @@ export async function recoverInterruptedCompaction(
fence: number
): Promise<void> {
const command = store.getRecord(sessionId)?.conversationCommand
if (
command?.command !== 'compact' ||
command.phase !== 'prepared' ||
command.runtimeFence === undefined ||
command.runtimeFence === fence
) {
if (command?.runtimeFence === undefined || command.runtimeFence === fence) {
return
}
await settleInterruptedCompaction(store, sessionId, journal, fence)
}
/** A compaction still prepared whose child can no longer finish it: its outcome is unknown. */
export async function settleInterruptedCompaction(
store: AgentSessionRecordStore,
sessionId: string,
journal: AgentSessionJournal,
fence: number
): Promise<void> {
const command = store.getRecord(sessionId)?.conversationCommand
if (command?.command !== 'compact' || command.phase !== 'prepared') {
return
}
const error = 'Previous compaction completion could not be confirmed after session recovery.'
@@ -48,9 +48,12 @@ export function conversationCommandBlocked(
: 'Wait for background tasks to finish before using this command.'
}
if (
ctx.journal
.submissions()
.some((entry) => entry.dispatchState === 'pending' || entry.dispatchState === 'unknown')
ctx.journal.submissions().some(
(entry) =>
entry.dispatchState === 'pending' ||
// Doubt left by an earlier child is not this one's work in flight.
(entry.dispatchState === 'unknown' && entry.recovered !== true && entry.fence === ctx.fence)
)
) {
return 'Resolve pending or unconfirmed messages before using this command.'
}
@@ -36,6 +36,7 @@ export type AgentSessionMutationOperationAdmission = {
hostFingerprint: string
now: number
operationIdScope?: 'global'
conversationWrite?: true
}
export type AgentSessionMutationOperationDecision = {
@@ -149,7 +150,8 @@ export function admitAgentSessionMutationOperation(
envelope: args.envelope,
hostFingerprint: args.hostFingerprint,
ledger: ledger.decision,
lease: record.lease
lease: record.lease,
...(args.conversationWrite ? { conversationWrite: true } : {})
})
if (ledger.decision.decision === 'admit' && admission.decision === 'refused') {
ledger.rows.delete(agentSessionOperationKey(operation.callerKey, operation.operationId))
@@ -1,8 +1,8 @@
// A Claude chat is published the moment its child spawns, before the CLI has answered initialize.
// A send in that window — into a fresh start, or into the restart a send itself asked for after
// a start that failed — is admitted and held until the child proves its start. When the CLI dies
// first, the held message is rejected with the CLI's own diagnostic, the chat shows the cause
// once, and nothing is left as a delivery nobody can confirm. Against the production runtime,
// A send in that window — into a fresh start, or into the restart the delivery loop makes for a
// send after a start that failed — is accepted and stays queued until the child proves its start.
// When the CLI dies first, the queued message is rejected with the CLI's own diagnostic, the chat
// shows the cause once, and nothing is left as a delivery nobody can confirm. Against the production runtime,
// adapter, record store and host, with only the CLI process scripted.
import { afterEach, describe, expect, it, vi } from 'vitest'
@@ -39,7 +39,7 @@ async function send(host: StructuredAgentSessionHost, text: string): Promise<str
},
body
})
// Admitted and held for the start, never refused: the message shows as sent.
// Accepted and queued for the start, never refused: the message shows as sent.
expect(sent, JSON.stringify(sent)).toMatchObject({
ok: true,
replayed: false,
@@ -85,10 +85,13 @@ describe('a send into a Claude chat whose CLI keeps failing at startup', () => {
expect(statusRows(host)).toEqual([expect.stringContaining('not signed in')])
const releasedFence = fence(host)
// The send asks for the child back and is held for its start; the CLI dies again first.
// The delivery loop asks for the child back and the message waits for its start; the CLI
// dies again first.
const held = await send(host, 'hello?')
expect(claude.children(SESSION)).toHaveLength(2)
expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live')
await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(2))
await vi.waitFor(() =>
expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live')
)
await failLatestStart(host, 2)
// Rejected with the cause, not left in doubt; one row for this attempt names it.
@@ -111,8 +114,8 @@ describe('a send into a Claude chat whose CLI keeps failing at startup', () => {
// The user signs in and retries: one restart, proven, written to the CLI.
claude.behave(SESSION, {})
await send(host, 'hello again')
expect(claude.children(SESSION)).toHaveLength(3)
expect(fence(host)).toBe(releasedFence + 3)
await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(3))
await vi.waitFor(() => expect(fence(host)).toBe(releasedFence + 3))
await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send'))
await vi.waitFor(() =>
expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live')
@@ -122,7 +125,7 @@ describe('a send into a Claude chat whose CLI keeps failing at startup', () => {
})
describe('a send while the first Claude start is still answering initialize', () => {
it('is held, and written once the CLI proves its start', async () => {
it('is queued, and written once the CLI proves its start', async () => {
claude.behave(SESSION, { initHangs: true })
const host = await claude.install()
await host.attach(CALLER, claude.attachParams(SESSION, null))
@@ -130,7 +133,7 @@ describe('a send while the first Claude start is still answering initialize', ()
await send(host, 'hello')
expect(claude.child(SESSION).calls).not.toContain('send')
// The CLI answers: startup lands and the held message is written to the proven child.
// The CLI answers: startup lands and the queued message is written to the proven child.
claude.child(SESSION).answerInit()
await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send'))
@@ -1,10 +1,9 @@
// A send restarts a chat's Claude child and is admitted against it while it is still starting.
// When that child dies before the send's dispatch reaches the adapter, the adapter has no session
// to hold the message for, so the dispatch throws. A child that never proved its start accepted
// nothing — input is only written after initialize — so the send settles `rejected` with the
// child's own diagnostic, never as a delivery nobody can confirm, and a client that was
// subscribed the whole time receives the failure row and the rejected submission over the wire.
// Against the production runtime, adapter, record store and host, with only the CLI scripted.
// A send is accepted into a chat whose Claude child is gone, and its delivery restarts the child.
// When that child dies before it proves its start, the message was never handed to it — delivery
// waits for the start — so the send settles `rejected` with the child's own diagnostic, never as a
// delivery nobody can confirm, and a client that was subscribed the whole time receives the
// failure row and the rejected submission over the wire. Against the production runtime, adapter,
// record store and host, with only the CLI scripted.
import { afterEach, describe, expect, it, vi } from 'vitest'
import { computeAgentSessionPayloadFingerprint } from '../../shared/agent-session-mutation-envelope'
@@ -18,6 +17,11 @@ const SESSION = 'claude-send-restart-dies-first'
const CALLER = { callerKey: 'client-1' }
const DIAGNOSTIC = 'claude stream-json exited (code 1): claude: not signed in (rig)'
/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */
function eventually(assertion: () => void): Promise<void> {
return vi.waitFor(assertion, { timeout: 10_000 })
}
let claude = createScriptedClaudeRuntime([SESSION])
let operations = 0
/** The dispatch state each send was answered with, before any exit settled it. */
@@ -86,24 +90,14 @@ function submission(host: StructuredAgentSessionHost, clientMessageId: string) {
}
async function failLatestStart(host: StructuredAgentSessionHost, count: number): Promise<void> {
await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(count))
await eventually(() => expect(claude.children(SESSION)).toHaveLength(count))
claude.child(SESSION).exit(new Error(DIAGNOSTIC))
await waitForStructuredAgentSessionRecovery()
await vi.waitFor(() =>
await eventually(() =>
expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released')
)
}
/** The restarted child dies the instant the send's dispatch reaches the adapter. */
function killChildAtDispatch(host: StructuredAgentSessionHost): void {
const adapter = host.deps.adapter
const dispatch = adapter.dispatch.bind(adapter)
vi.spyOn(adapter, 'dispatch').mockImplementationOnce((input) => {
claude.child(SESSION).exit(new Error(DIAGNOSTIC))
return dispatch(input)
})
}
/** Everything a subscriber received, flattened to the rows and submissions it was shown. */
function received(events: AgentSessionSubscribeEvent[]) {
const statusTexts: string[] = []
@@ -129,7 +123,7 @@ function received(events: AgentSessionSubscribeEvent[]) {
return { statusTexts, submissions, fences }
}
describe('a send whose restarted Claude child dies before the dispatch reaches the adapter', () => {
describe('a send whose restarted Claude child dies before it proves its start', () => {
it('settles rejected with the diagnostic, keeps one failure row, and a Retry is one new attempt', async () => {
claude.behave(SESSION, { initHangs: true })
const host = await claude.install()
@@ -139,18 +133,13 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t
await failLatestStart(host, 1)
const releasedFence = fence(host)
killChildAtDispatch(host)
const sent = await send(host, 'hello?')
// The send's own answer already says it was not delivered; it does not wait for the exit.
expect(answered.get(sent)).toBe('rejected')
expect(claude.children(SESSION)).toHaveLength(2)
await waitForStructuredAgentSessionRecovery()
await vi.waitFor(() =>
expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released')
)
// Accepted: the answer comes before the restart it needs.
expect(answered.get(sent)).toBe('pending')
await failLatestStart(host, 2)
// Provably not delivered, with the cause; not "unconfirmed".
await vi.waitFor(() =>
await eventually(() =>
expect(submission(host, sent)).toMatchObject({
dispatchState: 'rejected',
// Worded for the user: the red line under the composer shows it as it stands.
@@ -168,40 +157,14 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t
// Retry under a new id: one restart, and once the CLI is healthy the message is written.
claude.behave(SESSION, {})
await send(host, 'hello again')
expect(claude.children(SESSION)).toHaveLength(3)
await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send'))
await eventually(() => expect(claude.children(SESSION)).toHaveLength(3))
await eventually(() => expect(claude.child(SESSION).calls).toContain('send'))
expect(claude.child(SESSION).calls.filter((call) => call === 'send')).toHaveLength(1)
expect(statusRows(host)).toHaveLength(2)
})
it('names the diagnostic even when the exit was fully processed before the dispatch arrived', async () => {
claude.behave(SESSION, { initHangs: true })
const host = await claude.install()
await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({
ok: true
})
await failLatestStart(host, 1)
const adapter = host.deps.adapter
const dispatch = adapter.dispatch.bind(adapter)
vi.spyOn(adapter, 'dispatch').mockImplementationOnce(async (input) => {
claude.child(SESSION).exit(new Error(DIAGNOSTIC))
// The adapter settles and publishes the exit; the host's own settlement waits behind this send.
await new Promise((resolve) => setTimeout(resolve, 300))
return dispatch(input)
})
const sent = await send(host, 'hello?')
expect(answered.get(sent)).toBe('rejected')
await waitForStructuredAgentSessionRecovery()
expect(submission(host, sent)).toMatchObject({
dispatchState: 'rejected',
reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.`
})
expect(statusRows(host)).toHaveLength(2)
})
// A restart refused because its child died before it was handed over leaves one row, from the
// send, in the words any failed start uses.
// delivery, in the words any failed start uses, and rejects the message with them.
it.each(['spawn', 'start-time-read'] as const)(
'leaves one row for a restart whose child exits at %s',
async (at) => {
@@ -213,10 +176,13 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t
await failLatestStart(host, 1)
claude.behave(SESSION, { exitsDuringSpawn: { diagnostic: DIAGNOSTIC, at } })
await expect(attempt(host, 'hello?')).resolves.toMatchObject({
ok: false,
refusal: { code: 'agent_session_owner_restart_failed' }
})
const sent = await send(host, 'hello?')
await eventually(() =>
expect(submission(host, sent)).toMatchObject({
dispatchState: 'rejected',
reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.`
})
)
await waitForStructuredAgentSessionRecovery()
expect(statusRows(host)).toEqual([
@@ -242,14 +208,10 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t
})
try {
await failLatestStart(host, 1)
killChildAtDispatch(host)
const sent = await send(host, 'hello?')
await waitForStructuredAgentSessionRecovery()
await vi.waitFor(() =>
expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released')
)
await failLatestStart(host, 2)
await vi.waitFor(() => {
await eventually(() => {
const seen = received(events)
expect(seen.submissions.get(sent)).toBe('rejected')
expect(seen.statusTexts).toContainEqual(
@@ -285,7 +247,7 @@ describe('a chat whose Claude CLI keeps failing to start, seen by a subscriber o
return rows
}
it('shows one row naming the cause per failed attempt, admitted or refused, and none once the CLI is fixed', async () => {
it('shows one row naming the cause per failed attempt, however the start died, and none once the CLI is fixed', async () => {
claude.behave(SESSION, { initHangs: true })
const host = await claude.install()
const events: AgentSessionSubscribeEvent[] = []
@@ -301,37 +263,38 @@ describe('a chat whose Claude CLI keeps failing to start, seen by a subscriber o
})
try {
await failLatestStart(host, 1)
await vi.waitFor(() => expect([...shownRows(events).values()]).toEqual([STARTUP_FAILURE]))
await eventually(() => expect([...shownRows(events).values()]).toEqual([STARTUP_FAILURE]))
// Send: admitted against the restarted child, which dies before starting.
killChildAtDispatch(host)
const sent = await attempt(host, 'hello?')
await waitForStructuredAgentSessionRecovery()
expect(sent).toMatchObject({
ok: true,
value: { submission: { dispatchState: 'rejected', reason: STARTUP_FAILURE } }
})
await vi.waitFor(() =>
// Send: accepted, and its delivery restarts the child, which dies before starting.
const sent = await send(host, 'hello?')
await failLatestStart(host, 2)
await eventually(() =>
expect(submission(host, sent)).toMatchObject({
dispatchState: 'rejected',
reason: STARTUP_FAILURE
})
)
await eventually(() =>
expect([...shownRows(events).values()]).toEqual([STARTUP_FAILURE, STARTUP_FAILURE])
)
// Retry while still broken: this restart dies before its child is handed over, so the send
// is refused before admission. Still one row, saying the same thing.
// Retry while still broken: this restart dies before its child is handed over, so the
// delivery's start is refused. Still one row, saying the same thing, on the rejected message.
claude.behave(SESSION, {
exitsDuringSpawn: {
diagnostic: 'claude stream-json exited (code 1): claude: not signed in (rig)',
at: 'start-time-read'
}
})
await expect(attempt(host, 'hello?')).resolves.toMatchObject({
ok: false,
refusal: {
code: 'agent_session_owner_restart_failed',
message: expect.stringMatching(/couldn't restart: .*not signed in \(rig\)/)
}
})
const retried = await send(host, 'hello?')
await eventually(() =>
expect(submission(host, retried)).toMatchObject({
dispatchState: 'rejected',
reason: STARTUP_FAILURE
})
)
await waitForStructuredAgentSessionRecovery()
await vi.waitFor(() =>
await eventually(() =>
expect([...shownRows(events).values()]).toEqual([
STARTUP_FAILURE,
STARTUP_FAILURE,
@@ -342,7 +305,7 @@ describe('a chat whose Claude CLI keeps failing to start, seen by a subscriber o
// The CLI is fixed: Retry delivers and adds no row.
claude.behave(SESSION, {})
await expect(attempt(host, 'hello?')).resolves.toMatchObject({ ok: true })
await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send'))
await eventually(() => expect(claude.child(SESSION).calls).toContain('send'))
expect(shownRows(events).size).toBe(3)
} finally {
unsubscribe()
@@ -91,7 +91,7 @@ describe('structured mailbox pointer host', () => {
value: { submission: { dispatchState } }
})
)
hostRef.current = { send }
hostRef.current = { send, waitForSendSettlement: async () => undefined }
await expect(
createStructuredMailboxPointerHost().send({
sessionId: 's1',
@@ -107,6 +107,28 @@ describe('structured mailbox pointer host', () => {
expect(send.mock.calls[0]![1]!.retryUnknown).toBeUndefined()
})
it('consumes mail once an accepted nudge is delivered while the worker starts (W10)', async () => {
hostRef.current = {
send: async () => ({
ok: true,
value: { clientMessageId: 'op1', submission: { dispatchState: 'pending' } }
}),
waitForSendSettlement: async () => ({
value: { clientMessageId: 'op1', submission: { dispatchState: 'accepted' } }
})
}
await expect(
createStructuredMailboxPointerHost().send({
sessionId: 's1',
dispatchId: 'd1',
operationId: 'op1',
expectedRuntimeFence: 1,
payloadFingerprint: 'fp',
body: { kind: 'message', role: 'user', blocks: [] }
} as never)
).resolves.toEqual({ kind: 'sent', state: 'accepted' })
})
it('scopes direct peer mail to the session when there is no dispatch to scope to', async () => {
// Direct mail is addressed to the worker's own handle, so there may be no dispatch at all.
// The ledger is keyed on (callerKey, operationId): a key derived from the session keeps that
@@ -6,6 +6,7 @@
* the send and reports what the host said.
*/
import { ORCHESTRATION_READINESS_TIMEOUT_MS } from '../../../shared/orchestration-timing-budgets'
import { AGENT_SESSION_NOT_ATTACHED } from '../../native-chat/agent-session-wire/structured-agent-session-mutation-admission'
import { getStructuredAgentSessionHost } from '../../native-chat/agent-session-wire/structured-agent-session-registry'
import type { StructuredMailboxPointerHost } from './structured-mailbox-pointer-delivery'
@@ -94,8 +95,19 @@ export function createStructuredMailboxPointerHost(): StructuredMailboxPointerHo
? { kind: 'unattached' }
: { kind: 'sent', state: 'rejected' }
}
// `pending` is not yet an acknowledgement; only `accepted` may consume mail.
const state = result.value.submission.dispatchState
// `pending` is not yet an acknowledgement; only `accepted` may consume mail. Accepted is not
// delivered, so wait out a start; a wait that runs out parks for the next journal edge.
const submission =
result.value.submission.dispatchState === 'pending'
? ((
await host
.waitForSendSettlement(input.sessionId, result.value.clientMessageId, {
budgetMs: ORCHESTRATION_READINESS_TIMEOUT_MS
})
.catch(() => undefined)
)?.value.submission ?? result.value.submission)
: result.value.submission
const state = submission.dispatchState
return {
kind: 'sent',
state: state === 'accepted' ? 'accepted' : state === 'rejected' ? 'rejected' : 'unknown'
@@ -224,10 +224,17 @@ describe('structured worker session hold', () => {
})
describe('structured worker dispatch preamble', () => {
function hostWithSubmission(submission: Record<string, unknown>) {
function hostWithSubmission(
submission: Record<string, unknown>,
delivered?: Record<string, unknown>
) {
return {
deps: { store: { getRecord: () => ({ lease: { runtimeFence: 7 } }) } },
send: async () => ({ ok: true, value: { clientMessageId: 'c1', submission } })
send: async () => ({ ok: true, value: { clientMessageId: 'c1', submission } }),
// What the submission settled as while the worker's agent started; undefined when the
// start outlasted the wait.
waitForSendSettlement: async () =>
delivered ? { value: { clientMessageId: 'c1', submission: delivered } } : undefined
} as never
}
@@ -240,6 +247,17 @@ describe('structured worker dispatch preamble', () => {
).resolves.toBeUndefined()
})
it('waits for an accepted preamble to be delivered, and reports that delivery (W10)', async () => {
await expect(
send(
hostWithSubmission(
{ dispatchState: 'pending', reason: null },
{ dispatchState: 'accepted', reason: null }
)
)
).resolves.toBeUndefined()
})
it('never claims delivery for a submission the provider never acknowledged', async () => {
// `dispatchSafely` turns ANY thrown adapter call — provider child dead, transport dropped —
// into `unknown`, and `performSend` still returns ok. Reporting that as `dispatch_input:
@@ -14,6 +14,7 @@
import { randomUUID } from 'node:crypto'
import { isDefinitiveAgentSessionCreateRefusal } from '../../../../shared/agent-session-definitive-refusal'
import type { AgentJournalMessageItem } from '../../../../shared/agent-session-journal-types'
import { ORCHESTRATION_READINESS_TIMEOUT_MS } from '../../../../shared/orchestration-timing-budgets'
import type { StructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-host'
import { getStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry'
import type { OrcaRuntimeService } from '../../orca-runtime'
@@ -244,7 +245,18 @@ export async function sendStructuredWorkerPreamble(args: {
if (!result.ok) {
throw new Error(`The dispatch preamble was refused: ${result.refusal.message}`)
}
const submission = result.value.submission
// Accepted is not delivered: the worker's agent may still be starting. Never re-sent; a wait
// that runs out is reported as unknown below and settles when the submission does.
const submission =
result.value.submission.dispatchState === 'pending'
? ((
await args.host
.waitForSendSettlement(args.sessionId, result.value.clientMessageId, {
budgetMs: ORCHESTRATION_READINESS_TIMEOUT_MS
})
.catch(() => undefined)
)?.value.submission ?? result.value.submission)
: result.value.submission
if (submission.dispatchState === 'accepted') {
return
}
@@ -0,0 +1,84 @@
// Which clients get a send answered at acceptance, and which have their reply held until the
// message is handed over: a client that cannot show a rejection after `pending` must not see one.
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import {
ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES,
STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY
} from '../../../../shared/protocol-version'
import { DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES } from '../../../ipc/desktop-renderer-runtime-capabilities'
import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement'
import {
call,
clearStructuredHostStub,
hostCalls,
installStructuredHostStub,
SESSION,
sendParams,
STRUCTURED_CLIENT
} from './structured-agent-session-rpc.test-fixture'
beforeEach(() => {
installStructuredHostStub()
})
afterEach(() => {
clearStructuredHostStub()
})
describe('agentSession.send reply timing', () => {
it('holds a pending reply until handover for a client that cannot show a later rejection', async () => {
hostCalls.send.mockResolvedValueOnce(pendingSendResult())
hostCalls.waitForSendSettlement.mockResolvedValueOnce(undefined)
await call('agentSession.send', sendParams(), STRUCTURED_CLIENT)
expect(hostCalls.waitForSendSettlement).toHaveBeenCalledWith(SESSION, 'client-1', {
until: 'handed-over',
budgetMs: STRUCTURED_AGENT_SESSION_START_WAIT_MS
})
})
it('answers at acceptance for the local desktop and paired desktop clients (W2)', async () => {
for (const clientCapabilities of [
DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES,
// A paired desktop gains the structured surface as its own capability; the reply rule rides
// on the list it already sends.
[...ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY]
]) {
hostCalls.send.mockResolvedValueOnce(pendingSendResult())
const response = await call('agentSession.send', sendParams(), {
clientKind: 'runtime',
clientCapabilities: [...clientCapabilities]
})
expect(response).toMatchObject({
ok: true,
result: { value: { submission: { dispatchState: 'pending' } } }
})
}
expect(hostCalls.waitForSendSettlement).not.toHaveBeenCalled()
})
})
function pendingSendResult() {
return {
ok: true,
replayed: false,
fence: 1,
cursor: { epoch: 'epoch-a', sequence: 1 },
value: {
clientMessageId: 'client-1',
submission: {
clientMessageId: 'client-1',
fence: 1,
payloadFingerprint: 'fingerprint',
dispatchState: 'pending' as const,
providerItemId: null,
reason: null,
submittedAt: 1,
resolvedAt: null,
handoverRecorded: true as const
}
}
}
}
@@ -1,26 +1,43 @@
import { AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version'
import {
AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY,
AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY
} from '../../../../shared/protocol-version'
import type { StructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-host'
import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement'
import type { RpcContext } from '../core'
import { requireStructuredHost, structuredCallerFor } from './structured-agent-session-gate'
/**
* A send answers once the host accepts it. A client that predates that answer cannot show a
* message rejected after it, so its reply is held until the message is handed over or rejected;
* one that predates pending replies at all waits, as before, for the provider's answer.
*/
export async function sendStructuredAgentSessionForClient(
params: Parameters<StructuredAgentSessionHost['send']>[1],
context: RpcContext
) {
const host = requireStructuredHost(context)
const result = await host.send(structuredCallerFor(context), params)
const capabilities = context.clientCapabilities ?? []
if (
!result.ok ||
result.value.submission.dispatchState !== 'pending' ||
context.clientKind === undefined ||
context.clientCapabilities?.includes(AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY)
capabilities.includes(AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY)
) {
return result
}
// The start that used to run before the reply now runs after acceptance, so both waits cover it.
const settled = await host.waitForSendSettlement(
params.envelope.sessionId,
result.value.clientMessageId,
context.signal
{
until: capabilities.includes(AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY)
? 'handed-over'
: 'answered',
budgetMs: STRUCTURED_AGENT_SESSION_START_WAIT_MS,
...(context.signal ? { signal: context.signal } : {})
}
)
return settled ? { ...result, ...settled } : result
}
@@ -12,6 +12,7 @@ import {
STRUCTURED_AGENT_SESSION_REVEAL_RUNTIME_CAPABILITY,
STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY
} from '../../../../shared/protocol-version'
import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement'
import { computeAgentSessionPayloadFingerprint } from '../../../../shared/agent-session-mutation-envelope'
import { ALL_RPC_METHODS } from './index'
import { STRUCTURED_AGENT_SESSION_METHODS } from './structured-agent-session'
@@ -252,11 +253,11 @@ describe('capability gating', () => {
signal: controller.signal
})
expect(hostCalls.waitForSendSettlement).toHaveBeenCalledWith(
SESSION,
'client-1',
controller.signal
)
expect(hostCalls.waitForSendSettlement).toHaveBeenCalledWith(SESSION, 'client-1', {
until: 'answered',
budgetMs: STRUCTURED_AGENT_SESSION_START_WAIT_MS,
signal: controller.signal
})
expect(response).toMatchObject({
ok: true,
result: {
@@ -302,36 +303,6 @@ describe('capability gating', () => {
})
})
it('returns durable pending immediately to clients that understand admission', async () => {
hostCalls.send.mockResolvedValueOnce({
ok: true,
replayed: false,
fence: 1,
cursor: { epoch: 'epoch-a', sequence: 1 },
value: {
clientMessageId: 'client-1',
submission: {
clientMessageId: 'client-1',
fence: 1,
payloadFingerprint: 'fingerprint',
dispatchState: 'pending',
providerItemId: null,
reason: null,
submittedAt: 1,
resolvedAt: null
}
}
})
const response = await call('agentSession.send', sendParams(), STRUCTURED_CLIENT)
expect(hostCalls.waitForSendSettlement).not.toHaveBeenCalled()
expect(response).toMatchObject({
ok: true,
result: { value: { submission: { dispatchState: 'pending' } } }
})
})
it('requires the host structured-chat setting for mobile clients', async () => {
const response = await call('agentSession.send', sendParams(), STRUCTURED_MOBILE_CLIENT, {
getClientSettings: () => ({ experimentalStructuredNativeChat: false })
@@ -317,11 +317,14 @@ describe('a structured codex session over agentSession.*', () => {
body
}
const turnStarts = () => codex.live().calls.filter((entry) => entry.method === 'turn/start')
await ok('agentSession.send', params)
// Accepted first; the delivery loop hands it over once.
await vi.waitFor(() => expect(turnStarts()).toHaveLength(1))
const replay = await call('agentSession.send', params)
expect(replay).toMatchObject({ ok: true, result: { ok: true, replayed: true } })
expect(codex.live().calls.filter((entry) => entry.method === 'turn/start')).toHaveLength(1)
expect(turnStarts()).toHaveLength(1)
})
it('joins an acquired attach through journal bind before draining final rows', async () => {
@@ -218,6 +218,11 @@ function createIntentParams() {
}
let codex: CodexScript
/** Accepted first; the delivery loop hands the send over as `turn/start` after the reply. */
const handedOverAs = (params: Record<string, unknown>) =>
vi.waitFor(() =>
expect(codex.live().calls.at(-1)).toMatchObject({ method: 'turn/start', params })
)
let root: string
let dispatcher: RpcDispatcher
let bootEnvironmentReads: number
@@ -466,10 +471,7 @@ describe('a structured codex session over agentSession.*', () => {
// send coalesced into a running turn is answered with that turn's id, so
// which message landed where is knowable only from the echo.
expect(sent.submission).toMatchObject({ dispatchState: 'pending', providerItemId: null })
expect(codex.live().calls.at(-1)).toMatchObject({
method: 'turn/start',
params: { threadId: THREAD, clientUserMessageId: sent.clientMessageId }
})
await handedOverAs({ threadId: THREAD, clientUserMessageId: sent.clientMessageId })
codex.notify('turn/started', { turn: { id: TURN } })
// Codex echoes the message back carrying the `clientId` it was sent under,
@@ -557,14 +559,11 @@ describe('a structured codex session over agentSession.*', () => {
// "delivery unconfirmed" — it carries no identity yet, because the response
// to a coalesced send names the running turn rather than this message.
expect(sent.submission).toMatchObject({ dispatchState: 'pending', providerItemId: null })
expect(codex.live().calls.at(-1)).toMatchObject({
method: 'turn/start',
params: {
threadId: THREAD,
clientUserMessageId: sent.clientMessageId,
model: 'gpt-live',
effort: 'high'
}
await handedOverAs({
threadId: THREAD,
clientUserMessageId: sent.clientMessageId,
model: 'gpt-live',
effort: 'high'
})
// ── stream ──────────────────────────────────────────────────────────────
@@ -128,7 +128,8 @@ describe('structured session runtime provider-exit wiring', () => {
await expect(
host.send({ callerKey: 'runtime-test' }, { envelope, body })
).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } })
expect(turn).toBe(1)
// The send answers at acceptance; the delivery loop hands it to the reacquired child after.
await vi.waitFor(() => expect(turn).toBe(1))
})
it('does not reacquire when the production exit callback comes from a requested close', async () => {

Some files were not shown because too many files have changed in this diff Show More