diff --git a/src/main/acp/acp-structured-compaction.test.ts b/src/main/acp/acp-structured-compaction.test.ts index 34e21afff44..d690b2474a2 100644 --- a/src/main/acp/acp-structured-compaction.test.ts +++ b/src/main/acp/acp-structured-compaction.test.ts @@ -446,6 +446,8 @@ describe('ACP compaction settlement', () => { failure: expect.objectContaining({ kind: 'notSignedIn' }) }) ]) + // A sign-in failure is the agent's, so cards waiting behind the compaction stay held. + expect(rig.rig.journal.inputOrder.newestTurn.failure()).toBeGreaterThan(0) }) it('fails a compaction the agent ended with any other stop reason', async () => { @@ -456,6 +458,8 @@ describe('ACP compaction settlement', () => { expect(inside).toEqual([ expect.objectContaining({ failure: expect.objectContaining({ kind: 'compactionFailed' }) }) ]) + // Its own verdict neither holds nor releases the cards waiting behind it. + expect(rig.rig.journal.inputOrder.newestTurn.failure()).toBe(0) }) it('ends a stopped compaction as cancelled, with no result row', async () => { diff --git a/src/main/acp/acp-timeline-turn-failures.test.ts b/src/main/acp/acp-timeline-turn-failures.test.ts index d6a87e41400..28b0feb904a 100644 --- a/src/main/acp/acp-timeline-turn-failures.test.ts +++ b/src/main/acp/acp-timeline-turn-failures.test.ts @@ -258,6 +258,8 @@ describe('a failed ACP turn says why', () => { ) f.apply(lane.promptResult('c1', { stopReason: 'max_tokens' }, 1003)) expect(statusRows(await f.rig.rows())).toEqual([]) + // The failed outcome alone still holds the cards waiting behind it. + expect(f.rig.journal.inputOrder.newestTurn.failure()).toBeGreaterThan(0) }) it('does not reopen or retarget an ended prompt for its late frame', async () => { diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts index 1125f0dca08..378131807b0 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts @@ -124,6 +124,17 @@ export function replaceJournalEpoch(input: { } const rows: JournalRow[] = [epochRow] applyJournalRow(state, epochRow) + const { lifted, liveStop, reopened, failureHeld } = input.queuePause + const place = () => ({ state, seq: state.lastSequence + 1, fence: input.fence, ts: input.now() }) + const resume = () => { + const row = buildJournalQueueResumeRow(place()) + applyJournalRow(state, row) + rows.push(row) + } + // A rebuild preserves a failure hold without re-holding a queue the person already resumed. + if (lifted && failureHeld) { + resume() + } for (const item of input.items) { const row = buildJournalItemRow({ state, @@ -202,12 +213,8 @@ export function replaceJournalEpoch(input: { : change ) } - const { lifted, liveStop, reopened } = input.queuePause - const place = () => ({ state, seq: state.lastSequence + 1, fence: input.fence, ts: input.now() }) - if (lifted) { - const row = buildJournalQueueResumeRow(place()) - applyJournalRow(state, row) - rows.push(row) + if (lifted && !failureHeld) { + resume() } if (liveStop) { const row = buildJournalStopEventRow({ ...place(), event: liveStop }) diff --git a/src/main/native-chat/agent-session-journal/journal-failed-queue-scaling.test.ts b/src/main/native-chat/agent-session-journal/journal-failed-queue-scaling.test.ts new file mode 100644 index 00000000000..832b9f1115e --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-failed-queue-scaling.test.ts @@ -0,0 +1,663 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { applyJournalRow, createJournalReducerState } from './journal-reducer' +import { createTrackedJournalOpener } from './journal-host-database-test-support' +import { + agentSessionJournalProviderHandle, + codexProviderHandle +} from '../../../shared/agent-session/lifecycle/agent-session-provider-handle-encoding' +import { readQueuePublication } from '../agent-session-wire/structured-agent-session-queued-publication' +import { StructuredAgentSessionCurrentWork } from '../agent-session-wire/structured-agent-session-current-work' +import { journalInputOrder } from './journal-input-order' +import { JournalInputCardIndex } from './journal-input-card-index' +import { JournalInputPlaceIndex } from './journal-input-place-index' +import { + buildJournalQueueClearRow, + buildJournalQueueResumeRow +} from './journal-stop-and-resume-rows' +import { isolateJournalTransition } from './journal-reducer-transition' +import { failedQueueTurnSequence } from './queued-message-pause' +import type { JournalRow } from './journal-row-schema' +import { JournalNewestTurn } from './journal-newest-turn' +import { structuredAgentSessionCommandTurn } from '../../../shared/agent-session/view/structured-agent-session-command-turn-identity' +import { agentSessionFailureFact } from '../../../shared/agent-session/failures/agent-session-failure' +import { agentSessionFailureWords } from '../../../shared/agent-session/failures/agent-session-failure-words' + +function usageLimit(turnItemId: string) { + return { + kind: 'item' as const, + itemId: `limit:${turnItemId}`, + revision: 1, + body: { + kind: 'status' as const, + tone: 'error' as const, + ...agentSessionFailureWords(agentSessionFailureFact('usageLimitReached'), { surface: 'row' }) + }, + turnScope: { kind: 'turn' as const, turnItemId } + } +} + +/** Live failure position next to what a restore over the same items would derive. */ +function failureFold() { + const state = createJournalReducerState('session', 'epoch') + const revisions = new Map() + const apply = ( + row: Omit< + Extract, + 'v' | 'epoch' | 'seq' | 'ts' | 'fence' | 'revision' + > + ) => { + const revision = (revisions.get(row.itemId) ?? 0) + 1 + revisions.set(row.itemId, revision) + const seq = state.lastSequence + 1 + applyJournalRow(state, { v: 4, epoch: 'epoch', seq, ts: seq, fence: 1, revision, ...row }) + return seq + } + const turn = (itemId: string, turnId = itemId, outcome: 'failure' | 'success' = 'failure') => + apply({ kind: 'item', itemId, body: { kind: 'turn', turnId, state: 'completed', outcome } }) + const command = (id: string) => { + const { itemId, turnId } = structuredAgentSessionCommandTurn(id) + return { itemId, settle: () => turn(itemId, turnId) } + } + return { + state, + apply, + turn, + command, + live: () => journalInputOrder(state).newestTurn.failure(), + restored: () => new JournalNewestTurn(state).failure() > 0 + } +} + +describe('which failure holds the queue beneath refused commands', () => { + it('releases when the failure beneath a refused command is revised to success', () => { + const f = failureFold() + const failedAt = f.turn('ordinary') + f.command('refused').settle() + expect(f.live()).toBe(failedAt) + f.turn('ordinary', 'ordinary', 'success') + expect([f.live(), f.restored()]).toEqual([0, false]) + }) + + it("holds on an older command's late usage limit beneath a newer refused command", () => { + const f = failureFold() + const older = f.command('older') + older.settle() + f.command('newer').settle() + expect(f.live()).toBe(0) + const reported = f.apply(usageLimit(older.itemId)) + expect([f.live(), f.restored()]).toEqual([reported, true]) + }) + + it('holds on a usage limit saved before its command turn row', () => { + const f = failureFold() + const command = f.command('out-of-order') + f.apply(usageLimit(command.itemId)) + const settled = command.settle() + expect([f.live(), f.restored()]).toEqual([settled, true]) + }) + + it('leaves the live failure untouched when a draft adding its evidence is discarded', () => { + const f = failureFold() + const failedAt = f.turn('ordinary') + const command = f.command('refused') + command.settle() + const pending = isolateJournalTransition(f.state, (draft) => + applyJournalRow(draft, { + v: 4, + epoch: 'epoch', + seq: 9, + ts: 9, + fence: 1, + ...usageLimit(command.itemId) + }) + ) + expect(journalInputOrder(pending.state).newestTurn.failure()).toBe(9) + expect(f.live()).toBe(failedAt) + }) +}) + +it.each([100, 1_000])( + 'checks failed-turn queue state without scanning %s history items', + (size) => { + const state = createJournalReducerState('session', 'epoch') + const base = (seq: number) => ({ v: 4, epoch: 'epoch', seq, ts: seq, fence: 1 }) + for (let n = 1; n <= size; n++) { + applyJournalRow(state, { + ...base(n), + kind: 'item', + itemId: `message-${n}`, + revision: 1, + body: { kind: 'message', role: 'assistant', blocks: [] } + }) + } + const turn: JournalRow = { + ...base(size + 1), + kind: 'item', + itemId: 'turn', + revision: 1, + body: { kind: 'turn', turnId: 'turn', state: 'running' } + } + applyJournalRow(state, turn) + const order = journalInputOrder(state) + expect(order.newestTurn.read()?.itemId).toBe('turn') + // Drafts read the live map through keys(), so count every way of walking it. + const walks = (['keys', 'values', 'entries', 'forEach', Symbol.iterator] as const).map( + (method) => vi.spyOn(state.items, method) + ) + const failed: JournalRow = { + ...turn, + ...base(size + 2), + revision: 2, + body: { kind: 'turn', turnId: 'turn', state: 'completed', outcome: 'failure' } + } + const pending = isolateJournalTransition(state, (draft) => applyJournalRow(draft, failed)) + expect(failedQueueTurnSequence(order.newestTurn.failure(), 0)).toBe(0) + const committed = pending.adopt() + for (let n = 0; n < 10; n++) { + const owner = journalInputOrder(committed) + expect(failedQueueTurnSequence(owner.newestTurn.failure(), 0)).toBe(size + 2) + expect(owner.head(committed, null)).toBeUndefined() + } + // A command refused on its own terms, then the agent's late usage-limit row for it. + const command = structuredAgentSessionCommandTurn('late') + applyJournalRow(committed, { + ...base(size + 3), + kind: 'item', + itemId: command.itemId, + revision: 1, + body: { kind: 'turn', turnId: command.turnId, state: 'completed', outcome: 'failure' } + }) + expect(journalInputOrder(committed).newestTurn.failure()).toBe(size + 2) + applyJournalRow(committed, { ...base(size + 4), ...usageLimit(command.itemId) }) + expect(journalInputOrder(committed).newestTurn.failure()).toBe(size + 4) + for (const walk of walks) { + expect(walk).not.toHaveBeenCalled() + walk.mockRestore() + } + for (const itemId of [command.itemId, 'turn']) { + applyJournalRow(committed, { + ...base(committed.lastSequence + 1), + kind: 'tombstone', + itemId, + revision: 3 + }) + } + expect(journalInputOrder(committed).newestTurn.read()).toBeNull() + } +) + +it.each([100, 1_000])( + 'publishes a failure pause without walking %s recovered sends', + async (size) => { + const root = await mkdtemp(join(tmpdir(), 'orca-failed-queue-scaling-')) + const journals = createTrackedJournalOpener() + const identity = { + sessionId: 's', + workspaceId: 'w', + hostId: 'h', + agent: 'codex' as const, + providerHandle: codexProviderHandle('thread') + } + const providerHandle = agentSessionJournalProviderHandle(identity) + const body = { kind: 'message' as const, role: 'user' as const, blocks: [] } + try { + const journal = await journals.open({ + identity, + stateDirectory: root + }) + await journal.queuedMessages.insert({ messageId: 'waiting', body }) + const state = journal.queuedMessages.inputState() + let seq = state.lastSequence + const base = () => ({ v: 4, epoch: state.epoch, seq: ++seq, ts: seq, fence: 1 }) + for (let n = 0; n < size; n++) { + const clientMessageId = `old-${n}` + applyJournalRow(state, { + ...base(), + kind: 'submission', + clientMessageId, + body, + providerHandle, + payloadFingerprint: clientMessageId, + handoverRecorded: true + }) + applyJournalRow(state, { + ...base(), + kind: 'dispatch', + clientMessageId, + state: 'unknown', + reason: 'host_restarted_before_acknowledgement', + providerItemId: null, + recovered: true + }) + } + applyJournalRow(state, { + ...base(), + kind: 'item', + itemId: 'turn', + revision: 1, + body: { kind: 'turn', turnId: 'turn', state: 'completed', outcome: 'failure' } + }) + const work = new StructuredAgentSessionCurrentWork(journal, null) + const gate = () => ({ record: null, work }) + const published = readQueuePublication(journal, gate) + expect(published.queuePause).toEqual({ reason: 'failed' }) + // The working check walks entered sends on main already; this contract covers the pause. + vi.spyOn(journal.inputOrder, 'enteredSubmissions').mockReturnValue([]) + // Count every read of a recovered send's state, however publication reaches it. + let reads = 0 + for (const entry of state.submissions.values()) { + const dispatchState = entry.dispatchState + Object.defineProperty(entry, 'dispatchState', { + get: () => (reads++, dispatchState), + configurable: true, + enumerable: true + }) + } + for (let n = 0; n < 10; n++) { + expect(readQueuePublication(journal, gate)).toBe(published) + } + expect(reads).toBe(0) + } finally { + vi.restoreAllMocks() + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } + } +) + +/** An AVL tree of T sources is at most 1.45 log2(T + 2) deep; per level, one source's update + * reaches a node, its sibling and their cached summaries, plus the row's fixed input-order work. */ +const perSource = (sources: number) => 32 * (Math.ceil(Math.log2(sources + 2)) + 1) + +/** Commits rows through real isolated transactions, counting each one's derived draft nodes and + * every walk of the saved items or of any map the failure projection owns. */ +function sourceIndexFold(size: number) { + const state = createJournalReducerState('session', 'epoch') + const revisions = new Map() + const revise = (itemId: string) => { + const revision = (revisions.get(itemId) ?? 0) + 1 + revisions.set(itemId, revision) + return revision + } + let seq = 0 + const base = () => ({ v: 4, epoch: 'epoch', seq: ++seq, ts: seq, fence: 1 }) as const + const row = (mutation: Record & { itemId: string }): JournalRow => + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: callers build item and tombstone rows of the v4 schema. + ({ ...base(), kind: 'item', revision: revise(mutation.itemId), ...mutation }) as JournalRow + const turn = (itemId: string, turnId: string, outcome?: 'failure' | 'success') => ({ + kind: 'item', + itemId, + body: outcome + ? { kind: 'turn', turnId, state: 'completed', outcome } + : { kind: 'turn', turnId, state: 'running' } + }) + const refusal = (id: string) => { + const { itemId, turnId } = structuredAgentSessionCommandTurn(id) + const settle = (): JournalRow => ({ + ...base(), + kind: 'lifecycle-batch', + settlementId: `command-settled:${id}`, + mutations: [ + { + kind: 'item', + itemId, + revision: revise(itemId), + body: { kind: 'turn', turnId, state: 'completed', outcome: 'failure' } + }, + { + kind: 'item', + itemId: `orca:command-result:${id}`, + revision: 1, + body: { + kind: 'status', + tone: 'error', + ...agentSessionFailureWords(agentSessionFailureFact('commandRefused'), { + surface: 'row' + }) + }, + turnScope: { kind: 'turn', turnItemId: itemId } + } + ] + }) + return { itemId, turnId, settle } + } + for (let n = 0; n < size; n++) { + applyJournalRow( + state, + row({ itemId: `message-${n}`, body: { kind: 'message', role: 'assistant', blocks: [] } }) + ) + } + const owner = journalInputOrder(state).newestTurn + const walks: { mock: { calls: readonly unknown[] } }[] = [] + let derived = 0 + const watch = () => { + const maps = [state.items, ...Object.values(owner).filter((value) => value instanceof Map)] + for (const map of maps) { + for (const method of ['keys', 'values', 'entries', 'forEach', Symbol.iterator] as const) { + walks.push(vi.spyOn(map, method)) + } + } + } + /** Keeps the most draft nodes one transaction reached per source it changed. */ + const commit = (rows: JournalRow[], sources = 1) => { + const pending = isolateJournalTransition(state, (draft) => { + for (const next of rows) { + applyJournalRow(draft, next) + } + }) + derived = Math.max(derived, pending.trackedDerived() / sources) + pending.adopt() + return rows.at(-1)!.seq + } + return { + state, + owner, + row, + turn, + refusal, + commit, + watch, + counts: () => ({ walks: walks.reduce((sum, spy) => sum + spy.mock.calls.length, 0), derived }), + status: (turnItemId: string, itemId: string) => row({ ...usageLimit(turnItemId), itemId }) + } +} + +describe.each([100, 1_000])('failure source index cost with %s commands', (size) => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it('updates only touched sources for refused chains and late or early evidence', () => { + const f = sourceIndexFold(size) + const failedAt = f.commit([f.row(f.turn('ordinary', 'ordinary', 'failure'))]) + const chain = Array.from({ length: size }, (_, n) => f.refusal(`chain-${n}`)) + for (const command of chain) { + applyJournalRow(f.state, command.settle()) + } + f.watch() + for (let n = 0; n < 10; n++) { + expect(f.commit([f.refusal(`more-${n}`).settle()])).toBeGreaterThan(failedAt) + expect(f.owner.failure()).toBe(failedAt) + } + for (let n = 0; n < 10; n++) { + const late = chain[size / 2 + n] + const reported = f.commit([f.status(late.itemId, `late-${n}`)]) + expect(f.owner.failure()).toBe(reported) + } + for (let n = 0; n < 10; n++) { + const early = f.refusal(`early-${n}`) + f.commit([f.status(early.itemId, `early-report-${n}`)]) + const settled = f.commit([early.settle()]) + expect(f.owner.failure()).toBe(settled) + expect(f.owner.read()?.itemId).toBe(early.itemId) + } + const { walks, derived } = f.counts() + expect(walks).toBe(0) + expect(derived).toBeLessThanOrEqual(perSource(2 * size + 32)) + }) + + it('demotes a started command to the newest qualifying one without a walk', () => { + const f = sourceIndexFold(size) + const successes = Array.from({ length: size }, (_, n) => + structuredAgentSessionCommandTurn(`ok-${n}`) + ) + for (const { itemId, turnId } of successes) { + applyJournalRow(f.state, f.row(f.turn(itemId, turnId, 'success'))) + } + f.watch() + for (let n = 0; n < 10; n++) { + const command = f.refusal(`refused-${n}`) + f.commit([f.row(f.turn(command.itemId, command.turnId))]) + expect(f.owner.read()?.itemId).toBe(command.itemId) + f.commit([command.settle()]) + expect([f.owner.read()?.itemId, f.owner.failure()]).toEqual([command.itemId, 0]) + } + f.commit([f.status(successes.at(-1)!.itemId, 'qualified')]) + expect(f.owner.failure()).toBe(0) + const { walks, derived } = f.counts() + expect(walks).toBe(0) + expect(derived).toBeLessThanOrEqual(perSource(size + 10)) + }) + + it('keeps a revised anchor and its failure position without visiting newer commands', () => { + const f = sourceIndexFold(size) + const failedAt = f.commit([f.row(f.turn('ordinary', 'ordinary', 'failure'))]) + for (let n = 0; n < size; n++) { + applyJournalRow(f.state, f.refusal(`chain-${n}`).settle()) + } + f.watch() + for (let n = 0; n < 10; n++) { + f.commit([f.row(f.turn('ordinary', 'ordinary', 'failure'))]) + expect(f.owner.failure()).toBe(failedAt) + } + const { walks, derived } = f.counts() + expect(walks).toBe(0) + expect(derived).toBeLessThanOrEqual(perSource(size + 1)) + }) + + it('removes rewound commands, including the actual latest, without rebuilding', () => { + const f = sourceIndexFold(size) + f.commit([f.row(f.turn('ordinary', 'ordinary', 'failure'))]) + const chain = Array.from({ length: size }, (_, n) => f.refusal(`chain-${n}`)) + for (const command of chain) { + applyJournalRow(f.state, command.settle()) + } + f.watch() + // The newest ten one by one, each the actual latest when removed. + for (const command of chain.slice(-10).toReversed()) { + f.commit([f.row({ kind: 'tombstone', itemId: command.itemId })]) + expect(f.owner.read()?.itemId).not.toBe(command.itemId) + } + // Then the rest in one rewind transaction, oldest first, as the writer emits them. + const rewound = chain + .slice(0, -10) + .map((command) => f.row({ kind: 'tombstone', itemId: command.itemId })) + const boundary = f.commit(rewound, rewound.length) + // What lay beneath them has unknown settlement order: no earlier Resume releases it. + expect([f.owner.read()?.itemId, f.owner.failure()]).toEqual(['ordinary', boundary]) + f.commit([f.row({ kind: 'tombstone', itemId: 'ordinary' })]) + expect([f.owner.read(), f.owner.failure()]).toEqual([null, 0]) + const { walks, derived } = f.counts() + expect(walks).toBe(0) + expect(derived).toBeLessThanOrEqual(perSource(size + 1)) + }) +}) + +describe.each([100, 1_000])('queue failures beside %s retained cards', (size) => { + afterEach(() => { + vi.restoreAllMocks() + }) + + // A failure holds every waiting card alike, so it gates reads and never revisits a card; only + // Stop, clear and reopen rebuild the card index. + it.each([1, 3])('holds %s waiting cards without touching retained history', (live) => { + const state = createJournalReducerState('session', 'epoch') + let seq = 0 + const base = () => ({ v: 4, epoch: 'epoch', seq: ++seq, ts: seq, fence: 1 }) as const + const place = () => ({ state, seq: ++seq, fence: 1, ts: seq }) + const body = { kind: 'message' as const, role: 'user' as const, blocks: [] } + const providerHandle = agentSessionJournalProviderHandle({ + agent: 'codex', + providerHandle: codexProviderHandle('thread') + }) + const queue = (messageId: string, position: number) => + applyJournalRow(state, { + ...base(), + kind: 'queue', + messageId, + change: { op: 'queued', position, body, queuedAt: { epoch: 'epoch', sequence: seq } } + }) + // Retained history no pause can hold: withdrawn cards and accepted hand-offs. + const history: string[] = [] + for (let n = 0; n < size; n++) { + const messageId = `old-${n}` + history.push(messageId) + queue(messageId, n) + if (n % 2 === 0) { + applyJournalRow(state, { ...base(), kind: 'queue', messageId, change: { op: 'withdrawn' } }) + continue + } + const clientMessageId = `sent-${n}` + applyJournalRow(state, { + ...base(), + kind: 'submission', + clientMessageId, + body, + providerHandle, + payloadFingerprint: clientMessageId, + handoverRecorded: true, + queuedMessageId: messageId + }) + applyJournalRow(state, { + ...base(), + kind: 'dispatch', + clientMessageId, + state: 'accepted', + providerItemId: null, + reason: null + }) + } + const waiting = Array.from({ length: live }, (_, n) => `waiting-${n}`) + waiting.forEach((messageId, n) => queue(messageId, size + n)) + /** Reads the head, its pause and the observed summary behind a new send. */ + const read = (target = state) => { + const owner = journalInputOrder(target) + const head = owner.head(target, null) + expect(head?.messageId).toBe('waiting-0') + const paused = owner.paused(target, head!, null) + const cards = Object.values(owner).find((value) => value instanceof JournalInputCardIndex)! + const source = { epoch: 'epoch', sequence: target.lastSequence + 1 } + const cut = owner.lineage.at(target.lastSequence) + 1 + const query = cards.query({ coordinate: source, context: { basis: 'observed', source, cut } }) + expect(query.head?.messageId).toBe('waiting-0') + expect(query.waiting?.messageId).toBe(paused ? undefined : 'waiting-0') + return paused + } + expect(read()).toBe(false) + const cards = Object.values(journalInputOrder(state)).find( + (value) => value instanceof JournalInputCardIndex + )! + const warmLegacy = () => { + const source = { epoch: 'epoch', sequence: state.lastSequence + 1 } + const cut = journalInputOrder(state).lineage.at(state.lastSequence) + 1 + cards.query({ coordinate: source, context: { basis: 'legacy', source, cut } }) + } + warmLegacy() + // Count every walk of the canonical cards and of each Map the card index owns, place and + // domain collections included, as plain call counts. + const methods = ['keys', 'values', 'entries', 'forEach', Symbol.iterator] as const + const canonical = methods.map((method) => vi.spyOn(state.queue.cards, method)) + const watched = new Set([state.queue.cards]) + const owned: { mock: { calls: unknown[] } }[] = [] + const attach = () => { + const seen = new Set() + const collect = (value: unknown) => { + if (typeof value !== 'object' || value === null || seen.has(value)) { + return + } + seen.add(value) + const entries = + value instanceof Map || value instanceof Set + ? [...value.values()] + : Object.values(Object.getOwnPropertyDescriptors(value)).map((field) => field.value) + if (value instanceof Map && !watched.has(value)) { + watched.add(value) + owned.push(...methods.map((method) => vi.spyOn(value, method))) + } + entries.forEach(collect) + } + collect(cards) + } + attach() + const sum = (spies: typeof owned) => spies.reduce((n, spy) => n + spy.mock.calls.length, 0) + const visits = vi.spyOn(JournalInputCardIndex.prototype, 'indexCard') + const sets = vi.spyOn(JournalInputPlaceIndex.prototype, 'set') + // Outside the clear's rebuild, every ensure keeps the indexes and walks nothing. + const keptNames = ['index', 'currentCoordinates', 'domains', 'indexed'] + const kept = (index: JournalInputCardIndex) => [ + index['index'], + index['currentCoordinates'], + index['domains'], + index['indexed'] + ] + const ensure = JournalInputCardIndex.prototype.ensure + let rebuilding = false + let ensures = 0 + vi.spyOn(JournalInputCardIndex.prototype, 'ensure').mockImplementation(function ( + this: JournalInputCardIndex, + target, + floor + ) { + const walked = sum(canonical) + sum(owned) + const before = kept(this) + ensure.call(this, target, floor) + if (!rebuilding) { + ensures++ + expect(sum(canonical) + sum(owned)).toBe(walked) + const after = kept(this) + expect(keptNames.filter((_, n) => after[n] !== before[n])).toEqual([]) + } + }) + /** Commits one row; `inDraft` reads before adoption too. No step may visit or place a card. */ + const commit = (row: JournalRow, paused: boolean, inDraft = false) => { + const before = [sum(canonical), sum(owned), sets.mock.calls.length, ensures] + const pending = isolateJournalTransition(state, (draft) => { + applyJournalRow(draft, row) + if (inDraft) { + expect(read(draft)).toBe(paused) + } + }) + pending.adopt() + expect(read()).toBe(paused) + // Main's spendReportedCommandCards fold scans the canonical cards once per item row. + expect(sum(canonical) - before[0]).toBe(row.kind === 'item' ? 1 : 0) + expect([sum(owned), sets.mock.calls.length]).toEqual(before.slice(1, 3)) + expect(ensures).toBeGreaterThan(before[3]) + expect(visits.mock.calls.length).toBe(0) + expect(pending.trackedDerived()).toBeLessThanOrEqual(512) + } + const turn = (n: number, outcome?: 'failure'): JournalRow => ({ + ...base(), + kind: 'item', + itemId: `turn-${n}`, + revision: outcome ? 2 : 1, + body: outcome + ? { kind: 'turn', turnId: `turn-${n}`, state: 'completed', outcome } + : { kind: 'turn', turnId: `turn-${n}`, state: 'running' } + }) + for (let n = 0; n < 10; n++) { + commit(turn(n), false) + commit(turn(n, 'failure'), true, n % 2 === 0) + } + // A discarded Resume leaves the live hold as it was. + isolateJournalTransition(state, (draft) => { + applyJournalRow(draft, buildJournalQueueResumeRow(place())) + expect(read(draft)).toBe(false) + }) + expect(read()).toBe(true) + expect(visits.mock.calls.length).toBe(0) + commit(buildJournalQueueResumeRow(place()), false) + // A clear naming every retained card rebuilds the index once, as it always has. + rebuilding = true + applyJournalRow( + state, + buildJournalQueueClearRow({ + ...place(), + clear: { operationId: 'clear', messageIds: [...history, ...waiting] } + }) + ) + expect(read()).toBe(true) + warmLegacy() + rebuilding = false + attach() + visits.mockClear() + // A failure beneath the clear neither revisits a card nor reads the clear's payload again. + const payload = vi.spyOn(state.queuePauseMarks.cleared!.messageIds, Symbol.iterator) + commit(turn(10), true) + commit(turn(10, 'failure'), true, true) + expect(payload).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-input-card-index.ts b/src/main/native-chat/agent-session-journal/journal-input-card-index.ts index 0a57fcfb24c..37d6f097db4 100644 --- a/src/main/native-chat/agent-session-journal/journal-input-card-index.ts +++ b/src/main/native-chat/agent-session-journal/journal-input-card-index.ts @@ -2,11 +2,13 @@ import type { AgentJournalCursor } from '../../../shared/agent-session/journal/a import type { JournalSubmissionRow } from './journal-row-schema' import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session/requests/agent-session-queued-submission' import type { JournalReducerState } from './journal-reducer' +import { journalInputOrder } from './journal-input-order' import type { QueuedMessageRow } from './queued-message-table' -import type { JournalInputPlace } from './journal-input-order' +import type { JournalInputPlace } from './journal-input-order-state' import { queuePauseHolding, journalQueueStopHolds, + failedQueueTurnSequence, type DerivedQueuePause } from './queued-message-pause' import { @@ -73,12 +75,18 @@ export class JournalInputCardIndex { private floor: { epoch: string; sequence: number } | null = null private pauses: readonly DerivedQueuePause[] = [] private cleared = new Set() + // A failure holds every waiting card alike, so it gates reads instead of the cached flags. + private failed = false constructor(private readonly lineage: JournalInputLineageOwner) {} hasOverlay(id: string): boolean { return this.overlays.has(id) } + /** Whether a failed turn holds waiting input, as of the last ensure. */ + failureHolds(): boolean { + return this.failed + } command(): QueuedMessageRow | undefined { - return this.currentCoordinates?.allPlaces().command + return this.failed ? undefined : this.currentCoordinates?.allPlaces().command } releaseHistory(): void { this.domains = undefined @@ -90,7 +98,9 @@ export class JournalInputCardIndex { return ( card.state === 'waiting' && card.holdReason === null && - (this.cleared.has(card.messageId) || Boolean(queuePauseHolding(this.pauses, card))) + (this.failed || + this.cleared.has(card.messageId) || + Boolean(queuePauseHolding(this.pauses, card))) ) } admission( @@ -121,18 +131,20 @@ export class JournalInputCardIndex { } queryCurrent(place: JournalInputPlace): InputPlaceQuery { - return queryInputContext( - place.context, - place.original, - (coordinate) => this.currentCoordinates!.query(coordinate.epoch, coordinate.sequence), - () => this.currentCoordinates!.allPlaces(), - mergeInputPlaces + return this.gate( + queryInputContext( + place.context, + place.original, + (coordinate) => this.currentCoordinates!.query(coordinate.epoch, coordinate.sequence), + () => this.currentCoordinates!.allPlaces(), + mergeInputPlaces + ) ) } query(place: JournalInputPlace): InputPlaceQuery { if (place.context.basis === 'observed') { - return this.index!.query('introduction', place.context.cut) + return this.gate(this.index!.query('introduction', place.context.cut)) } if (!this.domains) { this.domains = new JournalInputDomainIndex() @@ -143,7 +155,15 @@ export class JournalInputCardIndex { } } } - return this.domains.query(place.context, place.original) + return this.gate(this.domains.query(place.context, place.original)) + } + + private gate(result: InputPlaceQuery): InputPlaceQuery { + if (!this.failed) { + return result + } + const { head, handoff, position } = result + return { ...(head ? { head } : {}), ...(handoff ? { handoff } : {}), position } } ensure(state: JournalReducerState, floor?: { epoch: string; sequence: number } | null): void { @@ -152,6 +172,11 @@ export class JournalInputCardIndex { } floor = this.floor const marks = state.queuePauseMarks + this.failed = + failedQueueTurnSequence( + journalInputOrder(state).newestTurn.failure(), + marks.resumedSequence + ) > 0 const lifted = Math.max(state.latestAcceptedTurnSequence, marks.resumedSequence) const reopened = Math.max( marks.reopenedSequence, diff --git a/src/main/native-chat/agent-session-journal/journal-input-legacy-baseline.test.ts b/src/main/native-chat/agent-session-journal/journal-input-legacy-baseline.test.ts index 27295f0f988..9746dfe15fe 100644 --- a/src/main/native-chat/agent-session-journal/journal-input-legacy-baseline.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-input-legacy-baseline.test.ts @@ -19,6 +19,7 @@ import type { JournalRow } from './journal-row-schema' import type { JournalReducerState } from './journal-reducer' import type { AgentJournalMessageItem } from '../../../shared/agent-session/journal/agent-session-journal-types' import { holdUnsentSends } from './journal-unsent-send-hold' +import { agentSessionSendBodyFingerprint } from '../../../shared/agent-session/requests/structured-agent-session-send-mutation' import { createTrackedJournalOpener, loadTestJournal, @@ -295,3 +296,60 @@ it.each([10, 40])( } } ) + +// A failed import leaves an older build's card beside a canonical one at the same position: the +// introduction, then the id, decides which comes first, and a returned card stays a barrier. +it.each(['waiting', 'returned'] as const)( + 'keeps a tied %s card from a failed import ahead of a canonical card', + async (legacyState) => { + const root = await mkdtemp(join(tmpdir(), 'orca-legacy-tie-')) + const journals = createTrackedJournalOpener({ legacyInputs: true }) + const identity = { + sessionId: 'tie', + workspaceId: 'folder', + hostId: 'ssh', + agent: 'claude' as const, + providerHandle: claudeProviderHandle('thread', null) + } + const warned = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const body = message('message') + try { + let journal = await journals.open({ identity, stateDirectory: root }) + await journal.queuedMessages.insert({ messageId: 'z-canonical', body }) + await journal.close() + const { db } = openTestJournalHostDatabase(root) + db.prepare( + `INSERT INTO queued_messages (session_id, message_id, position, body_json, fingerprint, + created_at, host_instance, state, returned_reason) + VALUES (?, 'a-legacy', 1, ?, ?, 0, 'earlier', ?, ?)` + ).run( + identity.sessionId, + JSON.stringify(body), + agentSessionSendBodyFingerprint(identity.sessionId, body), + legacyState, + legacyState === 'returned' ? 'refused before start' : null + ) + db.exec(`CREATE TEMP TRIGGER fail_import BEFORE DELETE ON queued_messages + BEGIN SELECT RAISE(ABORT, 'disk full'); END`) + journal = await journals.open({ identity, stateDirectory: root }) + expect(warned).toHaveBeenCalled() + const state = journal.queuedMessages.inputState() + const owner = journalInputOrder(state) + const source = { epoch: state.epoch, sequence: state.lastSequence + 1 } + const observed = owner['cards'].query({ + coordinate: source, + context: { basis: 'observed', source, cut: owner.lineage.at(state.lastSequence) + 1 } + }) + expect(journal.queuedMessages.list().map((card) => card.position)).toEqual([1, 1]) + expect(owner.head(state, null)?.messageId).toBe('a-legacy') + expect(observed.head?.messageId).toBe('a-legacy') + expect(journal.queuedMessages.nextInputCard()?.messageId).toBe( + legacyState === 'waiting' ? 'a-legacy' : undefined + ) + } finally { + warned.mockRestore() + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } + } +) diff --git a/src/main/native-chat/agent-session-journal/journal-input-order.test.ts b/src/main/native-chat/agent-session-journal/journal-input-order.test.ts index bcc1c7a71b6..c2bd7b5f5c3 100644 --- a/src/main/native-chat/agent-session-journal/journal-input-order.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-input-order.test.ts @@ -564,3 +564,82 @@ it('retries partial keeps from committed facts and carries rank moves on the fir await rm(root, { recursive: true, force: true }) } }) + +// A sent card's released lineage leaves the history an observed send was placed behind +// once the queue next pauses, however many pauses came before that send. +it.each([0, 1])( + 'recovers an older direct send ahead of a newer card after a sent card settles, %s pauses before it', + async (pauses) => { + const root = await mkdtemp(join(tmpdir(), 'orca-settled-order-')) + const journals = createTrackedJournalOpener() + const journal = await journals.open({ + identity: { + sessionId: 'settled', + workspaceId: 'folder', + hostId: 'ssh', + agent: 'claude', + providerHandle: claudeProviderHandle('thread', null) + }, + stateDirectory: root + }) + try { + await journal.queuedMessages.insert({ + messageId: 'C', + body: { ...message('/compact'), command: { name: 'compact' } } + }) + await journal.queuedMessages.insert({ messageId: 'A', body: message('old waiting') }) + const sent = await journal.queuedMessages.insert({ messageId: 'S', body: message('sent') }) + await journal.appendSubmission( + { + clientMessageId: 'sent-S', + body: sent.body, + payloadFingerprint: sent.fingerprint, + fence: 0, + handoverRecorded: true, + origin: 'client' + }, + { messageId: 'S', expect: 'waiting' } + ) + await journal.resolveDispatch({ + clientMessageId: 'sent-S', + state: 'accepted', + providerIdentity: null, + fence: 0 + }) + expect(journal.inputOrder.lineage.cards.has('S')).toBe(false) + for (let n = 0; n < pauses; n++) { + await journal.appendStopEvent({ reason: 'user-stop' }, 0) + journal.queuedMessages.nextInputCard() + await journal.appendQueueResume(0) + journal.queuedMessages.nextInputCard() + } + await journal.appendSubmission({ + clientMessageId: 'M', + body: message('unhanded direct send'), + payloadFingerprint: 'M', + fence: 0, + handoverRecorded: true, + origin: 'client' + }) + expect(journal.inputOrder.forced('M')).toBe(true) + await journal.queuedMessages.insert({ messageId: 'L', body: message('newer card') }) + await journal.queuedMessages.move({ messageId: 'L', beforeMessageId: 'A' }) + await journal.queuedMessages.withdraw(['A']) + const position = () => + journal.inputOrder.predecessors(journal.queuedMessages.inputState(), 'M').position + expect(position()).toBe(3) + await journal.appendStopEvent({ reason: 'user-stop' }, 0) + expect(position()).toBe(1) + await holdUnsentSends(journal, { fence: 0, hold: { cause: 'chatClosed' } }) + expect( + journal.queuedMessages + .list() + .filter((card) => card.state === 'waiting') + .map((card) => card.messageId) + ).toEqual(['C', 'M', 'L']) + } finally { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } + } +) diff --git a/src/main/native-chat/agent-session-journal/journal-input-order.ts b/src/main/native-chat/agent-session-journal/journal-input-order.ts index a9805a85696..f3fa2769f5b 100644 --- a/src/main/native-chat/agent-session-journal/journal-input-order.ts +++ b/src/main/native-chat/agent-session-journal/journal-input-order.ts @@ -5,11 +5,12 @@ import { journalInputOrderChanged, isJournalInputOverride } from './journal-input-order-state' -export type { JournalInputPlace } from './journal-input-order-state' import { JournalSubmissionLinks } from './journal-submission-links' +import { JournalSubmissionIndex } from './journal-submission-index' import type { AgentJournalCursor } from '../../../shared/agent-session/journal/agent-session-journal-identity' import type { AgentJournalSubmission } from '../../../shared/agent-session/journal/agent-session-journal-types' +import { JournalNewestTurn } from './journal-newest-turn' import { parseAgentJournalItemKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key' import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session/requests/agent-session-queued-submission' import type { JournalReducerState } from './journal-reducer' @@ -35,18 +36,18 @@ export function setJournalInputOrder(state: JournalReducerState, owner: JournalI owners.set(state, owner) } -/** Own admission places and shared command witnesses are reconstructed by the journal fold. */ export class JournalInputOrder { readonly links = new JournalSubmissionLinks() private reconstructing = false private legacyPlaces = 0 private readonly recoveryOwed = new Set() private readonly originals = new Map() - private readonly unsettled = new Map() + private readonly unsettled = new JournalSubmissionIndex() + // Recovered unknown sends stay unsettled forever; only pending ones can still be accepted. + private readonly sending = new JournalSubmissionIndex() private readonly queued = new Map() private readonly entered = new Map() private readonly overrides = new Map() - private pending: readonly AgentJournalSubmission[] | undefined readonly lineage: JournalInputLineageOwner readonly cards: JournalInputCardIndex private receiptCount = 0 @@ -55,7 +56,8 @@ export class JournalInputOrder { constructor( private readonly state: JournalReducerState, - private readonly originalAcceptance?: (id: string) => AgentJournalCursor | undefined + private readonly originalAcceptance?: (id: string) => AgentJournalCursor | undefined, + readonly newestTurn = new JournalNewestTurn(state) ) { this.lineage = new JournalInputLineageOwner(state.inputCards) this.cards = new JournalInputCardIndex(this.lineage) @@ -175,7 +177,10 @@ export class JournalInputOrder { return this.entered.values() } unsettledSubmissions(): readonly AgentJournalSubmission[] { - return (this.pending ??= [...this.unsettled.values()]) + return this.unsettled.list() + } + sendingSubmissions(): readonly AgentJournalSubmission[] { + return this.sending.list() } version(): number { return this.revision @@ -218,6 +223,7 @@ export class JournalInputOrder { } afterRow(state: JournalReducerState, row: JournalRow): void { + this.newestTurn.afterRow(state, row) if (journalInputOrderChanged(state, row, (key) => this[key])) { this.revision++ } @@ -249,17 +255,12 @@ export class JournalInputOrder { private refresh(state: JournalReducerState, id: string): void { const entry = state.submissions.get(id) this.links.refresh(id, entry, state.items) - if (entry?.dispatchState === 'pending' || entry?.dispatchState === 'unknown') { - if (this.unsettled.get(id) !== entry) { - this.unsettled.set(id, entry) - this.pending = undefined - this.revision++ - } - } else if (this.unsettled.delete(id)) { - this.pending = undefined + const dispatch = entry?.dispatchState + if (this.unsettled.keep(id, entry, dispatch === 'pending' || dispatch === 'unknown')) { this.revision++ } - if (entry?.dispatchState !== 'pending') { + this.sending.keep(id, entry, dispatch === 'pending') + if (dispatch !== 'pending') { this.releasePlace(state, id) } if (entry && isQueuedAgentJournalSubmission(entry)) { diff --git a/src/main/native-chat/agent-session-journal/journal-newest-turn-tree.ts b/src/main/native-chat/agent-session-journal/journal-newest-turn-tree.ts new file mode 100644 index 00000000000..70d78ba2d89 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-newest-turn-tree.ts @@ -0,0 +1,96 @@ +import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types' +import { compareAgentJournalItems } from '../../../shared/agent-session/journal/agent-session-journal-position' + +/** A turn or context clear, with the row where it first became a holding failure (zero while it is + * not one). A command that failed on its own terms is not `relevant`: it neither holds nor releases. */ +export type TurnSource = { + item: AgentJournalRenderItem + failedAt: number + ordinary: boolean + relevant: boolean +} +/** Creation-ordered; each subtree caches its newest source, ordinary turn or clear, and relevant source. */ +export type TurnSourceNode = { + source: TurnSource + height: number + latest: TurnSource + anchor: TurnSource | undefined + winner: TurnSource | undefined + left?: TurnSourceNode + right?: TurnSourceNode +} + +export const compareTurnSources = (a: TurnSource, b: TurnSource): number => + compareAgentJournalItems(a.item, b.item) || a.item.itemId.localeCompare(b.item.itemId) +const height = (node?: TurnSourceNode) => node?.height ?? 0 +function update(node: TurnSourceNode): TurnSourceNode { + const { source, left, right } = node + node.height = 1 + Math.max(height(left), height(right)) + node.latest = right?.latest ?? source + node.anchor = right?.anchor ?? (source.ordinary ? source : left?.anchor) + node.winner = right?.winner ?? (source.relevant ? source : left?.winner) + return node +} +function rotateLeft(node: TurnSourceNode): TurnSourceNode { + const next = node.right! + node.right = next.left + next.left = update(node) + return update(next) +} +function rotateRight(node: TurnSourceNode): TurnSourceNode { + const next = node.left! + node.left = next.right + next.right = update(node) + return update(next) +} +function balance(node: TurnSourceNode): TurnSourceNode { + update(node) + if (height(node.left) - height(node.right) > 1) { + if (height(node.left!.right) > height(node.left!.left)) { + node.left = rotateLeft(node.left!) + } + return rotateRight(node) + } + if (height(node.right) - height(node.left) > 1) { + if (height(node.right!.left) > height(node.right!.right)) { + node.right = rotateRight(node.right!) + } + return rotateLeft(node) + } + return node +} + +/** Inserts, replaces (same position) or removes one source along a single root-to-leaf path. */ +export function putTurnSource( + node: TurnSourceNode | undefined, + source: TurnSource, + remove: boolean +): TurnSourceNode | undefined { + if (!node) { + return remove + ? undefined + : update({ source, height: 1, latest: source, anchor: undefined, winner: undefined }) + } + const order = compareTurnSources(source, node.source) + if (order < 0) { + node.left = putTurnSource(node.left, source, remove) + } else if (order > 0) { + node.right = putTurnSource(node.right, source, remove) + } else if (!remove) { + node.source = source + } else { + if (!node.left) { + return node.right + } + if (!node.right) { + return node.left + } + let next = node.right + while (next.left) { + next = next.left + } + node.source = next.source + node.right = putTurnSource(node.right, next.source, true) + } + return balance(node) +} diff --git a/src/main/native-chat/agent-session-journal/journal-newest-turn.ts b/src/main/native-chat/agent-session-journal/journal-newest-turn.ts new file mode 100644 index 00000000000..5e88af33c2c --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-newest-turn.ts @@ -0,0 +1,190 @@ +import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types' +import { readAgentJournalTurn } from '../../../shared/agent-session/lifecycle/agent-session-turn-record' +import { isAgentSessionContextClear } from '../../../shared/agent-session/agent-session-context-clear' +import { parseAgentJournalItemKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key' +import { isRootAgentJournalItem } from '../../../shared/agent-session/journal/agent-session-journal-producer' +import { isStructuredAgentSessionCommandTurnId } from '../agent-session-wire/structured-agent-session-command-turn' +import type { JournalReducerState } from './journal-reducer' +import type { JournalRow } from './journal-row-schema' +import { + compareTurnSources, + putTurnSource, + type TurnSource, + type TurnSourceNode +} from './journal-newest-turn-tree' + +/** Owned by the transaction-isolated input order; never saved beside the journal. Every turn and + * clear sits in a creation-ordered tree, so each row updates only the sources it touches. */ +export class JournalNewestTurn { + private root: TurnSourceNode | undefined + private readonly sources = new Map() + /** Agent failure rows (usage limit, sign-in, terminal error) and the turn each reports on, + * indexed whether or not that turn has arrived yet. */ + private readonly evidence = new Map() + private readonly reports = new Map() + /** Settlement order at or before this row is unknown (restore, or removal of a source that could + * decide), so a failure seen earlier holds until a Resume written after it. */ + private boundary: number + + /** Only on restore; streamed rows never rescan conversation history. */ + constructor(state: JournalReducerState) { + this.boundary = state.lastSequence + const turns: AgentJournalRenderItem[] = [] + const reported = new Set() + for (const item of state.items.values()) { + this.index(state, item.itemId, item, reported) + if (isTurnOrClear(item)) { + turns.push(item) + } + } + for (const item of turns) { + this.place(item, state.lastSequence) + } + } + + read(): AgentJournalRenderItem | null { + return this.root?.latest.item ?? null + } + + /** Releasing Resume marks must be written after this position. */ + failure(): number { + const failedAt = this.root?.winner?.failedAt ?? 0 + return failedAt && Math.max(failedAt, this.boundary) + } + + afterRow(state: JournalReducerState, row: JournalRow): void { + const itemIds = + row.kind === 'item' || row.kind === 'tombstone' + ? [row.itemId] + : row.kind === 'lifecycle-batch' + ? row.mutations.map((mutation) => mutation.itemId) + : [] + const touched = new Set() + for (const raw of itemIds) { + const itemId = state.aliases.get(raw) ?? raw + const item = state.items.get(itemId) + const prior = this.sources.get(itemId) + if (prior && !isTurnOrClear(item)) { + // Removing one at or after the newest ordinary turn or clear exposes what lies beneath. + const anchor = this.root?.anchor + if (!anchor || compareTurnSources(prior, anchor) >= 0) { + this.boundary = row.seq + } + this.sources.delete(itemId) + this.root = putTurnSource(this.root, prior, true) + } + this.index(state, itemId, item, touched) + if (isTurnOrClear(item)) { + touched.add(itemId) + } + } + for (const itemId of touched) { + this.place(state.items.get(itemId), row.seq) + } + } + + /** Keeps the first failure position across same-failure revisions; leaving failure clears it. */ + private place(item: AgentJournalRenderItem | undefined, at: number): void { + if (!item || !isTurnOrClear(item)) { + return + } + const prior = this.sources.get(item.itemId) + const command = isCommand(item) + const failed = settledFailure(item) + const reported = (this.reports.get(item.itemId) ?? 0) > 0 + const source: TurnSource = { + item, + failedAt: failed && (!command || reported) ? prior?.failedAt || at : 0, + ordinary: !command, + relevant: !command || !failed || reported + } + if ( + prior?.item === item && + prior.failedAt === source.failedAt && + prior.ordinary === source.ordinary && + prior.relevant === source.relevant + ) { + return + } + if (prior && compareTurnSources(prior, source) !== 0) { + this.root = putTurnSource(this.root, prior, true) + } + this.sources.set(item.itemId, source) + this.root = putTurnSource(this.root, source, false) + } + + private index( + state: JournalReducerState, + itemId: string, + item: AgentJournalRenderItem | undefined, + touched: Set + ): void { + const turn = item ? reportedTurn(state, item) : null + const prior = this.evidence.get(itemId) ?? null + if (turn === prior) { + return + } + for (const [target, step] of [ + [prior, -1], + [turn, 1] + ] as const) { + if (target !== null) { + const count = (this.reports.get(target) ?? 0) + step + if (count > 0) { + this.reports.set(target, count) + } else { + this.reports.delete(target) + } + touched.add(target) + } + } + if (turn === null) { + this.evidence.delete(itemId) + } else { + this.evidence.set(itemId, turn) + } + } +} + +function isTurnOrClear(item: AgentJournalRenderItem | undefined): boolean { + return !!item && (!!readAgentJournalTurn(item.body) || isAgentSessionContextClear(item.body)) +} + +function settledFailure(item: AgentJournalRenderItem): boolean { + const turn = readAgentJournalTurn(item.body) + return turn?.state !== 'running' && turn?.outcome === 'failure' +} + +/** A command turn holds the queue only with an agent failure row in its turn. */ +function isCommand(item: AgentJournalRenderItem): boolean { + const turnId = readAgentJournalTurn(item.body)?.turnId + return !!turnId && isStructuredAgentSessionCommandTurnId(turnId) +} + +/** A command's own verdict, whichever writer saved it, and a retry still under way are not the + * agent reporting a failure. */ +const NOT_REPORTS: ReadonlySet = new Set([ + 'compactionFailed', + 'compactionUnconfirmed', + 'commandRefused', + 'providerRetrying' +]) + +/** The turn a root failure row reports for, unless it is a command's own verdict: one of the kinds + * above, or anything Orca wrote on a command's result identity (its refusals). */ +function reportedTurn(state: JournalReducerState, item: AgentJournalRenderItem): string | null { + const { body, turnScope } = item + if ( + body.kind !== 'status' || + turnScope?.kind !== 'turn' || + !isRootAgentJournalItem(item) || + !(body.failure ? !NOT_REPORTS.has(body.failure.kind) : body.tone === 'error') + ) { + return null + } + const identity = parseAgentJournalItemKey(item.itemId) + if (identity?.provider === 'orca' && identity.clientMessageId.startsWith('command-result:')) { + return null + } + return state.aliases.get(turnScope.turnItemId) ?? turnScope.turnItemId +} diff --git a/src/main/native-chat/agent-session-journal/journal-queued-messages.ts b/src/main/native-chat/agent-session-journal/journal-queued-messages.ts index fd6bd314ea9..c75f9b3617c 100644 --- a/src/main/native-chat/agent-session-journal/journal-queued-messages.ts +++ b/src/main/native-chat/agent-session-journal/journal-queued-messages.ts @@ -298,6 +298,7 @@ export class JournalQueuedMessages { epoch: state.epoch, marks: state.queuePauseMarks, latestAcceptedTurnSequence: state.latestAcceptedTurnSequence, + failedAt: journalInputOrder(state).newestTurn.failure(), cards, reopenFloor: this.deps.reopenFloor() }) diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.test.ts b/src/main/native-chat/agent-session-journal/journal-reducer.test.ts index 6acc824a4dd..f12387cdf9e 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.test.ts @@ -60,6 +60,17 @@ function fold(rows: JournalRow[]): JournalReducerState { .map((entry) => entry.clientMessageId) .sort() ) + expect( + journalInputOrder(state) + .sendingSubmissions() + .map((entry) => entry.clientMessageId) + .sort() + ).toEqual( + [...state.submissions.values()] + .filter((entry) => entry.dispatchState === 'pending') + .map((entry) => entry.clientMessageId) + .sort() + ) for (const entry of state.submissions.values()) { if (entry.dispatchState !== 'pending') { expect(journalInputOrder(state).forced(entry.clientMessageId)).toBe(false) diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 7784c02efb8..eb1c2fe64b4 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -22,10 +22,8 @@ import type { import { agentJournalItemKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key' import type { AgentSessionContextUsage } from '../../../shared/agent-session/agent-session-context-usage' import { latestStructuredAgentContextFacts } from '../../../shared/agent-session/structured-agent-session-context-usage' -import { - newestStructuredAgentSessionTurnBySequence, - runningStructuredAgentSessionTurnItemBySequence -} from '../../../shared/agent-session/view/structured-agent-session-live-turn' +import { runningStructuredAgentSessionTurnItemBySequence } from '../../../shared/agent-session/view/structured-agent-session-live-turn' +import { readAgentJournalTurn } from '../../../shared/agent-session/lifecycle/agent-session-turn-record' import { journalWroteBeforeOpen, readJournalSince } from './journal-cursor' import type { JournalHostDatabase } from './journal-host-database' import { journalRowsAfterReader } from './journal-open' @@ -228,7 +226,7 @@ export class AgentSessionJournal { runningStructuredAgentSessionTurnItemBySequence(this.state.items.values()) newestTurn = (): AgentJournalTurnLifecycle | null => - newestStructuredAgentSessionTurnBySequence(this.state.items.values()) + readAgentJournalTurn(journalInputOrder(this.state).newestTurn.read()?.body) threadGoalSource = (): JournalThreadGoalSource => journalThreadGoalSource(this.state.items.values()) diff --git a/src/main/native-chat/agent-session-journal/journal-submission-index.ts b/src/main/native-chat/agent-session-journal/journal-submission-index.ts new file mode 100644 index 00000000000..7f4737d7787 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-submission-index.ts @@ -0,0 +1,27 @@ +import type { AgentJournalSubmission } from '../../../shared/agent-session/journal/agent-session-journal-types' + +/** Submissions in one dispatch phase, listed without walking the rest of history. */ +export class JournalSubmissionIndex { + private readonly entries = new Map() + private listed: readonly AgentJournalSubmission[] | undefined + + /** Returns whether membership or the indexed entry changed. */ + keep(id: string, entry: AgentJournalSubmission | undefined, member: boolean): boolean { + if (entry && member) { + if (this.entries.get(id) === entry) { + return false + } + this.entries.set(id, entry) + } else if (!this.entries.delete(id)) { + return false + } + this.listed = undefined + return true + } + values(): Iterable { + return this.entries.values() + } + list(): readonly AgentJournalSubmission[] { + return (this.listed ??= [...this.entries.values()]) + } +} diff --git a/src/main/native-chat/agent-session-journal/queued-message-pause.ts b/src/main/native-chat/agent-session-journal/queued-message-pause.ts index 661804b6d2e..5fcb972ecc5 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-pause.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-pause.ts @@ -1,11 +1,11 @@ -// Derive pauses from journal state: personal Stop, cleared IDs and cards predating reopen. +// Derive pauses from journal state: failed turns, personal Stop, clear and reopen. // Later accepted turns/Resume lift pauses; per-card holds still require their own action. import type { AgentJournalCursor } from '../../../shared/agent-session/journal/agent-session-journal-identity' import type { AgentJournalSubmission } from '../../../shared/agent-session/journal/agent-session-journal-types' import type { JournalStopEvent, JournalTombstoneRow } from './journal-row-schema' -export type QueuePauseReason = 'stopped' | 'cleared' | 'restarted' +export type QueuePauseReason = 'stopped' | 'failed' | 'cleared' | 'restarted' /** Memory-only unnamed Stop binding; failedOn affects display, never cancellation ownership. */ export type JournalStopSettle = { @@ -110,6 +110,7 @@ export function journalUserStopInForce( /** Restate clear-lift first, then active Stop/reopen marks so the lift cannot erase them. */ export type JournalQueuePauseRestatement = { lifted: boolean + failureHeld?: boolean liveStop: JournalStopEvent | null reopened: boolean cleared?: { operationId: string; messageIds: string[]; lifted?: true } @@ -122,6 +123,7 @@ export function journalQueuePauseRestatement( ): JournalQueuePauseRestatement { return { lifted: latestAcceptedTurnSequence > 0 || marks.resumedSequence > 0, + failureHeld: pauses.some((pause) => pause.reason === 'failed'), liveStop: journalUserStopInForce(marks, latestAcceptedTurnSequence)?.event ?? null, reopened: pauses.some((pause) => pause.reason === 'restarted'), ...(marks.cleared @@ -139,12 +141,20 @@ export function journalQueuePauseRestatement( } } +/** A steer joins the same turn, and a Resume written before the failure cannot waive it. A rebuilt + * failure sits at the rebuild point, so a Resume at that point was written before it too. */ +export function failedQueueTurnSequence(failedAt: number, resumedSequence: number): number { + return failedAt > 0 && failedAt >= resumedSequence ? failedAt : 0 +} + /** Order determines which reason a card held by multiple pauses displays. */ export function deriveQueuePauses(input: { /** Sequences compare only within the same epoch. */ epoch: string marks: JournalQueuePauseMarks latestAcceptedTurnSequence: number + /** Where the newest turn settled as failed; zero when it has not. */ + failedAt?: number cards: readonly QueueCard[] /** Conservative boundary when this handle could not write a reopen mark. */ reopenFloor: AgentJournalCursor | null @@ -155,6 +165,10 @@ export function deriveQueuePauses(input: { if (stop) { pauses.push({ reason: 'stopped', since: { epoch, sequence: stop.sequence } }) } + const failed = failedQueueTurnSequence(input.failedAt ?? 0, marks.resumedSequence) + if (failed) { + pauses.push({ reason: 'failed', since: { epoch, sequence: failed } }) + } const waiting = input.cards.filter((card) => card.state === 'waiting') const cleared = marks.cleared if ( @@ -200,6 +214,9 @@ function reopenPause(input: { } function queuedBefore(pause: DerivedQueuePause, card: QueueCard): boolean { + if (pause.reason === 'failed') { + return true + } if (pause.reason === 'cleared') { return pause.messageIds?.includes(card.messageId ?? '') ?? false } diff --git a/src/main/native-chat/agent-session-journal/queued-message-stop-pause.spec.ts b/src/main/native-chat/agent-session-journal/queued-message-stop-pause.spec.ts index 82a13e35ff2..e5ed8a16dff 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-stop-pause.spec.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-stop-pause.spec.ts @@ -7,11 +7,62 @@ import { QueuedMessageNotConsumableError } from './queued-message-consume-error' import { applyJournalRow, createJournalReducerState } from './journal-reducer' import { parseJournalRow } from './journal-row-schema' import type { AgentSessionJournal } from './journal-store' +import { structuredAgentSessionCommandTurn } from '../../../shared/agent-session/view/structured-agent-session-command-turn-identity' export function registerQueuedMessageStopPauseTests( fixture: ReturnType ): void { describe("the queue's pause, derived from the journal", () => { + it.each([false, true])( + 'preserves failure Resume state across a forked rewind (resumed=%s)', + async (resumed) => { + let journal = await fixture.open() + await fixture.turn(journal, 'accepted') + const identity = { provider: 'claude', sessionId: 'native-1', uuid: 'failed-turn' } as const + const body = { + kind: 'turn', + turnId: 'failed-turn', + state: 'completed', + outcome: 'failure' + } as const + await journal.appendItem(identity, body, { fence: 0, turnScope: { kind: 'thread' } }) + // Two /compact turns refused on their own terms after it: see-through, never a release. + const refused = ['c1', 'c2'].map((id) => { + const command = structuredAgentSessionCommandTurn(id) + return { identity: command.identity, body: { ...body, turnId: command.turnId } } + }) + for (const command of refused) { + await journal.appendItem(command.identity, command.body, { + fence: 0, + turnScope: { kind: 'thread' } + }) + } + await fixture.queueDraft(journal, 'carried') + if (resumed) { + await journal.appendQueueResume(0) + } + expect(fixture.reason(journal)).toBe(resumed ? null : 'failed') + await journal.replaceEpochItems('handle_forked', 0, [ + { identity, body, ownerFence: 0 }, + ...refused.map((command) => ({ ...command, ownerFence: 0 })) + ]) + expect(fixture.reason(journal)).toBe(resumed ? null : 'failed') + expect(journal.queuedMessages.nextInputCard()?.messageId ?? null).toBe( + resumed ? 'carried' : null + ) + await journal.close() + journal = await fixture.open() + // Settlement order is not saved, so a reopened failure holds until a Resume written after it. + const failed = () => journal.queuedMessages.pauses().some((p) => p.reason === 'failed') + expect(failed()).toBe(true) + expect(journal.queuedMessages.list()).toMatchObject([ + { messageId: 'carried', state: 'waiting' } + ]) + await journal.appendQueueResume(0) + expect(failed()).toBe(false) + } + ) + it("a person's Stop event pauses the queue, even with no card yet, survives reopen, and stores nothing", async () => { let journal = await fixture.open() await fixture.userStop(journal) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-action-receipts.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-action-receipts.test.ts index fc629725489..32a4e41e843 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-action-receipts.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-action-receipts.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { agentSessionFailureWords } from '../../../shared/agent-session/failures/agent-session-failure-words' import { agentJournalItemKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key' -import { holdLane } from './structured-agent-session-delivery-hold.test-fixture' +import { openRigTurnFor } from './structured-agent-session-queued-rig-turn.test-fixture' import { AGENT_SESSION_MAX_NEW_OPERATION_AGE_MS } from '../../../shared/agent-session/agent-session-host-authority' import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { @@ -596,36 +596,42 @@ describe('source-bound ordinary Continue', () => { expect(journal.submissions()).toHaveLength(2) }) - it('refuses a stale action and actionable backlog before writing a submission or queue card', async () => { - const { journal, turnItemId } = await limitedTurn('old') - await rig.send('ordinary message wins').result - const before = journal.snapshot().items.map((item) => item.itemId) - const requests = journal.submissions().map((entry) => entry.clientMessageId) - expect(await rig.host.send(CALLER, params(turnItemId))).toMatchObject(superseded) - expect(journal.snapshot().items.map((item) => item.itemId)).toEqual(before) - expect(journal.submissions().map((entry) => entry.clientMessageId)).toEqual(requests) - expect(journal.queuedMessages.list()).toHaveLength(0) - await rig.settleRejected(journal.submissions()[0]!.clientMessageId, 'done') - const current = await limitedTurn('current') - const release = holdLane(rig.host, SESSION) - try { + it.each(['success', 'failure'] as const)( + 'refuses a stale action but runs ahead of failure-held backlog, which drains only on success: %s', + async (outcome) => { + const { journal, turnItemId } = await limitedTurn('old') + await rig.send('ordinary message wins').result + const before = journal.snapshot().items.map((item) => item.itemId) + const requests = journal.submissions().map((entry) => entry.clientMessageId) + expect(await rig.host.send(CALLER, params(turnItemId))).toMatchObject(superseded) + expect(journal.snapshot().items.map((item) => item.itemId)).toEqual(before) + expect(journal.submissions().map((entry) => entry.clientMessageId)).toEqual(requests) + expect(journal.queuedMessages.list()).toHaveLength(0) + await rig.settleRejected(journal.submissions()[0]!.clientMessageId, 'done') + const current = await limitedTurn('current') const row = await journal.queuedMessages.insert({ messageId: opId(), body: hostTestMessage('backlog') }) - journal.queuedMessages.editLeases.acquire( - { callerKey: CALLER.callerKey, editId: 'edit' }, - { messageId: row.messageId, fingerprint: row.fingerprint } - ) - const attempt = rig.host.send(CALLER, params(current.turnItemId)) - release() - expect(await attempt).toMatchObject(superseded) - expect(journal.queuedMessages.list()).toHaveLength(1) - expect(journal.submissions()).toHaveLength(1) - } finally { - release() + expect(journal.queuedMessages.pauses()).toMatchObject([{ reason: 'failed' }]) + // Continue is the person's recovery, so the held card waits behind it like any other hold. + expect(await rig.host.send(CALLER, params(current.turnItemId))).toMatchObject({ + ok: true, + replayed: false + }) + const continuation = journal.submissions().at(-1)!.clientMessageId + expect(await rig.handoff(row.messageId)).toBeUndefined() + await openRigTurnFor(rig, continuation) + await rig.settleAccepted(continuation, 'continued') + await openRigTurnFor(rig, continuation, 'completed', outcome) + if (outcome === 'success') { + await eventually(async () => expect(await rig.handoff(row.messageId)).toBeDefined()) + } else { + expect(journal.queuedMessages.pauses()).toMatchObject([{ reason: 'failed' }]) + expect(await rig.handoff(row.messageId)).toBeUndefined() + } } - }) + ) }) it.each(['send-now', 'automatic'] as const)( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-compact.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-compact.test.ts index a4cdb4d8327..8ba9bf9a35c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-compact.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-compact.test.ts @@ -1,8 +1,17 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key' +import { + agentJournalItemKey, + agentJournalSubmissionKey +} from '../../../shared/agent-session/journal/agent-session-journal-item-key' import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session/journal/agent-session-journal-types' import type { AgentChildWorkView } from '../../../shared/agent-status-child-work-view' import { reportedChildWork } from './structured-agent-session-reported-child-work.test-support' +import { createClaudeJournalTranslator } from '../../claude/structured/journal/claude-structured-journal-translation' +import { + CAPTURED_COMPACT_SESSION_ID, + CAPTURED_COMPACT_STOPPED_THEN_SEND +} from '../../claude/claude-captured-compact-frames.test-fixture' +import { openRigTurnFor } from './structured-agent-session-queued-rig-turn.test-fixture' import { agentSessionFailureFact } from '../../../shared/agent-session/failures/agent-session-failure' import { agentSessionFailureWords } from '../../../shared/agent-session/failures/agent-session-failure-words' import { ConversationCommandParams } from '../../../shared/rpc-contract/structured-agent-session-params' @@ -540,6 +549,136 @@ describe('a /compact that waits in line', () => { }) }) +describe('a queued /compact that fails', () => { + async function compactThenCard() { + const working = await rig.workingSend() + const compactId = await queuedCompact() + const later = await rig.send('sent after the compact', 'queue-if-active').result + if (!later.ok || !('queued' in later.value)) { + throw new Error('expected a queued receipt') + } + return { working, compactId, laterId: later.value.queued.messageId } + } + + async function handedOver() { + await eventually(() => expect(rig.compact).toHaveBeenCalledOnce()) + const { command } = rig.compact.mock.calls.at(-1)![0] + return { ...command, itemId: agentJournalItemKey(command.identity) } + } + + async function held(laterId: string) { + await settleMs() + expect(await rig.handoff(laterId)).toBeUndefined() + const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + expect(journal.queuedMessages.pauses()).toContainEqual( + expect.objectContaining({ reason: 'failed' }) + ) + } + + it("lets the next card run after Claude's recorded failed compaction", async () => { + const { working, laterId } = await compactThenCard() + await rig.settleAccepted(working, 'a') + const command = await handedOver() + const sink = rig.providerEvents() + const translator = createClaudeJournalTranslator({ sink }) + const capture = CAPTURED_COMPACT_STOPPED_THEN_SEND.filter((event) => !('interrupt' in event)) + const end = capture.findIndex((event) => 'frame' in event && event.frame.type === 'result') + for (const event of capture.slice(0, end + 1)) { + if ('sent' in event) { + const start = { providerSessionId: CAPTURED_COMPACT_SESSION_ID, sentUuid: event.sent.uuid } + translator.beginCommand({ ...command, ...start }) + } else if ('frame' in event) { + translator.handle({ + type: 'message', + sessionId: SESSION, + observedAt: event.at, + message: { session_id: CAPTURED_COMPACT_SESSION_ID, ...event.frame } + }) + } + } + translator.dispose() + await sink.written?.() + await eventually(async () => expect(await rig.handoff(laterId)).toBeDefined()) + const { items } = await rig.host.journalSnapshot(SESSION) + const turn = items.find((item) => item.itemId === command.itemId) + expect(turn?.body).toMatchObject({ state: 'completed', outcome: 'failure' }) + expect(items).toContainEqual( + expect.objectContaining({ body: expect.objectContaining({ tone: 'error' }) }) + ) + }) + + it('keeps an older failure holding the other card when Send runs the /compact into a refusal', async () => { + rig.compact.mockResolvedValue({ + state: 'rejected', + ...agentSessionFailureWords( + agentSessionFailureFact('providerRejected', { + detail: { text: 'Not enough messages to compact.', audience: 'person' } + }), + { surface: 'rejection' } + ) + }) + const { working, compactId, laterId } = await compactThenCard() + await openRigTurnFor(rig, working) + await rig.settleAccepted(working, 'a') + await openRigTurnFor(rig, working, 'completed', 'failure') + await held(laterId) + expect(await rig.sendNow(compactId)).toMatchObject({ ok: true }) + await eventually(async () => + expect(await rig.handoff(compactId)).toMatchObject({ dispatchState: 'rejected' }) + ) + await held(laterId) + await rig.restartHostProcess() + await held(laterId) + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + await eventually(async () => expect(await rig.handoff(laterId)).toBeDefined()) + }) + + it.each(['before', 'after'] as const)( + 'holds when a usage limit lands %s the compaction ends, across a restart', + async (order) => { + const { working, laterId } = await compactThenCard() + await rig.settleAccepted(working, 'a') + const command = await handedOver() + const events = rig.providerEvents() + const limit = () => + events.appendItem( + { provider: 'orca', clientMessageId: `terminal-failure:claude:${command.itemId}` }, + { + kind: 'status', + tone: 'warning', + ...agentSessionFailureWords(agentSessionFailureFact('usageLimitReached'), { + surface: 'row' + }) + }, + { turnScope: { kind: 'turn', turnItemId: command.itemId } } + ) + if (order === 'before') { + limit() + } + events.appendLifecycleBatch!( + `turn-completed:${command.clientMessageId}`, + [ + { + kind: 'item', + identity: command.identity, + body: { ...command.running, state: 'completed', outcome: 'failure', completedAt: 2 }, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + } + ], + { lifecycle: true } + ) + if (order === 'after') { + limit() + } + await held(laterId) + await rig.restartHostProcess() + await held(laterId) + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + await eventually(async () => expect(await rig.handoff(laterId)).toBeDefined()) + } + ) +}) + describe('/compact instructions', () => { const INSTRUCTIONS = 'keep the open decisions\nand the plan' function instructedCompact(delivery?: 'queue-if-active') { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-failure.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-failure.test.ts new file mode 100644 index 00000000000..74d50372bd5 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-failure.test.ts @@ -0,0 +1,341 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { readFile } from 'node:fs/promises' +import { z } from 'zod' +import { createClaudeJournalTranslator } from '../../claude/structured/journal/claude-structured-journal-translation' +import { createCodexJournalTranslator } from '../../codex/codex-structured-journal-translation' +import { readCodexThreadId } from '../../codex/codex-structured-thread-facts' +import { + createQueuedMessageTestRig, + eventually, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { openRigTurnFor } from './structured-agent-session-queued-rig-turn.test-fixture' +import { + HOST_TEST_SESSION as SESSION, + hostTestMessage, + hostTestOperationId +} from './structured-agent-session-host-test-data' +import { sendAgentTurn } from '../../runtime/orchestration/send-agent-turn' +import { structuredSessionPointerCallerKey } from '../../runtime/orchestration/structured-mailbox-pointer-host' +import { createStructuredSessionObservationClock } from './structured-agent-session-observation-clock.test-fixture' + +let rig: QueuedMessageTestRig +const clock = createStructuredSessionObservationClock(() => rig.host, SESSION) +beforeEach(async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + clock.start() +}) +afterEach(() => clock.restoreBeforeCleanup(() => rig.dispose())) + +async function queue(text: string): Promise { + const result = await rig.send(text, 'queue-if-active').result + if (!result.ok || !('queued' in result.value)) { + throw new Error('expected a queued message') + } + return result.value.queued.messageId +} + +/** An orchestration message, sent the way the mailbox host sends one. */ +function mail(text: string) { + const id = hostTestOperationId() + const outcome = sendAgentTurn({ + kind: 'structured-session', + host: rig.host, + sessionId: SESSION, + callerKey: structuredSessionPointerCallerKey(SESSION), + turn: { + body: { ...hostTestMessage(text), from: { kind: 'agent', senders: [], orchestration: null } }, + delivery: 'queue', + operationId: id, + expectedRuntimeFence: 1, + beforeRun: () => {} + } + }) + return { id, outcome } +} + +async function start(): Promise { + const id = await rig.workingSend() + await openRigTurnFor(rig, id) + await rig.settleAccepted(id, id) + return id +} + +async function held(...ids: string[]): Promise { + await clock.observe(250) + expect(await rig.drafts()).toEqual(ids.map((messageId) => ({ messageId, state: 'waiting' }))) + for (const id of ids) { + expect(await rig.handoff(id)).toBeUndefined() + } + expect(await rig.queuePause()).toEqual({ reason: 'failed' }) + const page = await rig.host.history({ sessionId: SESSION, direction: 'tail' }) + expect(page).toMatchObject({ ok: true, page: { nextQueuedMessageId: null } }) +} + +async function dispatched(card: string): Promise { + await eventually(async () => expect((await rig.handoff(card))?.handedOverAt).toBeDefined()) + return rig.handoffId(card) +} + +describe('queued follow-ups after an accepted turn fails', () => { + it.each(['claude', 'codex'] as const)( + 'holds after recorded real %s failure output', + async (agent) => { + const id = await rig.workingSend() + const sink = rig.providerEvents() + let cards: [string, string] | undefined + const queueDuringTurn = async () => { + await sink.written?.() + const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + if (!cards && journal.newestTurn()?.state === 'running') { + await rig.settleAccepted(id, id) + cards = [ + await queue('first recorded follow-up'), + await queue('second recorded follow-up') + ] + } + } + if (agent === 'claude') { + const translator = createClaudeJournalTranslator({ sink }) + const capture = await readFile( + new URL( + '../../claude/__fixtures__/claude-lifecycle-capture-auth-failed.jsonl', + import.meta.url + ), + 'utf8' + ) + const frame = z.object({ at: z.number(), frame: z.record(z.string(), z.unknown()) }) + for (const line of capture.trim().split('\n')) { + const value: unknown = JSON.parse(line) + if (z.looseObject({ kind: z.string() }).parse(value).kind !== 'frame') { + continue + } + const entry = frame.parse(value) + translator.handle({ + type: 'message', + sessionId: SESSION, + observedAt: entry.at, + message: entry.frame + }) + await queueDuringTurn() + } + translator.dispose() + } else { + const capture = await readFile( + new URL('../../codex/__fixtures__/codex-app-server-turn-endings.jsonl', import.meta.url), + 'utf8' + ) + const frame = z.object({ + case: z.string(), + t: z.number(), + method: z.string(), + params: z.record(z.string(), z.unknown()) + }) + const frames = capture + .trim() + .split('\n') + .map((line) => frame.parse(JSON.parse(line))) + .filter((entry) => entry.case === '0.158.0-overloaded-503') + expect(frames.length).toBeGreaterThan(0) + const threadId = readCodexThreadId(frames[0]?.params) ?? '' + const translator = createCodexJournalTranslator({ + sink, + sessionId: SESSION, + primaryThreadId: () => threadId + }) + for (const entry of frames) { + translator.handle({ + type: 'notification', + sessionId: SESSION, + threadId, + method: entry.method, + params: entry.params, + observedAt: entry.t + }) + await queueDuringTurn() + } + } + await sink.written?.() + await rig.host.flushStreamedEvents(SESSION) + if (!cards) { + throw new Error('capture did not open a turn') + } + const [first, second] = cards + await held(first, second) + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + await dispatched(first) + expect(await rig.handoff(second)).toBeUndefined() + } + ) + + it('holds every mid-turn card across repeated wakes, then resumes one turn at a time', async () => { + const working = await start() + const first = await queue('first') + const second = await queue('second') + await openRigTurnFor(rig, working, 'completed', 'failure') + await held(first, second) + const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + expect(journal.queuedMessages.nextInputCard()).toBeNull() + rig.providerEvents().publish() + rig.providerEvents().publish() + await rig.host.flushStreamedEvents(SESSION) + await held(first, second) + + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + const handoff = await dispatched(first) + await openRigTurnFor(rig, handoff) + await rig.settleAccepted(handoff, 'retried') + await openRigTurnFor(rig, handoff, 'completed', 'failure') + await held(second) + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + await dispatched(second) + }) + + it('lets a fresh send recover; another failure holds and a successful turn releases the cards in order', async () => { + const working = await start() + const first = await queue('first') + const second = await queue('second') + await openRigTurnFor(rig, working, 'completed', 'failure') + await held(first, second) + + const retrySend = rig.send('retry', 'queue-if-active') + expect(await retrySend.result).toMatchObject({ + ok: true, + value: { submission: expect.anything() } + }) + await eventually(async () => + expect((await rig.submission(retrySend.id))?.handedOverAt).toBeDefined() + ) + const retry = retrySend.id + expect(await rig.queuePause()).toBeNull() + await openRigTurnFor(rig, retry) + await rig.settleAccepted(retry, 'retry') + await openRigTurnFor(rig, retry, 'completed', 'failure') + await held(first, second) + + const recovered = await start() + await openRigTurnFor(rig, recovered, 'completed', 'success') + const handoff = await dispatched(first) + expect(await rig.handoff(second)).toBeUndefined() + await openRigTurnFor(rig, handoff) + await rig.settleAccepted(handoff, 'first') + await openRigTurnFor(rig, handoff, 'completed', 'success') + await dispatched(second) + }) + + it('lets explicit Send retry a chosen card without consuming the others', async () => { + const working = await start() + const first = await queue('first') + const second = await queue('chosen') + await openRigTurnFor(rig, working, 'completed', 'failure') + await held(first, second) + expect(await rig.sendNow(second)).toMatchObject({ ok: true }) + const handoff = await dispatched(second) + expect(await rig.handoff(first)).toBeUndefined() + await openRigTurnFor(rig, handoff) + await rig.settleAccepted(handoff, 'chosen') + await openRigTurnFor(rig, handoff, 'completed', 'failure') + await held(first) + }) + + it('does not mistake a steer accepted inside the failed turn for recovery', async () => { + const working = await start() + const first = await queue('keep waiting') + const steer = rig.send('steer into the running turn') + await steer.result + await rig.settleAccepted(steer.id, 'steered') + await openRigTurnFor(rig, working, 'completed', 'failure') + await held(first) + }) + + it('does not let a Resume written while the turn still runs waive its later failure', async () => { + const working = await start() + const first = await queue('first') + const second = await queue('second') + rig.cancelTurn.mockImplementationOnce(async () => { + throw new Error('interrupt timed out') + }) + rig.closeSession.mockImplementationOnce(async () => { + throw new Error('exit not proven') + }) + await rig.stop() + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + await clock.observe(250) + expect(await rig.handoff(first)).toBeUndefined() + await openRigTurnFor(rig, working, 'completed', 'failure') + await rig.host.flushStreamedEvents(SESSION) + await held(first, second) + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + await dispatched(first) + expect(await rig.handoff(second)).toBeUndefined() + }) + + it('re-derives the hold after restart; a later Resume releases it and late revisions do not re-hold', async () => { + const working = await start() + const first = await queue('survives restart') + const second = await queue('second') + await openRigTurnFor(rig, working, 'completed', 'failure') + await held(first, second) + await rig.restartHostProcess() + expect(await rig.handoff(first)).toBeUndefined() + const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + expect(journal.newestTurn()).toMatchObject({ outcome: 'failure' }) + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + const handoff = await dispatched(first) + const failed = () => journal.queuedMessages.pauses().some((pause) => pause.reason === 'failed') + await openRigTurnFor(rig, working, 'completed', 'failure') + expect(failed()).toBe(false) + await openRigTurnFor(rig, handoff) + await rig.settleAccepted(handoff, 'first') + await openRigTurnFor(rig, working, 'completed', 'failure') + expect(failed()).toBe(false) + await openRigTurnFor(rig, handoff, 'completed', 'success') + await dispatched(second) + }) + + it('holds orchestration mail that arrives during the hold until Resume', async () => { + const working = await start() + const first = await queue('first') + await openRigTurnFor(rig, working, 'completed', 'failure') + await held(first) + const message = mail('worker finished') + expect(await message.outcome).toMatchObject({ kind: 'queued' }) + await held(first, message.id) + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + const handoff = await dispatched(first) + expect(await rig.handoff(message.id)).toBeUndefined() + await openRigTurnFor(rig, handoff) + await rig.settleAccepted(handoff, 'first') + await openRigTurnFor(rig, handoff, 'completed', 'success') + await dispatched(message.id) + }) + + it("lets held mail's Send deliver it, and a person's fresh send still recovers", async () => { + const working = await start() + await openRigTurnFor(rig, working, 'completed', 'failure') + const message = mail('worker finished') + expect(await message.outcome).toMatchObject({ kind: 'queued' }) + await held(message.id) + expect(await rig.sendNow(message.id)).toMatchObject({ ok: true }) + const handoff = await dispatched(message.id) + await openRigTurnFor(rig, handoff) + await rig.settleAccepted(handoff, 'mail') + await openRigTurnFor(rig, handoff, 'completed', 'failure') + const later = mail('another worker finished') + expect(await later.outcome).toMatchObject({ kind: 'queued' }) + await held(later.id) + + const retrySend = rig.send('retry', 'queue-if-active') + expect(await retrySend.result).toMatchObject({ + ok: true, + value: { submission: expect.anything() } + }) + await eventually(async () => + expect((await rig.submission(retrySend.id))?.handedOverAt).toBeDefined() + ) + await openRigTurnFor(rig, retrySend.id) + await rig.settleAccepted(retrySend.id, 'retry') + await openRigTurnFor(rig, retrySend.id, 'completed', 'success') + await dispatched(later.id) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index 3c7ab211b5f..0cfdf424a48 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -69,7 +69,7 @@ export function structuredQueueHold(input: StructuredQueueGateInput): Structured /** Publish the drain’s live pick; backlog itself never gates a lone draft. */ export function nextStructuredQueuedMessage( - input: StructuredQueueGateInput & { journal: AgentSessionJournal } + input: StructuredQueueGateInput & { journal: AgentSessionJournal; working?: boolean } ): QueuedMessageRow | null { // Every gate before the queue is asked: choosing a card can read mail authority, and a session // that is working, blocked or waiting on a prompt can send nothing anyway. An empty queue has @@ -78,7 +78,7 @@ export function nextStructuredQueuedMessage( const cards = journalInputOrder(state).cards cards.ensure(state) if ( - structuredInputExecutionWorking(input.journal, input.work) || + (input.working ?? structuredInputExecutionWorking(input.journal, input.work)) || !input.record || structuredAgentSessionSendBlock(input.record) || !cards.head() || @@ -92,7 +92,7 @@ export function nextStructuredQueuedMessage( /** Queue behind work/actionable backlog; held or returned cards cannot trap a fresh send that lifts their pause. */ export function shouldQueueStructuredAgentSessionSend( - input: StructuredQueueGateInput & { journal: AgentSessionJournal } + input: StructuredQueueGateInput & { journal: AgentSessionJournal; fromAgent?: boolean } ): boolean { const hold = structuredQueueHold(input) if (hold === 'blocked') { @@ -102,6 +102,15 @@ export function shouldQueueStructuredAgentSessionSend( if (hold !== null) { return true } + if (input.fromAgent) { + // Only the person's own send recovers a failed turn; another agent's message waits held with the rest. + const state = input.journal.queuedMessages.inputState() + const cards = journalInputOrder(state).cards + cards.ensure(state) + if (cards.failureHolds()) { + return true + } + } return oldestActionableQueuedMessage(input.journal, false) !== null } @@ -157,7 +166,8 @@ export async function maybeQueueStructuredAgentSessionSend( !shouldQueueStructuredAgentSessionSend({ journal: ctx.journal, record: context.deps.store.getRecord(ctx.sessionId), - work: contextStructuredAgentSessionCurrentWork(ctx) + work: contextStructuredAgentSessionCurrentWork(ctx), + fromAgent: !params.userSend && !params.personsMessage && params.body.from !== undefined }) ) { return null diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts index 094dabd1b81..05cecad072e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts @@ -21,6 +21,7 @@ import { heldStructuredAgentSessionCurrentWork, type StructuredAgentSessionCurrentWorkHost } from './structured-agent-session-host-current-work' +import { structuredInputExecutionWorking } from './structured-agent-session-input-selection' export type QueuePublication = { queuedMessages: AgentSessionQueuedMessage[] @@ -115,26 +116,34 @@ export function readQueuePublication( gate: QueueSendGate ): QueuePublication { const queuedMessages = readPublishedQueuedMessages(journal) - // Show personal Stop only when Resume can send a card; hide old pauses after clear/reopen. + // Show pauses only when Resume can send a card; hide old pauses after clear/reopen. // Re-derive per emit because submission acceptance can lift a pause. const pauses = structuredQueuePauses(journal) - const stop = pauses.find((pause) => pause.reason === 'stopped')?.since + const visible = Math.max( + 0, + ...pauses + .filter((pause) => pause.reason === 'stopped' || pause.reason === 'failed') + .map((pause) => pause.since?.sequence ?? 0) + ) const silent = pauses.some( (pause) => pause.reason === 'cleared' || - (pause.reason === 'restarted' && - (!stop || - !pause.since || - stop.epoch !== pause.since.epoch || - stop.sequence < pause.since.sequence)) + (pause.reason === 'restarted' && (!visible || !pause.since || visible < pause.since.sequence)) ) const resumable = silent ? null : resumableQueuePause(pauses, journal.queuedMessages.list()) - // Avoid scanning submissions when no pause can show. + // The input order indexes pending sends, so a held queue never rescans history. const pause = - resumable && !queuePauseLiftOnItsWay(resumable, journal.submissions()) ? resumable : null - const queuePause = pause?.reason === 'stopped' ? { reason: pause.reason } : null + resumable && !queuePauseLiftOnItsWay(resumable, journal.inputOrder.sendingSubmissions()) + ? resumable + : null + const input = gate(journal) + const working = structuredInputExecutionWorking(journal, input.work) + const queuePause = + pause?.reason === 'stopped' || (pause?.reason === 'failed' && !working) + ? { reason: pause.reason } + : null const nextQueuedMessageId = - nextStructuredQueuedMessage({ journal, ...gate(journal) })?.messageId ?? null + nextStructuredQueuedMessage({ journal, ...input, working })?.messageId ?? null const previous = publications.get(journal) if ( previous && diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-rig-turn.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-rig-turn.test-fixture.ts index 0cc37784bd3..aed9c1df466 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-rig-turn.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-rig-turn.test-fixture.ts @@ -7,12 +7,14 @@ import { HOST_TEST_THREAD as THREAD } from './structured-agent-session-host-test-data' import type { QueuedMessageTestRig } from './structured-agent-session-queued-message-rig.test-fixture' +import type { AgentJournalTurnOutcome } from '../../../shared/agent-turn-outcome' /** Open the send’s turn; repeat with interrupted to end it. */ export async function openRigTurnFor( rig: QueuedMessageTestRig, id: string, - state: 'running' | 'interrupted' = 'running' + state: 'running' | 'interrupted' | 'completed' = 'running', + outcome?: AgentJournalTurnOutcome ): Promise { const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)?.journal if (!journal) { @@ -25,6 +27,7 @@ export async function openRigTurnFor( turnId: `turn-of-${id}`, startedAt: Date.now(), userItemId: agentJournalSubmissionKey(id), + ...(outcome ? { outcome } : {}), ...(state === 'running' ? { state } : { state, completedAt: Date.now() }) }, { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-usage-limit-continue.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-usage-limit-continue.test.ts index 62ddfd1c837..9a624a99e94 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-usage-limit-continue.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-usage-limit-continue.test.ts @@ -81,6 +81,21 @@ async function limitTurn(journal: AgentSessionJournal, resetsAt: number, now: nu return turnItemId } +function resumeQueue(current: RestTestRig) { + return current.host.queuedMessagesResume(REST_TEST_CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.queuedMessagesResume', + sessionId: SESSION, + fields: {} + }) + } + }) +} + it('waits for the buffered reset, accepts one ordinary continuation and never sends twice', async () => { const { current, sweep, resetsAt } = await setup() expect(sweep.readState(SESSION)?.state).toBe('scheduled') @@ -218,11 +233,14 @@ it('lets later due chats run while an earlier chat has an edited queued message, blocks: [{ type: 'text' as const, text: 'backlog' }] } const row = await journal.queuedMessages.insert({ messageId: hostTestOperationId(), body }) + // The limit failure holds the card; Resume makes it actionable, the edit lease keeps it waiting. + const resumed = resumeQueue(current) journal.queuedMessages.editLeases.acquire( { callerKey: REST_TEST_CALLER.callerKey, editId: 'held-edit' }, { messageId: row.messageId, fingerprint: row.fingerprint } ) release() + expect(await resumed).toMatchObject({ ok: true, value: { resumed: true } }) const send = vi.spyOn(current.host, 'send') current.clock.now = resetsAt + 15_001 await sweep.tick() @@ -238,6 +256,53 @@ it('lets later due chats run while an earlier chat has an edited queued message, expect(sweep.readState(SESSION)?.state).toBe('scheduled') }) +it('continues ahead of a failure-held card, which drains only after the continuation succeeds', async () => { + const { current, sweep, journal, resetsAt } = await setup() + await journal.queuedMessages.insert({ + messageId: hostTestOperationId(), + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'held backlog' }] } + }) + const pause = async () => { + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + return page.ok ? page.page.queuePause : undefined + } + const dispatched = () => + current.adapter.dispatch.mock.calls.map(([{ body }]) => + body.kind === 'message' && body.blocks[0]?.type === 'text' ? body.blocks[0].text : null + ) + expect(await pause()).toEqual({ reason: 'failed' }) + const send = vi.spyOn(current.host, 'send') + current.clock.now = resetsAt + 15_000 + await sweep.tick() + expect(await send.mock.results[0]!.value).toMatchObject({ ok: true }) + await vi.waitFor(() => + expect(dispatched()).toEqual(['hello', NATIVE_CHAT_USAGE_LIMIT_CONTINUE_MESSAGE]) + ) + // The continuation's turn is still open: the card stays held. + expect(await pause()).toEqual({ reason: 'failed' }) + await journal.appendItem( + { provider: 'orca', clientMessageId: 'continued-turn' }, + { + kind: 'turn', + turnId: 'turn-2', + state: 'completed', + outcome: 'success', + startedAt: current.clock.now, + completedAt: current.clock.now + }, + { fence: 1, turnScope: { kind: 'thread' } } + ) + current.host.publishWorkChanged(SESSION) + await vi.waitFor(() => + expect(dispatched()).toEqual([ + 'hello', + NATIVE_CHAT_USAGE_LIMIT_CONTINUE_MESSAGE, + 'held backlog' + ]) + ) + expect(await pause()).toBeNull() +}) + it('reuses the status projection and does not cold replay disabled sweeps', async () => { const { current, sweep, journal } = await setup() const snapshot = vi.spyOn(journal, 'snapshot') diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-queued-messages.resume.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session-queued-messages.resume.test.tsx index c0eebb1ebc5..be313d0ab2a 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-queued-messages.resume.test.tsx +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-queued-messages.resume.test.tsx @@ -90,7 +90,7 @@ afterEach(() => { }) describe("the header row's pause", () => { - it.each(['stopped', 'cleared', 'some-newer-reason'])( + it.each(['stopped', 'failed', 'cleared', 'some-newer-reason'])( "names the reason the host holds the queue for ('%s')", (reason) => { // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a newer host may publish a reason this client's type does not list. diff --git a/src/shared/agent-session/requests/agent-session-queued-message-wire.ts b/src/shared/agent-session/requests/agent-session-queued-message-wire.ts index 67f004e8dd1..e937506cf29 100644 --- a/src/shared/agent-session/requests/agent-session-queued-message-wire.ts +++ b/src/shared/agent-session/requests/agent-session-queued-message-wire.ts @@ -8,8 +8,8 @@ export const QUEUED_MESSAGE_PAUSED_SEND_FAILED = 'send_failed' as const export type AgentSessionQueuedMessagePausedReason = typeof QUEUED_MESSAGE_PAUSED_SEND_FAILED -/** stopped pauses all automatic sends until Resume or a new turn starts. Send-now delivers one card; others wait for that turn. /clear/reopen holds are unpublished because nothing runs then. Unknown reasons render a plain pause. */ -export type AgentSessionQueuePause = { reason: 'stopped' } +/** Stop/failure pause automatic sends until Resume or a new turn starts. Send-now delivers one card; others wait for that turn. /clear/reopen holds are unpublished because nothing runs then. Unknown reasons render a plain pause. */ +export type AgentSessionQueuePause = { reason: 'stopped' | 'failed' } /** What rides beside a frame's `queuedMessages`, published together with the list. */ export type AgentSessionQueuePublicationFields = {