diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a78f24667fe..9edeeece8f3 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -873,6 +873,7 @@ jobs: tests/e2e/cross-version-wire/cross-version-worktree-identity-downgrade.unit.test.ts tests/e2e/cross-version-wire/cross-version-session-tabs-retirement-proof.unit.test.ts tests/e2e/cross-version-wire/agent-session-resume-marker-downgrade.unit.test.ts + tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts tests/e2e/cross-version-wire/cross-version-worktree-ps-verdict.unit.test.ts managed_hook_node18: diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts index 52a5478c381..51d3a4b2fd5 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts @@ -4,6 +4,7 @@ import type { } from '../../../shared/agent-session-journal-types' import type { JournalHostDatabase } from './journal-host-database' import { replaceJournalEpoch, type JournalReplacementItem } from './journal-epoch-replacement' +import type { JournalQueuePauseRestatement } from './queued-message-pause' import { publishNewEpoch } from './journal-epoch-rollover' import type { JournalLoad } from './journal-open' import type { AgentJournalEpochReason } from './journal-row-schema' @@ -20,6 +21,8 @@ export class JournalEpochController { readOnly: () => boolean setReadOnly: (readOnly: boolean) => void highestFence: () => number + /** What of the live epoch's Stop and Resume a replacement restates. */ + queuePauseRestatement: () => JournalQueuePauseRestatement cursor: () => AgentJournalCursor adopt: (loaded: JournalLoad) => void } @@ -67,6 +70,7 @@ export class JournalEpochController { reason, fence, items, + queuePause: this.deps.queuePauseRestatement(), now: this.deps.now, mintEpoch: this.deps.mintEpoch, onPublished: this.deps.adopt diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts index 646254b4932..2992459a787 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts @@ -67,6 +67,7 @@ function replace(input: { reason: 'legacy_import', fence: 1, items: input.items, + queuePause: { lifted: false, liveStop: null }, now, mintEpoch: () => `epoch-${clock}`, onPublished: input.onPublished ?? (() => undefined) diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts index 8d94c5b2b5b..70efc891283 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts @@ -16,6 +16,11 @@ import type { JournalLoad } from './journal-open' import { clearJournalRepairMarker } from './journal-repair-marker' import { applyJournalRow, createJournalReducerState } from './journal-reducer' import { buildJournalItemRow, journalRowBase } from './journal-row-builders' +import { + buildJournalQueueResumeRow, + buildJournalStopEventRow +} from './journal-stop-and-resume-rows' +import type { JournalQueuePauseRestatement } from './queued-message-pause' import { deleteJournalEpochRows, insertJournalRow, @@ -40,6 +45,9 @@ export function replaceJournalEpoch(input: { reason: AgentJournalEpochReason fence: number items: readonly JournalReplacementItem[] + /** Restated in the new epoch, or the rewind would release cards the person stopped, or bring + * back a /clear pause they already lifted. */ + queuePause: JournalQueuePauseRestatement now: () => number mintEpoch: () => string /** Called the instant the transaction commits, before any fallible follow-up. */ @@ -70,6 +78,18 @@ export function replaceJournalEpoch(input: { applyJournalRow(state, row) rows.push(row) } + const { lifted, liveStop } = input.queuePause + const place = () => ({ state, seq: state.lastSequence + 1, fence: input.fence, ts: input.now() }) + if (lifted) { + const row = buildJournalQueueResumeRow(place()) + applyJournalRow(state, row) + rows.push(row) + } + if (liveStop) { + const row = buildJournalStopEventRow({ ...place(), event: liveStop }) + applyJournalRow(state, row) + rows.push(row) + } const { sessionId } = input.identity input.database.transaction((db) => { diff --git a/src/main/native-chat/agent-session-journal/journal-open.ts b/src/main/native-chat/agent-session-journal/journal-open.ts index ae753839c40..8f947272dbc 100644 --- a/src/main/native-chat/agent-session-journal/journal-open.ts +++ b/src/main/native-chat/agent-session-journal/journal-open.ts @@ -19,7 +19,7 @@ import { } from './journal-row-table' import { JOURNAL_REPAIR_DISCLOSURE_ITEM_ID } from './journal-repair-disclosure' import { pendingJournalRepairSequence } from './journal-repair-marker' -import { parseJournalRow, type JournalRow } from './journal-row-schema' +import { isJournalStopOrResumeRow, parseJournalRow, type JournalRow } from './journal-row-schema' /** Every epoch row is sequence 1, and no compaction moves that floor. */ const FIRST_JOURNAL_SEQUENCE = 1 @@ -120,7 +120,8 @@ export function startJournalRowFold(input: JournalRowFoldInput): { } applyJournalRow(state, row) const disclosure = row.kind === 'item' && row.itemId === JOURNAL_REPAIR_DISCLOSURE_ITEM_ID - if (!disclosure) { + // A Stop or Resume is no history, so it never reads as a rebuilt or provider-backed epoch. + if (!disclosure && !isJournalStopOrResumeRow(row)) { repairHasContent ||= repairedFrom !== null && row.seq >= repairedFrom providerHasContent ||= row.seq >= FIRST_JOURNAL_SEQUENCE + 1 } diff --git a/src/main/native-chat/agent-session-journal/journal-queued-messages.ts b/src/main/native-chat/agent-session-journal/journal-queued-messages.ts index 96b45c9d17c..b8fcafc0b6c 100644 --- a/src/main/native-chat/agent-session-journal/journal-queued-messages.ts +++ b/src/main/native-chat/agent-session-journal/journal-queued-messages.ts @@ -13,17 +13,16 @@ import { import type { JournalHostDatabase } from './journal-host-database' import type { JournalReducerState } from './journal-reducer' import type { JournalRow } from './journal-row-schema' +import type { JournalRowTransactionHook } from './journal-row-writer' import type { JournalSubmissionConsume } from './journal-store-contracts' import { adoptQueuedMessages, holdQueuedMessages } from './queued-message-holds' import { - clearQueuePause, - queuePauseHoldsBack, - readQueuePause, - recordQueuePause, - retireQueuePauseIfNothingHeld, - type QueuePauseFact, - type QueuePauseReason -} from './queued-message-pause-table' + deriveQueuePauses, + journalUserStopInForce, + nextSendableQueuedCard, + type DerivedQueuePause, + type JournalQueuePauseMarks +} from './queued-message-pause' import { consumeQueuedMessageInTransaction, getQueuedMessage, @@ -37,7 +36,6 @@ import { import { draftsDeliveredByAppliedEcho } from './queued-message-delivered-echo' import { pruneQueuedMessages, retainedSubmissionVerdict } from './queued-message-retention' import { - owedBackToWaiting, queuedMessageSettlementOwed, settleOwedQueuedMessages, settleQueuedMessagesForRow @@ -55,6 +53,8 @@ export type JournalQueuedMessagesDeps = { database: () => JournalHostDatabase readOnly: () => boolean state: () => JournalReducerState + /** Whether this handle found the row at `sequence` on disk when it opened. */ + wroteBeforeOpen: (sequence: number) => boolean /** The journal's own commit notification. Every standalone draft-table * transaction that changed rows fires it after COMMIT, so a draft or hold * change publishes and wakes the drain through the same path a journal row @@ -67,7 +67,6 @@ export class JournalQueuedMessages { /** Bumped on every draft-table write, so publication memos recompute only when they must. */ private changeRevision = 0 private listed: { revision: number; rows: readonly QueuedMessageRow[] } | null = null - private paused: { revision: number; fact: QueuePauseFact | null } | null = null constructor(private readonly deps: JournalQueuedMessagesDeps) {} @@ -75,12 +74,6 @@ export class JournalQueuedMessages { return this.changeRevision } - /** The submission row of the latest accepted turn a person asked for; 0 when none. What - * ends the queue's pause, read from the reducer in O(1). */ - latestPersonTurnSequence(): number { - return this.deps.state().latestPersonTurnSequence - } - /** A journal transaction rolled back: nothing read inside it may stay cached. */ invalidate(): void { this.changeRevision++ @@ -107,33 +100,32 @@ export class JournalQueuedMessages { return queuedMessagesSettledByOp(this.deps.database().db, this.deps.sessionId, settledByOp) } - /** `pausedBy`: the queue is paused in the SAME transaction as this card lands - * (a /clear's carry), so the drain never sees it unpaused and no pause fact - * exists without a card under it. */ + /** `carriedFrom`: a /clear's carry. The card is its own 'cleared' pause, so it lands paused. */ insert(input: { messageId: string body: AgentJournalMessageItem fingerprint: string hostInstance: string - pausedBy?: QueuePauseReason + carriedFrom?: string }): Promise { - const { pausedBy, ...draft } = input const { sessionId } = this.deps let inserted = false return this.transact( (db) => { - const existing = getQueuedMessage(db, sessionId, draft.messageId) + const existing = getQueuedMessage(db, sessionId, input.messageId) if (existing) { // One id, one draft: admission replays a recorded operation before it // gets here, so an existing row is the same accept landing twice. return existing } inserted = true - const row = insertQueuedMessage(db, { ...draft, sessionId, now: this.deps.now() }) - if (pausedBy) { - recordQueuePause(db, { sessionId, fact: this.pauseFact(pausedBy) }) - } - return row + const { epoch, lastSequence } = this.deps.state() + return insertQueuedMessage(db, { + ...input, + sessionId, + queuedAt: { epoch, sequence: lastSequence }, + now: this.deps.now() + }) }, () => inserted ) @@ -149,53 +141,44 @@ export class JournalQueuedMessages { ).then(() => undefined) } - /** Where the user's last Stop took effect, if it is still recorded; cached per revision. */ - pause(): QueuePauseFact | null { - if (this.paused?.revision !== this.changeRevision) { - this.paused = { - revision: this.changeRevision, - fact: readQueuePause(this.deps.database().db, this.deps.sessionId) - } - } - return this.paused.fact + /** The queue's pauses in force, derived from the fold and the cards (`queued-message-pause.ts`). */ + pauses(hostInstance: string): DerivedQueuePause[] { + return this.derivePauses(this.list(), hostInstance) } - /** A Stop took effect here: the queue is paused from this position on — if, judged in - * the same transaction, it holds back a card at all. Returns whether it recorded. */ - recordPause(reason: QueuePauseReason): Promise { - const fact = this.pauseFact(reason) - return this.transact( - (db) => - queuePauseHoldsBack(db, this.pauseScope()) && - (recordQueuePause(db, { sessionId: this.deps.sessionId, fact }), true), - (recorded) => recorded - ) + /** The person's Stop still pausing the queue, if any (`journalUserStopInForce`). */ + userStopInForce(): JournalQueuePauseMarks['latestStop'] { + const state = this.deps.state() + return journalUserStopInForce(state.queuePauseMarks, state.latestPersonTurnSequence) } - /** What `queuePauseHoldsBack` judges a pause by, from this journal's submissions. */ - private pauseScope() { - return { - sessionId: this.deps.sessionId, - owedToWaiting: owedBackToWaiting(this.deps.state().submissions) - } + private derivePauses( + cards: readonly QueuedMessageRow[], + hostInstance: string + ): DerivedQueuePause[] { + const state = this.deps.state() + return deriveQueuePauses({ + epoch: state.epoch, + marks: state.queuePauseMarks, + latestPersonTurnSequence: state.latestPersonTurnSequence, + cards, + hostInstance, + restartEnded: this.restartEnded() + }) } - private pauseFact(reason: QueuePauseReason): QueuePauseFact { - const { epoch, lastSequence: sequence } = this.deps.state() - return { reason, epoch, sequence, recordedAt: this.deps.now() } + /** A person's turn started since this handle opened, which ends a restart's pause. */ + restartEnded(): boolean { + const latest = this.deps.state().latestPersonTurnSequence + return latest > 0 && !this.deps.wroteBeforeOpen(latest) } - /** Ends the queue's pause: `stop` retires that Stop fact (never a later one), - * `adoptInto` adopts a restart's rows into this host instance. Returns whether - * anything changed. */ - liftPause(input: { stop: QueuePauseFact | null; adoptInto: string | null }): Promise { + /** Adopts waiting rows another host instance wrote into this one, ending a restart's pause. + * Returns whether anything changed. */ + adopt(hostInstance: string): Promise { const { sessionId } = this.deps return this.transact( - (db) => - (input.stop ? clearQueuePause(db, { sessionId, fact: input.stop }) : 0) + - (input.adoptInto === null - ? 0 - : adoptQueuedMessages(db, { sessionId, hostInstance: input.adoptInto })), + (db) => adoptQueuedMessages(db, { sessionId, hostInstance }), (changed) => changed > 0 ).then((changed) => changed > 0) } @@ -229,12 +212,8 @@ export class JournalQueuedMessages { ): Promise { return this.deps.serialize(async () => { assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) - const { result, retired } = this.deps.database().transaction((db) => ({ - result: run(db), - // Any draft write may take the last card a pause holds back. - retired: retireQueuePauseIfNothingHeld(db, this.pauseScope()) - })) - if (changed(result) || retired > 0) { + const result = this.deps.database().transaction(run) + if (changed(result)) { this.changeRevision++ this.deps.committed() } @@ -252,7 +231,6 @@ export class JournalQueuedMessages { row, now: this.deps.now() }) - this.changeRevision += retireQueuePauseIfNothingHeld(db, this.pauseScope()) } /** The in-transaction consume for `appendSubmission`; a false compare-and-set @@ -266,6 +244,16 @@ export class JournalQueuedMessages { // Same handle only: a second connection could not join the transaction. throw new AgentSessionJournalError('journal_closed', 'consume crossed database handles') } + if (input.yieldsToPause) { + // Judged again here, by the drain's own rule. Today a Stop cannot land between the drain's + // pick and this claim (both run on the session's serialized lane, held across the send), so + // this guards any pause-relevant row written off that lane from overtaking a held card. + const cards = listQueuedMessages(db, this.deps.sessionId) + const pauses = this.derivePauses(cards, input.yieldsToPause.hostInstance) + if (nextSendableQueuedCard(pauses, cards)?.messageId !== input.messageId) { + throw new QueuedMessageNotConsumableError(input.messageId, input.expect) + } + } const consumed = consumeQueuedMessageInTransaction(db, { ...input, sessionId: this.deps.sessionId, @@ -274,7 +262,6 @@ export class JournalQueuedMessages { if (!consumed) { throw new QueuedMessageNotConsumableError(input.messageId, input.expect) } - retireQueuePauseIfNothingHeld(db, this.pauseScope()) this.changeRevision++ } @@ -317,11 +304,11 @@ export class JournalQueuedMessages { * Open-time reconciliation, a re-derivation behind the stored fact: owed * settlements apply exactly as the live hook would have (covers consume → * crash → downgrade → upgrade, where the old build rejected the leftover with - * no hook), then retention runs; a pause left holding back nothing retires. + * no hook), then retention runs. */ repairAndPrune(): Promise { // No draft, no work, and no write: a chat whose first-use copy is still owed stays uncopied. - if (this.deps.readOnly() || (this.list().length === 0 && this.pause() === null)) { + if (this.deps.readOnly() || this.list().length === 0) { return Promise.resolve() } const { sessionId } = this.deps @@ -348,7 +335,7 @@ export function queuedMessageConsumeHook( queuedMessages: JournalQueuedMessages, consumedAs: string, consume: JournalSubmissionConsume -): (db: Database.Database) => void { +): JournalRowTransactionHook { return (db) => queuedMessages.consumeInTransaction(db, { ...consume, consumedAs }) } diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index ed12c4a6cba..a7d6f150e70 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -26,10 +26,15 @@ import { structuredAgentSessionPayloadFingerprint } from '../../../shared/struct import { JournalDerivedTurnScope } from './journal-derived-turn-scope' import { removeJournalItem, statedOrDerivedTurnScope, upsertJournalItem } from './journal-item-fold' import { journalItemRevisionIsStale } from './journal-item-revision' -import type { JournalRow } from './journal-row-schema' +import { isJournalStopOrResumeRow, type JournalRow } from './journal-row-schema' import { acceptSubmissionFromProviderItem, applyJournalSubmission } from './journal-submission-fold' import { applyJournalDispatchRow } from './journal-dispatch-reducer' import { isWriteFailureSubmission } from '../../../shared/structured-agent-session-dispatch-rejection' +import { + createJournalQueuePauseMarks, + foldJournalQueuePauseMark, + type JournalQueuePauseMarks +} from './queued-message-pause' export const MAX_JOURNAL_APPLIED_SETTLEMENT_IDS = 4_096 @@ -57,6 +62,8 @@ export type JournalReducerState = { /** The submission row of the latest turn a person asked for (`origin: 'client'`) that the * provider accepted; 0 when none. Kept as it folds so the queue's pause reads it in O(1). */ latestPersonTurnSequence: number + /** The latest person's Stop event and Resume, what the queue's pause is derived from. */ + queuePauseMarks: JournalQueuePauseMarks } export function createJournalReducerState(sessionId: string, epoch: string): JournalReducerState { @@ -75,7 +82,8 @@ export function createJournalReducerState(sessionId: string, epoch: string): Jou aliases: new Map(), appliedSettlementIds: new Set(), derivedTurnScope: new JournalDerivedTurnScope(), - latestPersonTurnSequence: 0 + latestPersonTurnSequence: 0, + queuePauseMarks: createJournalQueuePauseMarks() } } @@ -101,6 +109,10 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo ) return } + if (isJournalStopOrResumeRow(row)) { + foldJournalQueuePauseMark(state.queuePauseMarks, row) + return + } if (row.kind === 'tombstone') { removeJournalItem(state, resolveItemId(state, row.itemId), row.revision) return diff --git a/src/main/native-chat/agent-session-journal/journal-row-schema.ts b/src/main/native-chat/agent-session-journal/journal-row-schema.ts index 1c7ea36eaef..e65d5919b39 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-schema.ts @@ -20,6 +20,7 @@ import { isAdmissibleAgentJournalMessageBody } from '../../../shared/agent-session-journal-schemas' import { isAdmissibleAgentSessionContextUsage } from '../../../shared/agent-session-context-usage-schema' +import type { StructuredAgentSessionStopCause } from '../agent-session-wire/structured-agent-session-stop-cause' /** Producer linkage rides the row BASE rather than the body: the two nested * prompt shapes are `.strict()`, so an unknown key on a body would make the @@ -72,6 +73,37 @@ export type JournalTombstoneRow = JournalRowBase & { kind: 'tombstone' itemId: string revision: number + /** Present: not a removal but a Stop's event, on an id no item ever takes. */ + stopEvent?: JournalStopEvent + /** Present: not a removal but a person's Resume of the queue, on an id no item ever takes. */ + queueResume?: true +} + +/** One Stop that took effect. Temporary carrier: a tombstone's extra key, because a released host + * deletes the journal from the first row kind it does not know (`journal-open.ts` then + * `journal-store-open.ts`) but ignores an unknown key; a row kind of its own once released hosts + * skip unknown kinds instead. */ +export type JournalStopEvent = { + /** Persisted: never rename an arm. Only `user-stop` pauses the queue. */ + reason: StructuredAgentSessionStopCause + /** The turn the Stop named, else the one running when it took effect. */ + turnId?: string + /** When it took effect; a rewind's restatement keeps it. */ + at: number + /** Who asked (`StructuredAgentSessionCaller.callerKey`). */ + caller?: string +} + +/** A tombstone that carries a Stop event or a Resume mark instead of removing an item. */ +export type JournalStopOrResumeRow = JournalTombstoneRow & + ( + | { stopEvent: NonNullable } + | { queueResume: NonNullable } + ) + +/** A Stop's event or a Resume. Any value counts, so a newer build's mark never removes an item. */ +export function isJournalStopOrResumeRow(row: JournalRow): row is JournalStopOrResumeRow { + return row.kind === 'tombstone' && (row.stopEvent !== undefined || row.queueResume !== undefined) } /** The write-ahead row. Durable BEFORE the adapter dispatches anything; it diff --git a/src/main/native-chat/agent-session-journal/journal-stop-and-resume-rows.test.ts b/src/main/native-chat/agent-session-journal/journal-stop-and-resume-rows.test.ts new file mode 100644 index 00000000000..8eee42a70d0 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-stop-and-resume-rows.test.ts @@ -0,0 +1,122 @@ +// A Stop's event and a Resume ride tombstones, but are no history: an app never receives them, +// and they never count as the content that makes a rebuilt or provider-backed epoch usable. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' +import { projectJournalBatch } from '../agent-session-wire/agent-session-journal-batch' +import { createTrackedJournalOpener } from './journal-host-database-test-support' +import { startJournalRowFold } from './journal-open' +import { createJournalReducerState } from './journal-reducer' +import { buildJournalItemRow, journalRowBase } from './journal-row-builders' +import { + serializeJournalRow, + type AgentJournalEpochReason, + type JournalRow +} from './journal-row-schema' +import { + buildJournalQueueResumeRow, + buildJournalStopEventRow +} from './journal-stop-and-resume-rows' + +const IDENTITY: AgentSessionJournalIdentity = { + sessionId: 'session-s', + workspaceId: 'ws-1', + hostId: 'host-1', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: 'thread-1' } +} +const EPOCH = 'epoch-1' + +let root: string +const journals = createTrackedJournalOpener() + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-stop-event-rows-')) +}) + +afterEach(async () => { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +/** One epoch's rows, folded as an open folds them. */ +function fold(rows: readonly JournalRow[], repairedFrom: number | null = null) { + const folding = startJournalRowFold({ sessionId: IDENTITY.sessionId, epoch: EPOCH, repairedFrom }) + for (const row of rows) { + folding.add({ seq: row.seq, rowJson: serializeJournalRow(row) }) + } + return folding.finish() +} + +/** An epoch row, then `after` built at the next sequences. */ +function epochWith( + reason: AgentJournalEpochReason, + after: readonly ('stop' | 'resume' | 'item')[] +): JournalRow[] { + const state = createJournalReducerState(IDENTITY.sessionId, EPOCH) + const rows: JournalRow[] = [ + { + kind: 'epoch', + reason, + providerHandle: IDENTITY.providerHandle, + ...journalRowBase(EPOCH, 1, 1, 1) + } + ] + for (const kind of after) { + const place = { state, seq: rows.length + 1, fence: 1, ts: rows.length + 1 } + rows.push( + kind === 'stop' + ? buildJournalStopEventRow({ ...place, event: { reason: 'user-stop', at: place.ts } }) + : kind === 'resume' + ? buildJournalQueueResumeRow(place) + : buildJournalItemRow({ + ...place, + identity: { provider: 'codex', threadId: 'thread-1', turnId: 'turn-1', ordinal: 0 }, + body: { kind: 'status', text: 'history' }, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) + ) + } + return rows +} + +describe("a Stop's event and a Resume", () => { + it('never reach an app: the batch projection skips them', async () => { + const journal = await journals.open({ identity: IDENTITY, stateDirectory: root }) + await journal.appendItem( + { provider: 'codex', threadId: 'thread-1', turnId: 'turn-1', ordinal: 0 }, + { kind: 'status', text: 'history' }, + { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + const cursor = journal.cursor() + await journal.appendStopEvent({ reason: 'user-stop' }, 1) + await journal.appendQueueResume(1) + const since = journal.readSince(cursor) + if (!since.ok) { + throw new Error(`expected rows, got reset ${since.reset}`) + } + expect(since.rows).toHaveLength(2) + const projected = projectJournalBatch({ + rows: since.rows, + snapshot: journal.snapshot(), + afterSequence: cursor.sequence + }) + expect(projected).toMatchObject({ ok: true, batch: { items: [], removedItemIds: [] } }) + }) + + it('are no provider history: an unreconcilable epoch holding only them still reads corrupt', () => { + expect(fold(epochWith('unreconcilable_prefix', ['stop', 'resume'])).corrupt).toBe(true) + expect(fold(epochWith('unreconcilable_prefix', ['stop', 'item'])).corrupt).toBe(false) + }) + + it('are no rebuilt history: a pending repair followed only by them still reads corrupt', () => { + expect(fold(epochWith('handle_forked', ['stop', 'resume']), 2).corrupt).toBe(true) + expect(fold(epochWith('handle_forked', ['stop', 'item']), 2).corrupt).toBe(false) + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-stop-and-resume-rows.ts b/src/main/native-chat/agent-session-journal/journal-stop-and-resume-rows.ts new file mode 100644 index 00000000000..e763af8b2e6 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-stop-and-resume-rows.ts @@ -0,0 +1,58 @@ +// The rows a Stop's event and a person's Resume append: tombstones of ids no item ever takes, +// each carrying its record as an extra key (`journal-row-schema.ts` says why not a row kind). + +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import type { JournalReducerState } from './journal-reducer' +import { journalRowBase } from './journal-row-builders' +import type { JournalStopEvent, JournalTombstoneRow } from './journal-row-schema' + +/** One id per mark kind; no item ever takes either. */ +const JOURNAL_STOP_EVENT_ITEM_ID = agentJournalItemKey({ + provider: 'orca', + clientMessageId: 'stop-event' +}) +const JOURNAL_QUEUE_RESUME_ITEM_ID = agentJournalItemKey({ + provider: 'orca', + clientMessageId: 'queue-resume' +}) + +type RowPlace = { state: JournalReducerState; seq: number; fence: number; ts: number } + +export function buildJournalStopEventRow( + input: RowPlace & { event: JournalStopEvent } +): JournalTombstoneRow { + return { + kind: 'tombstone', + itemId: JOURNAL_STOP_EVENT_ITEM_ID, + revision: 1, + stopEvent: input.event, + ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts) + } +} + +export function buildJournalQueueResumeRow(input: RowPlace): JournalTombstoneRow { + return { + kind: 'tombstone', + itemId: JOURNAL_QUEUE_RESUME_ITEM_ID, + revision: 1, + queueResume: true, + ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts) + } +} + +/** A Stop taking effect now: its event's time is its row's. */ +export function journalStopEventRowBuilder( + state: () => JournalReducerState, + event: Omit, + fence: number +): (seq: number, ts: number) => JournalTombstoneRow { + return (seq, ts) => + buildJournalStopEventRow({ state: state(), seq, fence, ts, event: { ...event, at: ts } }) +} + +export function journalQueueResumeRowBuilder( + state: () => JournalReducerState, + fence: number +): (seq: number, ts: number) => JournalTombstoneRow { + return (seq, ts) => buildJournalQueueResumeRow({ state: state(), seq, fence, ts }) +} diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 16d3795cc6a..647b64e0f00 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -13,6 +13,7 @@ import { JournalItemAppender } from './journal-item-appender' import { JournalLifecycleBatchAppender } from './journal-lifecycle-batch-appender' import type { JournalLoad } from './journal-open' import { JournalQueuedMessages } from './journal-queued-messages' +import { journalQueuePauseRestatement } from './queued-message-pause' import type { JournalReducerState } from './journal-reducer' import { JournalRowWriter } from './journal-row-writer' import { restoreJournalStore } from './journal-store-restore' @@ -70,6 +71,11 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal readOnly: host.readOnly, setReadOnly: host.setReadOnly, highestFence: () => host.state().highestFence, + queuePauseRestatement: () => + journalQueuePauseRestatement( + host.state().queuePauseMarks, + host.state().latestPersonTurnSequence + ), cursor: host.cursor, adopt: host.adopt }) @@ -80,6 +86,7 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal database: host.database, readOnly: host.readOnly, state: host.state, + wroteBeforeOpen: (sequence) => host.journal().wroteBeforeOpen(sequence), committed: host.notifyCommitted }) return { diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index 29f0db6bed6..85e6b99f902 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -94,6 +94,9 @@ export type JournalSubmissionConsume = { /** The host process handing it off, stamped on the draft so a hand-off withdrawn back to * waiting belongs to the process that sent it, not the one that first wrote the card. */ hostInstance?: string + /** The queue's own send: refused in the consume's transaction while the queue's pause, as + * this host instance derives it, holds the card. Send-now omits it. */ + yieldsToPause?: { hostInstance: string } } export type JournalItemAppendInput = { diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 5b046ab421f..08113e3bc7d 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -58,7 +58,11 @@ import type { ResolveDispatchInput } from './journal-store-contracts' import { queuedMessageConsumeHook, type JournalQueuedMessages } from './journal-queued-messages' -import type { AgentJournalEpochReason } from './journal-row-schema' +import { + journalQueueResumeRowBuilder, + journalStopEventRowBuilder +} from './journal-stop-and-resume-rows' +import type { AgentJournalEpochReason, JournalStopEvent } from './journal-row-schema' import type { JournalRowWriter } from './journal-row-writer' import type { JournalEpochController } from './journal-epoch-controller' import { JournalWriteQueue } from './journal-write-queue' @@ -307,6 +311,16 @@ export class AgentSessionJournal { ) } + /** A Stop that took effect, timed by its row (`JournalStopEvent`). */ + appendStopEvent(event: Omit, fence: number): Promise { + return this.rowWriter.append(journalStopEventRowBuilder(() => this.state, event, fence)) + } + + /** A person's Resume of the queue. */ + appendQueueResume(fence: number): Promise { + return this.rowWriter.append(journalQueueResumeRowBuilder(() => this.state, fence)) + } + appendLifecycleBatch(input: JournalLifecycleBatchInput): Promise { return this.lifecycleBatchAppender.append(input) } diff --git a/src/main/native-chat/agent-session-journal/queued-message-delivered-echo.test.ts b/src/main/native-chat/agent-session-journal/queued-message-delivered-echo.test.ts index 5b21c36f3af..4ce62c6192b 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-delivered-echo.test.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-delivered-echo.test.ts @@ -139,14 +139,6 @@ describe("a waiting draft whose 'never delivered' claim an echo disproves", () = expect(journal.queuedMessages.get('draft-1')?.state).toBe('waiting') }) - it('retires the pause in the per-row hook when that echo withdraws the last card it holds back', async () => { - const journal = await withdrawnDraft('did it land?') - expect(await journal.queuedMessages.recordPause('stopped')).toBe(true) - await echo(journal, 'echo-1', 'did it land?') - expect(journal.queuedMessages.get('draft-1')?.state).toBe('withdrawn') - expect(journal.queuedMessages.pause()).toBeNull() - }) - it('stays waiting for an echo of some other text', async () => { const journal = await withdrawnDraft('did it land?') await echo(journal, 'echo-1', 'something else') diff --git a/src/main/native-chat/agent-session-journal/queued-message-holds.ts b/src/main/native-chat/agent-session-journal/queued-message-holds.ts index 665466d60e7..00b90a832c1 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-holds.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-holds.ts @@ -1,7 +1,6 @@ // Per-draft holds: what keeps one card from auto-sending, stored on its row. A -// Stop or a restart pauses the whole queue instead (`queued-message-pause-table.ts`, -// and the host instance each row records); a per-draft hold is only a -// conversion that failed, which an explicit Send releases. +// Stop or a restart pauses the queue instead (`queued-message-pause.ts`, derived); +// a per-draft hold is only a conversion that failed, which an explicit Send releases. import type Database from '../../sqlite/sync-database' import type { QueuedMessageHoldReason } from './queued-message-table' diff --git a/src/main/native-chat/agent-session-journal/queued-message-pause-table.ts b/src/main/native-chat/agent-session-journal/queued-message-pause-table.ts deleted file mode 100644 index 41387165de0..00000000000 --- a/src/main/native-chat/agent-session-journal/queued-message-pause-table.ts +++ /dev/null @@ -1,161 +0,0 @@ -// The queue-level pause fact: where in the journal the user's last Stop (or a -// /clear, which starts its replacement paused) took effect. The pause itself is never stored — it is derived from this fact and -// the journal rows after it (a user-requested turn that started ends it); the -// fact only records the one event the journal's closed row kinds cannot carry. -// An explicit Resume retires it. - -import type Database from '../../sqlite/sync-database' - -export type QueuePauseReason = 'stopped' | 'cleared' - -export type QueuePauseFact = { - reason: QueuePauseReason - /** The journal position the Stop took effect at: rows after it are later. */ - epoch: string - sequence: number - recordedAt: number -} - -export function readQueuePause(db: Database.Database, sessionId: string): QueuePauseFact | null { - const row: unknown = db - .prepare( - 'SELECT reason, epoch, sequence, recorded_at FROM queued_message_pauses WHERE session_id = ?' - ) - .get(sessionId) - if ( - typeof row !== 'object' || - row === null || - !('reason' in row) || - (row.reason !== 'stopped' && row.reason !== 'cleared') || - !('epoch' in row) || - typeof row.epoch !== 'string' || - !('sequence' in row) || - typeof row.sequence !== 'number' || - !('recorded_at' in row) || - typeof row.recorded_at !== 'number' - ) { - // A reason this build cannot place reads as no pause rather than a wrong one. - return null - } - return { - reason: row.reason, - epoch: row.epoch, - sequence: row.sequence, - recordedAt: row.recorded_at - } -} - -/** The latest Stop replaces an earlier one: only the last interruption decides. */ -export function recordQueuePause( - db: Database.Database, - input: { sessionId: string; fact: QueuePauseFact } -): void { - db.prepare( - `INSERT INTO queued_message_pauses (session_id, reason, epoch, sequence, recorded_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT (session_id) DO UPDATE SET - reason = excluded.reason, epoch = excluded.epoch, - sequence = excluded.sequence, recorded_at = excluded.recorded_at` - ).run( - input.sessionId, - input.fact.reason, - input.fact.epoch, - input.fact.sequence, - input.fact.recordedAt - ) -} - -/** Compare-and-clear: only the fact the caller judged, so a Stop recorded since stands. */ -export function clearQueuePause( - db: Database.Database, - input: { sessionId: string; fact: Pick } -): number { - return Number( - db - .prepare( - 'DELETE FROM queued_message_pauses WHERE session_id = ? AND epoch = ? AND sequence = ?' - ) - .run(input.sessionId, input.fact.epoch, input.fact.sequence).changes ?? 0 - ) -} - -type QueueCardState = { state: string; holdReason: string | null } - -/** A card a pause holds back: waiting, with no hold of its own, wherever it sits. - * While one exists — or a hand-off still owed a return to waiting - * (`queuePauseHoldsBack`) — the pause is KEPT: a Stop records it, and it is - * retired only once none remains, so deleting a returned card that blocks such - * cards leaves them paused rather than sending them unasked. */ -export function isPausableQueuedMessage(row: QueueCardState): boolean { - return row.state === 'waiting' && row.holdReason === null -} - -/** Whether Resume would send anything: a pausable card not behind a returned one, - * which blocks everything after it until the user acts, exactly as the drain - * reads it. Only then is the kept pause PUBLISHED, so its header never offers a - * Resume that sends nothing. */ -export function hasResumableQueuedMessage(rows: readonly QueueCardState[]): boolean { - for (const row of rows) { - if (row.state === 'returned') { - return false - } - if (isPausableQueuedMessage(row)) { - return true - } - } - return false -} - -/** What a queue pause holds back, judged inside the caller's transaction: a waiting - * card with no hold of its own (`isPausableQueuedMessage`, in SQL), or a dispatched - * one whose settlement back to waiting is still owed — its hook was skipped, so the - * row has not caught up with its rejected submission, which only the journal's - * submissions can tell (`owedToWaiting`). */ -export function queuePauseHoldsBack( - db: Database.Database, - input: { sessionId: string; owedToWaiting: (consumedRef: string) => boolean } -): boolean { - const pausable = db - .prepare( - `SELECT 1 FROM queued_messages - WHERE session_id = ? AND state = 'waiting' AND hold_reason IS NULL LIMIT 1` - ) - .get(input.sessionId) - if (pausable !== undefined) { - return true - } - return db - .prepare( - `SELECT consumed_as FROM queued_messages - WHERE session_id = ? AND state = 'dispatched' AND consumed_as IS NOT NULL` - ) - .all(input.sessionId) - .some( - (row) => - typeof row === 'object' && - row !== null && - 'consumed_as' in row && - typeof row.consumed_as === 'string' && - input.owedToWaiting(row.consumed_as) - ) -} - -/** A pause is over the cards it paused: once it holds back none (`queuePauseHoldsBack`), - * the fact goes too, in the same transaction as the write that took the last one, so - * it can never outlive them and catch a card typed long after. */ -export function retireQueuePauseIfNothingHeld( - db: Database.Database, - input: { sessionId: string; owedToWaiting: (consumedRef: string) => boolean } -): number { - // Runs on every appended journal row: with no pause recorded there is nothing to judge. - const recorded = db - .prepare('SELECT 1 FROM queued_message_pauses WHERE session_id = ?') - .get(input.sessionId) - if (recorded === undefined || queuePauseHoldsBack(db, input)) { - return 0 - } - return Number( - db.prepare('DELETE FROM queued_message_pauses WHERE session_id = ?').run(input.sessionId) - .changes ?? 0 - ) -} diff --git a/src/main/native-chat/agent-session-journal/queued-message-pause.test.ts b/src/main/native-chat/agent-session-journal/queued-message-pause.test.ts new file mode 100644 index 00000000000..b8e45e0ef59 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/queued-message-pause.test.ts @@ -0,0 +1,608 @@ +// The queue's pause is a pure function of the journal: a Stop and a Resume are rows, a person's +// accepted turn is a row, and a /clear's carried card names its source. Nothing is stored beside +// them, so nothing has to retire. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import type { + AgentJournalMessageItem, + AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' +import Database from '../../sqlite/sync-database' +import { journalDatabasePath } from './journal-host-database' +import { + createTrackedJournalOpener, + liveTestJournalRows, + updateTestJournalRowJson +} from './journal-host-database-test-support' +import { QueuedMessageNotConsumableError } from './journal-queued-messages' +import { applyJournalRow, createJournalReducerState } from './journal-reducer' +import { parseJournalRow } from './journal-row-schema' +import type { AgentSessionJournal } from './journal-store' +import { + deriveQueuePauses, + nextSendableQueuedCard, + queuePauseHolding, + resumableQueuePause +} from './queued-message-pause' +import { agentSessionFailureFact } from '../../../shared/agent-session-failure' +import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' + +const IDENTITY: AgentSessionJournalIdentity = { + sessionId: 'session-p', + workspaceId: 'ws-1', + hostId: 'host-1', + agent: 'claude', + providerHandle: { kind: 'claude', sessionId: 'native-1', leafUuid: null } +} +const HOST = 'proc-1' + +let root: string +let clock = 1_000 +const journals = createTrackedJournalOpener() + +function message(text: string): AgentJournalMessageItem { + return { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] } +} + +function open(): Promise { + return journals.open({ + identity: IDENTITY, + stateDirectory: root, + now: () => ++clock, + mintEpoch: () => `epoch-${clock}` + }) +} + +function queueDraft(journal: AgentSessionJournal, messageId: string, carriedFrom?: string) { + return journal.queuedMessages.insert({ + messageId, + body: message(messageId), + fingerprint: `fp-${messageId}`, + hostInstance: HOST, + ...(carriedFrom ? { carriedFrom } : {}) + }) +} + +/** A turn sent, then accepted by the provider; `origin` says who asked for it. */ +async function turn( + journal: AgentSessionJournal, + id: string, + origin: 'client' | 'host', + accept = true +): Promise { + await journal.appendSubmission({ + clientMessageId: id, + origin, + payloadFingerprint: `fp-${id}`, + body: message(id), + fence: 0, + handoverRecorded: true + }) + if (accept) { + await acceptTurn(journal, id) + } +} + +function acceptTurn(journal: AgentSessionJournal, id: string) { + return journal.resolveDispatch({ + clientMessageId: id, + state: 'accepted', + providerIdentity: { provider: 'claude', sessionId: 'native-1', uuid: `echo-${id}` }, + fence: 0 + }) +} + +/** A person's Stop taking effect. */ +function userStop(journal: AgentSessionJournal) { + return journal.appendStopEvent({ reason: 'user-stop' }, 0) +} + +function reason(journal: AgentSessionJournal): string | null { + return journal.queuedMessages.pauses(HOST)[0]?.reason ?? null +} + +/** Each card, and whether a pause in force holds it. */ +function held(journal: AgentSessionJournal): [string, boolean][] { + const pauses = journal.queuedMessages.pauses(HOST) + return journal.queuedMessages + .list() + .filter((card) => card.state === 'waiting') + .map((card) => [card.messageId, queuePauseHolding(pauses, card) !== undefined]) +} + +/** The Stop events stored in the live epoch, oldest first. */ +function stopEvents(): unknown[] { + const db = new Database(journalDatabasePath(root), { readonly: true }) + try { + return liveTestJournalRows(db, IDENTITY.sessionId).flatMap((stored) => { + const parsed = parseJournalRow(stored.rowJson) + return parsed.ok && parsed.row.kind === 'tombstone' && parsed.row.stopEvent + ? [parsed.row.stopEvent] + : [] + }) + } finally { + db.close() + } +} + +/** Rewrites one key of each row at a sequence, as a corrupt write would leave it. */ +function rewriteRows(keys: Record): void { + const db = new Database(journalDatabasePath(root)) + try { + for (const stored of liveTestJournalRows(db, IDENTITY.sessionId)) { + const rewrite = keys[stored.seq] + if (rewrite) { + const row: Record = JSON.parse(stored.rowJson) + row[rewrite[0]] = rewrite[1] + updateTestJournalRowJson(db, IDENTITY.sessionId, stored.seq, JSON.stringify(row)) + } + } + } finally { + db.close() + } +} + +/** Tables that could store a pause: none, the journal rows are the only record. */ +function pauseTables(): number { + const db = new Database(journalDatabasePath(root), { readonly: true }) + try { + return db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE '%pause%'") + .all().length + } finally { + db.close() + } +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-queue-pause-')) + clock = 1_000 +}) + +afterEach(async () => { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +describe("the queue's pause, derived from the journal", () => { + it("a person's Stop event pauses the queue, even with no card yet, survives reopen, and stores nothing", async () => { + let journal = await open() + await userStop(journal) + expect(reason(journal)).toBe('stopped') + await queueDraft(journal, 'draft-1') + await journal.close() + journal = await open() + expect(reason(journal)).toBe('stopped') + expect(pauseTables()).toBe(0) + }) + + it("only a person's turn sent after the Stop and accepted lifts it; host turns never do", async () => { + const journal = await open() + await turn(journal, 'before-stop', 'client', false) + await userStop(journal) + // Sent before the Stop: its acceptance now does not end a Stop that came after it. + await acceptTurn(journal, 'before-stop') + await turn(journal, 'mail', 'host') + expect(reason(journal)).toBe('stopped') + await turn(journal, 'typed', 'client', false) + expect(reason(journal)).toBe('stopped') + await acceptTurn(journal, 'typed') + expect(reason(journal)).toBeNull() + }) + + it('a later Stop is the latest, and a Resume row lifts it', async () => { + const journal = await open() + await userStop(journal) + await turn(journal, 'typed', 'client') + expect(reason(journal)).toBeNull() + await userStop(journal) + expect(reason(journal)).toBe('stopped') + await journal.appendQueueResume(0) + expect(reason(journal)).toBeNull() + await userStop(journal) + expect(reason(journal)).toBe('stopped') + expect(pauseTables()).toBe(0) + }) + + it("a card /clear carried in pauses the replacement 'cleared' until a Resume there", async () => { + let journal = await open() + await queueDraft(journal, 'carried', 'source-session') + expect(reason(journal)).toBe('cleared') + await journal.close() + journal = await open() + expect(reason(journal)).toBe('cleared') + await journal.appendQueueResume(0) + expect(reason(journal)).toBeNull() + expect(pauseTables()).toBe(0) + }) + + it("a person's accepted turn on the replacement lifts 'cleared'; a host turn does not", async () => { + const journal = await open() + await queueDraft(journal, 'carried', 'source-session') + await turn(journal, 'launch', 'host') + expect(reason(journal)).toBe('cleared') + await turn(journal, 'typed', 'client') + expect(reason(journal)).toBeNull() + }) + + it('rides a tombstone of an id no item takes, so an older build reads it and changes nothing', async () => { + const journal = await open() + await turn(journal, 'typed', 'client') + await journal.appendStopEvent({ reason: 'user-stop', turnId: 'turn-1', caller: 'client-1' }, 0) + const db = new Database(journalDatabasePath(root), { readonly: true }) + const stored = liveTestJournalRows(db, IDENTITY.sessionId) + db.close() + const parsed = parseJournalRow(stored.at(-1)?.rowJson ?? '') + expect(parsed).toMatchObject({ + ok: true, + row: { + kind: 'tombstone', + stopEvent: { reason: 'user-stop', turnId: 'turn-1', caller: 'client-1' } + } + }) + if (!parsed.ok || parsed.row.kind !== 'tombstone') { + throw new Error('expected a tombstone row') + } + expect(parsed.row.stopEvent?.at).toBe(parsed.row.ts) + // An older build ignores the unknown key: the row is an ordinary removal of nothing. + const { stopEvent: _ignored, ...asOlderBuildReadsIt } = parsed.row + const state = createJournalReducerState(IDENTITY.sessionId, parsed.row.epoch) + for (const row of stored.slice(0, -1)) { + const earlier = parseJournalRow(row.rowJson) + if (earlier.ok) { + applyJournalRow(state, earlier.row) + } + } + const items = [...state.items.keys()] + const submissions = [...state.submissions.keys()] + applyJournalRow(state, asOlderBuildReadsIt) + expect([...state.items.keys()]).toEqual(items) + expect([...state.submissions.keys()]).toEqual(submissions) + }) + + it('a Stop or Resume row holding a value no build writes is ignored on read, never trusted', async () => { + let journal = await open() + await queueDraft(journal, 'held') + /** Appends a row, then reopens with one of its keys rewritten as a corrupt write would. */ + const corrupted = async ( + append: Promise<{ sequence: number }>, + key: string, + value: unknown + ) => { + const { sequence } = await append + await journal.close() + rewriteRows({ [sequence]: [key, value] }) + journal = await open() + } + const malformed = [null, 'stopped', { at: 1 }] + // Pauses nothing as the latest Stop row. + for (const value of malformed) { + await corrupted(userStop(journal), 'stopEvent', value) + expect(reason(journal)).toBeNull() + } + // Ends nothing after a person's Stop: neither as a later Stop nor as a Resume. + await userStop(journal) + for (const value of malformed) { + await corrupted(userStop(journal), 'stopEvent', value) + expect(reason(journal)).toBe('stopped') + } + await corrupted(journal.appendQueueResume(0), 'queueResume', 'yes') + expect(reason(journal)).toBe('stopped') + expect(held(journal)).toEqual([['held', true]]) + }) + + it("the queue's own consume is refused in its transaction while the pause holds the card; Send-now is not", async () => { + const journal = await open() + await queueDraft(journal, 'draft-1') + await userStop(journal) + const consume = (id: string, automatic: boolean) => + journal.appendSubmission( + { + clientMessageId: id, + origin: automatic ? 'host' : 'client', + payloadFingerprint: 'fp-draft-1', + body: message('draft-1'), + fence: 0, + handoverRecorded: true + }, + { + messageId: 'draft-1', + expect: 'waiting', + settledByOp: null, + hostInstance: HOST, + ...(automatic ? { yieldsToPause: { hostInstance: HOST } } : {}) + } + ) + await expect(consume('drain-1', true)).rejects.toBeInstanceOf(QueuedMessageNotConsumableError) + expect(journal.submission('drain-1')).toBeUndefined() + expect(journal.queuedMessages.get('draft-1')?.state).toBe('waiting') + await consume('send-now-1', false) + expect(journal.queuedMessages.get('draft-1')?.state).toBe('dispatched') + }) + + it("a rewind's epoch replacement restates a Stop still pausing, and not one a person ended", async () => { + const journal = await open() + await userStop(journal) + await journal.replaceEpochItems('handle_forked', 0, []) + expect(reason(journal)).toBe('stopped') + await turn(journal, 'typed', 'client') + await journal.replaceEpochItems('handle_forked', 0, []) + expect(reason(journal)).toBeNull() + }) + + it("only a person's Stop pauses: a host's, a close's or an unknown reason holds nothing", async () => { + const journal = await open() + await queueDraft(journal, 'draft-1') + for (const reason of ['user-close', 'host-stop', 'evict'] as const) { + await journal.appendStopEvent({ reason }, 0) + } + // A newer build's reason this one does not know. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: deliberately outside the type, as a newer build could write it. + await journal.appendStopEvent({ reason: 'future-reason' as 'evict' }, 0) + expect(reason(journal)).toBeNull() + }) + + it("a later Stop of any reason ends a person's Stop, without pausing itself", async () => { + const journal = await open() + await queueDraft(journal, 'draft-1') + for (const later of ['host-stop', 'evict', 'user-close'] as const) { + await userStop(journal) + expect(held(journal)).toEqual([['draft-1', true]]) + await journal.appendStopEvent({ reason: later }, 0) + expect(reason(journal)).toBeNull() + expect(held(journal)).toEqual([['draft-1', false]]) + } + }) + + it("a person's Stop with no later Stop event is not lifted", async () => { + const journal = await open() + await queueDraft(journal, 'held') + await userStop(journal) + // An eviction writer must write only when it ends a running turn, or it would lift this pause. + expect(reason(journal)).toBe('stopped') + expect(held(journal)).toEqual([['held', true]]) + }) + + it('the restated Stop is the same event: its reason, turn, caller and time', async () => { + const journal = await open() + await journal.appendStopEvent({ reason: 'user-stop', turnId: 'turn-7', caller: 'phone' }, 0) + const stopped = stopEvents() + await journal.replaceEpochItems('handle_forked', 0, []) + expect(stopEvents()).toEqual(stopped) + expect(stopped).toEqual([ + { reason: 'user-stop', turnId: 'turn-7', caller: 'phone', at: expect.any(Number) } + ]) + }) + + it.each([ + ["a person's turn", (journal: AgentSessionJournal) => turn(journal, 'typed', 'client')], + ['a Resume', (journal: AgentSessionJournal) => journal.appendQueueResume(0)] + ])('a /clear pause %s already lifted stays lifted across a rewind', async (_name, lift) => { + const journal = await open() + await queueDraft(journal, 'carried', 'source-session') + await lift(journal) + expect(reason(journal)).toBeNull() + await journal.replaceEpochItems('handle_forked', 0, []) + expect(reason(journal)).toBeNull() + }) + + it('a lifted /clear pause and a later Stop are both restated, the Stop still in force', async () => { + const journal = await open() + await queueDraft(journal, 'carried', 'source-session') + await turn(journal, 'typed', 'client') + await userStop(journal) + await journal.replaceEpochItems('handle_forked', 0, []) + expect(journal.queuedMessages.pauses(HOST).map((pause) => pause.reason)).toEqual(['stopped']) + await journal.appendQueueResume(0) + expect(reason(journal)).toBeNull() + }) +}) + +describe('which cards a pause holds', () => { + it('only cards queued before the Stop event: one queued after it is a new instruction', async () => { + let journal = await open() + await queueDraft(journal, 'before') + await userStop(journal) + await queueDraft(journal, 'after') + expect(held(journal)).toEqual([ + ['before', true], + ['after', false] + ]) + await journal.close() + journal = await open() + expect(held(journal)).toEqual([ + ['before', true], + ['after', false] + ]) + }) + + it('a steer the Stop withdrew comes back in its own place, and is held', async () => { + const journal = await open() + await queueDraft(journal, 'steered') + await journal.appendSubmission( + { + clientMessageId: 'send-now-1', + origin: 'client', + payloadFingerprint: 'fp-steered', + body: message('steered'), + fence: 0, + handoverRecorded: true + }, + { messageId: 'steered', expect: 'waiting', settledByOp: null, hostInstance: HOST } + ) + await userStop(journal) + await queueDraft(journal, 'after') + await journal.resolveDispatch({ + clientMessageId: 'send-now-1', + state: 'rejected', + ...agentSessionFailureWords(agentSessionFailureFact('cancelled'), { surface: 'rejection' }), + fence: 0 + }) + expect(held(journal)).toEqual([ + ['steered', true], + ['after', false] + ]) + }) + + it("the queue's own consume refuses a newer card while an older one is held: nothing overtakes", async () => { + const journal = await open() + await queueDraft(journal, 'held') + await userStop(journal) + await queueDraft(journal, 'newer') + await expect( + journal.appendSubmission( + { + clientMessageId: 'drain-newer', + origin: 'host', + payloadFingerprint: 'fp-newer', + body: message('newer'), + fence: 0, + handoverRecorded: true + }, + { + messageId: 'newer', + expect: 'waiting', + settledByOp: null, + hostInstance: HOST, + yieldsToPause: { hostInstance: HOST } + } + ) + ).rejects.toBeInstanceOf(QueuedMessageNotConsumableError) + expect(journal.queuedMessages.get('newer')?.state).toBe('waiting') + }) + + it("'cleared' holds the carried cards, not one typed after them", async () => { + const journal = await open() + await queueDraft(journal, 'carried-1', 'source-session') + await queueDraft(journal, 'carried-2', 'source-session') + await queueDraft(journal, 'typed-here') + expect(held(journal)).toEqual([ + ['carried-1', true], + ['carried-2', true], + ['typed-here', false] + ]) + }) +}) + +describe("a restart's pause", () => { + it("adopting a restart's rows moves them into this instance and clears an older build's stored 'stopped' hold; send_failed stays", async () => { + const journal = await open() + await journal.queuedMessages.insert({ + messageId: 'draft-restart', + body: message('written before the restart'), + fingerprint: 'fp-draft-restart', + hostInstance: 'proc-0' + }) + expect(reason(journal)).toBe('restarted') + await queueDraft(journal, 'draft-legacy') + await queueDraft(journal, 'draft-failed') + await journal.queuedMessages.hold({ messageIds: ['draft-failed'], reason: 'send_failed' }) + const db = new Database(journalDatabasePath(root)) + db.prepare("UPDATE queued_messages SET hold_reason = 'stopped' WHERE message_id = ?").run( + 'draft-legacy' + ) + db.close() + journal.queuedMessages.invalidate() + expect(await journal.queuedMessages.adopt(HOST)).toBe(true) + expect(reason(journal)).toBeNull() + expect( + journal.queuedMessages.list().map((row) => [row.messageId, row.hostInstance, row.holdReason]) + ).toEqual([ + ['draft-restart', HOST, null], + ['draft-legacy', HOST, null], + ['draft-failed', HOST, 'send_failed'] + ]) + }) + + it('an adoption with nothing to adopt changes nothing and fires no commit notification', async () => { + const journal = await open() + await queueDraft(journal, 'draft-1') + const revision = journal.queuedMessages.revision() + expect(await journal.queuedMessages.adopt(HOST)).toBe(false) + expect(journal.queuedMessages.revision()).toBe(revision) + }) +}) + +describe('which cards the pauses in force hold', () => { + type Card = Parameters[1] & { messageId: string } + const DEAD = 'proc-0' + + function card(messageId: string, queuedAfter: number, fields: Partial = {}): Card { + const queuedAt = { epoch: 'epoch-1', sequence: queuedAfter + 1 } + const base = { state: 'waiting', holdReason: null, hostInstance: HOST, carriedFrom: null } + return { messageId, ...base, queuedAt, ...fields } + } + + /** A Stop at sequence 5 unless `stopped` is 0; no person's turn or Resume since. */ + function pausesOver(cards: readonly Card[], stopped = 5) { + return deriveQueuePauses({ + epoch: 'epoch-1', + marks: { + latestStop: stopped ? { sequence: stopped, event: { reason: 'user-stop', at: 0 } } : null, + resumedSequence: 0 + }, + latestPersonTurnSequence: 0, + cards, + hostInstance: HOST, + restartEnded: false + }) + } + + function holding(cards: readonly Card[], stopped?: number): [string, string | null][] { + const pauses = pausesOver(cards, stopped) + return cards.map((each) => [each.messageId, queuePauseHolding(pauses, each)?.reason ?? null]) + } + + it('a Stop holds a card with no recorded position, and one queued before a rewind', () => { + const cards = [ + card('unrecorded', 5, { queuedAt: null }), + // Later in its own epoch than the Stop is in this one: only the epoch says it came first. + card('before-rewind', 5, { queuedAt: { epoch: 'epoch-0', sequence: 9 } }), + card('after', 5) + ] + expect(holding(cards)).toEqual([ + ['unrecorded', 'stopped'], + ['before-rewind', 'stopped'], + ['after', null] + ]) + }) + + it("a Stop that holds nothing never hides a restart's: a dead process's card queued after it waits", () => { + const after = card('after', 5, { hostInstance: DEAD }) + expect(pausesOver([after]).map((pause) => pause.reason)).toEqual(['stopped', 'restarted']) + expect(holding([after])).toEqual([['after', 'restarted']]) + expect(nextSendableQueuedCard(pausesOver([after]), [after])).toBeNull() + expect(resumableQueuePause(pausesOver([after]), [after])?.reason).toBe('restarted') + // Written by this process, nothing holds it: a card queued after a Stop sends normally. + const live = card('after', 5) + expect(nextSendableQueuedCard(pausesOver([live]), [live])).toBe(live) + }) + + it("a card names the first pause holding it, and the header names the first held card's", () => { + const cards = [ + card('before', 3, { hostInstance: DEAD }), + card('after', 5, { hostInstance: DEAD }) + ] + expect(holding(cards)).toEqual([ + ['before', 'stopped'], + ['after', 'restarted'] + ]) + expect(resumableQueuePause(pausesOver(cards), cards)?.reason).toBe('stopped') + }) + + it("a /clear's pause that holds nothing never hides a restart's", () => { + const cards = [ + card('carried', 1, { carriedFrom: 'source-session', holdReason: 'send_failed' }), + card('typed', 2, { hostInstance: DEAD }) + ] + expect(pausesOver(cards, 0).map((pause) => pause.reason)).toEqual(['cleared', 'restarted']) + expect(holding(cards, 0)).toEqual([ + ['carried', null], + ['typed', 'restarted'] + ]) + expect(nextSendableQueuedCard(pausesOver(cards, 0), cards)).toBeNull() + }) +}) diff --git a/src/main/native-chat/agent-session-journal/queued-message-pause.ts b/src/main/native-chat/agent-session-journal/queued-message-pause.ts new file mode 100644 index 00000000000..9039760ba43 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/queued-message-pause.ts @@ -0,0 +1,203 @@ +// The queue's pauses: a pure function of the journal fold and the cards, never a stored flag, so +// nothing has to retire them. Each holds its own cards (`queuePauseHolding`). In force when: +// - 'stopped': the latest Stop event is a person's (reason `user-stop`), with no later Resume row +// and no turn a person asked for sent after it and accepted. A later Stop of any reason +// supersedes it; only a person's pauses. +// - 'cleared': a card /clear carried into this conversation waits, and no person's turn or +// Resume has happened here since. +// - 'restarted': a waiting card was written by another host process, and no person's turn has +// started since this conversation opened. +// A person's turn is an accepted submission of origin `client`. Orchestration mail, a restart +// continuation, a launch prompt and the queue's own drain are `host` and never lift it. + +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import type { JournalStopEvent, JournalTombstoneRow } from './journal-row-schema' + +export type QueuePauseReason = 'stopped' | 'cleared' | 'restarted' + +/** The latest Stop event, whatever its reason, and the latest Resume row, folded by the reducer. */ +export type JournalQueuePauseMarks = { + latestStop: { sequence: number; event: JournalStopEvent } | null + /** 0 when none. */ + resumedSequence: number +} + +export type DerivedQueuePause = { + reason: QueuePauseReason + /** Where a Stop's pause began: a card queued at or after it is newer. Null for the others. */ + since: AgentJournalCursor | null +} + +type QueueCard = { + state: string + holdReason: string | null + hostInstance: string + carriedFrom: string | null + queuedAt: AgentJournalCursor | null +} + +export function createJournalQueuePauseMarks(): JournalQueuePauseMarks { + return { latestStop: null, resumedSequence: 0 } +} + +/** The keys are read from disk unchecked: a value no build writes (a corrupt row) is ignored. */ +function isReadableStopEvent(value: unknown): value is JournalStopEvent { + return ( + typeof value === 'object' && + value !== null && + 'reason' in value && + typeof value.reason === 'string' && + 'at' in value && + typeof value.at === 'number' && + Number.isFinite(value.at) + ) +} + +export function foldJournalQueuePauseMark( + marks: JournalQueuePauseMarks, + row: JournalTombstoneRow +): void { + if (isReadableStopEvent(row.stopEvent)) { + marks.latestStop = { sequence: row.seq, event: row.stopEvent } + } else if (row.queueResume === true) { + marks.resumedSequence = row.seq + } +} + +/** A person's Stop still pauses: it is the latest Stop, and nothing a person did since, and no + * Resume, ended it. A later Stop for any other reason ends it without pausing itself. */ +export function journalQueueStopHolds( + marks: JournalQueuePauseMarks, + latestPersonTurnSequence: number +): boolean { + return ( + marks.latestStop?.event.reason === 'user-stop' && + marks.latestStop.sequence > Math.max(latestPersonTurnSequence, marks.resumedSequence) + ) +} + +/** The latest person's Stop while it still pauses, with where it was written; else null. */ +export function journalUserStopInForce( + marks: JournalQueuePauseMarks, + latestPersonTurnSequence: number +): JournalQueuePauseMarks['latestStop'] { + return journalQueueStopHolds(marks, latestPersonTurnSequence) ? marks.latestStop : null +} + +/** What a rewind's new epoch restates so its pauses read as they did: a lift of /clear's pause (a + * person's turn or a Resume happened), then the Stop still in force, in that order so the lift + * never ends the Stop. */ +export type JournalQueuePauseRestatement = { + lifted: boolean + liveStop: JournalStopEvent | null +} + +export function journalQueuePauseRestatement( + marks: JournalQueuePauseMarks, + latestPersonTurnSequence: number +): JournalQueuePauseRestatement { + return { + lifted: latestPersonTurnSequence > 0 || marks.resumedSequence > 0, + liveStop: journalUserStopInForce(marks, latestPersonTurnSequence)?.event ?? null + } +} + +/** Every pause in force, in the order a card held by several names its reason. */ +export function deriveQueuePauses(input: { + /** The journal's epoch: sequences compare only within one. */ + epoch: string + marks: JournalQueuePauseMarks + latestPersonTurnSequence: number + cards: readonly QueueCard[] + hostInstance: string + /** A person's turn started since this conversation opened. */ + restartEnded: boolean +}): DerivedQueuePause[] { + const { epoch, marks, latestPersonTurnSequence } = input + const pauses: DerivedQueuePause[] = [] + const stop = journalUserStopInForce(marks, latestPersonTurnSequence) + if (stop) { + pauses.push({ reason: 'stopped', since: { epoch, sequence: stop.sequence } }) + } + const waiting = input.cards.filter((card) => card.state === 'waiting') + const carried = waiting.filter((card) => card.carriedFrom !== null) + if (carried.length > 0 && latestPersonTurnSequence === 0 && marks.resumedSequence === 0) { + pauses.push({ reason: 'cleared', since: null }) + } + if (!input.restartEnded && waiting.some((card) => card.hostInstance !== input.hostInstance)) { + // The process that wrote a card is gone: every card waits, whenever it was written. + pauses.push({ reason: 'restarted', since: null }) + } + return pauses +} + +/** Queued before the pause began: for /clear, a card it carried; for a restart, every card. For a + * Stop, a card queued before its row; one from another epoch (before a rewind) or from a build + * that recorded no position counts as before. A withdrawn steer keeps its position, so is held. */ +function queuedBeforePause(pause: DerivedQueuePause, card: QueueCard): boolean { + if (pause.reason === 'cleared') { + return card.carriedFrom !== null + } + const { since } = pause + return ( + since === null || + card.queuedAt === null || + card.queuedAt.epoch !== since.epoch || + card.queuedAt.sequence < since.sequence + ) +} + +// Product decision: a card queued AFTER a Stop is a new instruction and is not held; only cards +// queued before it, and a steer it withdrew, wait. It still never jumps ahead of a held card: the +// drain stops at the first one. true instead holds every waiting card, whenever it was queued. +const PAUSE_HOLDS_CARDS_QUEUED_AFTER_IT = false + +/** THE rule for which cards are held: by ANY pause in force, named by the first that holds it, so + * a Stop's that holds nothing never hides a restart's. The drain, its consume, and publication + * all read it. */ +export function queuePauseHolding( + pauses: readonly DerivedQueuePause[], + card: QueueCard +): DerivedQueuePause | undefined { + if (card.state !== 'waiting' || card.holdReason !== null) { + return undefined + } + return pauses.find((pause) => PAUSE_HOLDS_CARDS_QUEUED_AFTER_IT || queuedBeforePause(pause, card)) +} + +/** The card the queue sends next: the oldest waiting one with no hold of its own, unless a + * returned card or a held one comes first. The queue never reorders, so a newer card never + * overtakes a held one. The drain's pick and its consume both read this. */ +export function nextSendableQueuedCard( + pauses: readonly DerivedQueuePause[], + cards: readonly T[] +): T | null { + for (const card of cards) { + if (card.state === 'returned' || queuePauseHolding(pauses, card)) { + return null + } + if (card.state === 'waiting' && card.holdReason === null) { + return card + } + } + return null +} + +/** The pause to PUBLISH: the one holding the first card Resume would send, not behind a returned + * card, which blocks everything after it until the user acts. None otherwise, so its header + * never offers a Resume that sends nothing. */ +export function resumableQueuePause( + pauses: readonly DerivedQueuePause[], + cards: readonly QueueCard[] +): DerivedQueuePause | null { + for (const card of cards) { + if (card.state === 'returned') { + return null + } + const holding = queuePauseHolding(pauses, card) + if (holding) { + return holding + } + } + return null +} diff --git a/src/main/native-chat/agent-session-journal/queued-message-schema.ts b/src/main/native-chat/agent-session-journal/queued-message-schema.ts index 31806e6b71a..3f9d0122fda 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-schema.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-schema.ts @@ -9,7 +9,10 @@ const NULLABLE_COLUMNS: readonly (readonly [name: string, type: string])[] = [ ['returned_rejection', 'TEXT'], ['settled_at', 'INTEGER'], ['settled_by_op', 'TEXT'], - ['consumed_as', 'TEXT'] + ['consumed_as', 'TEXT'], + ['carried_from', 'TEXT'], + ['queued_epoch', 'TEXT'], + ['queued_sequence', 'INTEGER'] ] /** @@ -37,6 +40,9 @@ CREATE TABLE IF NOT EXISTS queued_messages ( settled_at INTEGER, settled_by_op TEXT, consumed_as TEXT, + carried_from TEXT, + queued_epoch TEXT, + queued_sequence INTEGER, PRIMARY KEY (session_id, message_id) ); `) @@ -61,12 +67,5 @@ CREATE TABLE IF NOT EXISTS queued_messages ( db.exec(` CREATE UNIQUE INDEX IF NOT EXISTS queued_messages_consumed_as ON queued_messages (session_id, consumed_as) WHERE consumed_as IS NOT NULL; -CREATE TABLE IF NOT EXISTS queued_message_pauses ( - session_id TEXT PRIMARY KEY, - reason TEXT NOT NULL, - epoch TEXT NOT NULL, - sequence INTEGER NOT NULL, - recorded_at INTEGER NOT NULL -); `) } diff --git a/src/main/native-chat/agent-session-journal/queued-message-settlement.ts b/src/main/native-chat/agent-session-journal/queued-message-settlement.ts index 75e62256780..8d2e9ec0acc 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-settlement.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-settlement.ts @@ -8,8 +8,7 @@ import type Database from '../../sqlite/sync-database' import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import { consumedSubmissionWasRejected, - journalDispatchRowNewlyRejects, - rejectedDraftSettlement + journalDispatchRowNewlyRejects } from './journal-dispatch-settlement' import type { JournalReducerState } from './journal-reducer' import type { JournalRow } from './journal-row-schema' @@ -36,20 +35,6 @@ export function queuedMessageSettlementOwed( ) } -/** A dispatched draft's consumed submission settled so that the draft is owed a - * return to waiting (a withdrawal, not a refusal): what a queue pause must still - * count as a card it holds back while that settlement is owed. */ -export function owedBackToWaiting(submissions: Submissions): (consumedRef: string) => boolean { - return (consumedRef) => { - const submission = submissions.get(consumedRef) - return ( - submission !== undefined && - consumedSubmissionWasRejected(submission) && - rejectedDraftSettlement(submission).state === 'waiting' - ) - } -} - /** Applies each owed settlement, and withdraws each waiting draft an applied echo proves * delivered (`draftsDeliveredByAppliedEcho`); returns how many drafts changed. */ export function settleOwedQueuedMessages( diff --git a/src/main/native-chat/agent-session-journal/queued-message-store.test.ts b/src/main/native-chat/agent-session-journal/queued-message-store.test.ts index 0e42a22be4d..c3910e46685 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-store.test.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-store.test.ts @@ -685,138 +685,6 @@ describe('holds', () => { }) }) -describe("the queue's Stop fact", () => { - it('records where the Stop took effect, survives reopen, and the latest Stop replaces an earlier one', async () => { - let journal = await open() - await queueDraft(journal, 'draft-1') - await journal.queuedMessages.recordPause('stopped') - const first = journal.queuedMessages.pause() - expect(first).toMatchObject({ reason: 'stopped', sequence: journal.cursor().sequence }) - await journal.appendItem( - { provider: 'orca', clientMessageId: 'later' }, - { kind: 'status', text: 'later' }, - { fence: 0, turnScope: AGENT_JOURNAL_THREAD_SCOPE } - ) - await journal.queuedMessages.recordPause('stopped') - expect(journal.queuedMessages.pause()?.sequence).toBe((first?.sequence ?? 0) + 1) - await journal.close() - journal = await open() - expect(journal.queuedMessages.pause()?.sequence).toBe((first?.sequence ?? 0) + 1) - // The pause is the queue's, never a row's. - expect(journal.queuedMessages.get('draft-1')?.holdReason).toBeNull() - }) - - it("a /clear's replacement records its pause as 'cleared', read back the same way", async () => { - let journal = await open() - await queueDraft(journal, 'draft-1') - await journal.queuedMessages.recordPause('cleared') - await journal.close() - journal = await open() - expect(journal.queuedMessages.pause()).toMatchObject({ reason: 'cleared' }) - }) - - it('retires in the write that takes the last card it holds back: a hold of its own', async () => { - const journal = await open() - await queueDraft(journal, 'draft-held') - expect(await journal.queuedMessages.recordPause('stopped')).toBe(true) - await journal.queuedMessages.hold({ messageIds: ['draft-held'], reason: 'send_failed' }) - expect(journal.queuedMessages.pause()).toBeNull() - }) - - it('records nothing over a queue with no card it holds back, judged in its own transaction', async () => { - const journal = await open() - expect(await journal.queuedMessages.recordPause('stopped')).toBe(false) - expect(journal.queuedMessages.pause()).toBeNull() - await queueDraft(journal, 'draft-1') - expect(await journal.queuedMessages.recordPause('stopped')).toBe(true) - expect(journal.queuedMessages.pause()).not.toBeNull() - }) - - it('a hand-off whose return to waiting is still owed (its hook skipped) keeps the pause', async () => { - const journal = await open() - await queueDraft(journal, 'draft-sent') - await queueDraft(journal, 'draft-other') - await consumeDraft(journal, 'draft-sent') - expect(await journal.queuedMessages.recordPause('stopped')).toBe(true) - const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) - const hook = vi - .spyOn(JournalQueuedMessages.prototype, 'onRowInTransaction') - .mockImplementationOnce(() => { - throw new Error('bookkeeping failed') - }) - try { - await journal.rejectQueuedSubmissions(0, STOP_WITHDRAWAL) - } finally { - hook.mockRestore() - warn.mockRestore() - } - expect(journal.queuedMessages.get('draft-sent')?.state).toBe('dispatched') - // The only waiting card goes; the owed one is still a card the pause holds back. - await journal.queuedMessages.withdraw({ messageIds: ['draft-other'], settledByOp: 'c\u0000op' }) - expect(journal.queuedMessages.pause()).not.toBeNull() - await journal.queuedMessages.settleOwed() - expect(journal.queuedMessages.get('draft-sent')?.state).toBe('waiting') - expect(journal.queuedMessages.pause()).not.toBeNull() - }) - - it('lifting retires only the Stop fact it judged, never one recorded since', async () => { - const journal = await open() - await queueDraft(journal, 'draft-1') - await journal.queuedMessages.recordPause('stopped') - const judged = journal.queuedMessages.pause() - await journal.appendItem( - { provider: 'orca', clientMessageId: 'later' }, - { kind: 'status', text: 'later' }, - { fence: 0, turnScope: AGENT_JOURNAL_THREAD_SCOPE } - ) - await journal.queuedMessages.recordPause('stopped') - expect(await journal.queuedMessages.liftPause({ stop: judged, adoptInto: null })).toBe(false) - expect(journal.queuedMessages.pause()).not.toBeNull() - expect( - await journal.queuedMessages.liftPause({ - stop: journal.queuedMessages.pause(), - adoptInto: null - }) - ).toBe(true) - expect(journal.queuedMessages.pause()).toBeNull() - }) - - it("adopting a restart's rows moves them into this instance and clears an older build's stored 'stopped' hold; send_failed stays", async () => { - const journal = await open() - await journal.queuedMessages.insert({ - messageId: 'draft-restart', - body: message('written before the restart'), - fingerprint: 'fp-draft-restart', - hostInstance: 'proc-0' - }) - await queueDraft(journal, 'draft-legacy') - await queueDraft(journal, 'draft-failed') - await journal.queuedMessages.hold({ messageIds: ['draft-failed'], reason: 'send_failed' }) - const db = new Database(journalDatabasePath(root)) - db.prepare("UPDATE queued_messages SET hold_reason = 'stopped' WHERE message_id = ?").run( - 'draft-legacy' - ) - db.close() - journal.queuedMessages.invalidate() - expect(await journal.queuedMessages.liftPause({ stop: null, adoptInto: 'proc-1' })).toBe(true) - expect( - journal.queuedMessages.list().map((row) => [row.messageId, row.hostInstance, row.holdReason]) - ).toEqual([ - ['draft-restart', 'proc-1', null], - ['draft-legacy', 'proc-1', null], - ['draft-failed', 'proc-1', 'send_failed'] - ]) - }) - - it('a lift with nothing to lift changes nothing and fires no commit notification', async () => { - const journal = await open() - await queueDraft(journal, 'draft-1') - const revision = journal.queuedMessages.revision() - expect(await journal.queuedMessages.liftPause({ stop: null, adoptInto: 'proc-1' })).toBe(false) - expect(journal.queuedMessages.revision()).toBe(revision) - }) -}) - describe('the commit listener', () => { it('draft-table writes fire it exactly when rows changed, so no caller publishes by hand', async () => { const journal = await open() diff --git a/src/main/native-chat/agent-session-journal/queued-message-table.ts b/src/main/native-chat/agent-session-journal/queued-message-table.ts index ffc9b240987..a2bd9c3e05a 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-table.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-table.ts @@ -9,7 +9,10 @@ import type Database from '../../sqlite/sync-database' import type { UnreadAgentSessionFailureFact } from '../../../shared/agent-session-failure' -import type { AgentJournalMessageItem } from '../../../shared/agent-session-journal-types' +import type { + AgentJournalCursor, + AgentJournalMessageItem +} from '../../../shared/agent-session-journal-types' import { rejectedDraftSettlement } from './journal-dispatch-settlement' import { readStoredRejectionFact } from './journal-dispatch-reducer' @@ -50,10 +53,16 @@ export type QueuedMessageRow = { * card, cleared when a withdrawal sends it back to waiting. Host-only; the published link is * the submission's `queuedMessageId`. */ consumedAs: string | null + /** The conversation /clear carried this card from; null for a card written here. What the + * replacement's 'cleared' pause is derived from. */ + carriedFrom: string | null + /** Where the journal stood when it was queued: a Stop's pause holds only cards queued before + * it. Null on rows from builds before it was recorded, which read as queued before any Stop. */ + queuedAt: AgentJournalCursor | null } const COLUMNS = - 'session_id, message_id, position, body_json, fingerprint, created_at, host_instance, state, hold_reason, returned_reason, returned_rejection, settled_at, settled_by_op, consumed_as' + 'session_id, message_id, position, body_json, fingerprint, created_at, host_instance, state, hold_reason, returned_reason, returned_rejection, settled_at, settled_by_op, consumed_as, carried_from, queued_epoch, queued_sequence' export function insertQueuedMessage( db: Database.Database, @@ -63,6 +72,8 @@ export function insertQueuedMessage( body: AgentJournalMessageItem fingerprint: string hostInstance: string + carriedFrom?: string + queuedAt: AgentJournalCursor now: number } ): QueuedMessageRow { @@ -73,7 +84,7 @@ export function insertQueuedMessage( const position = Number(highest?.p ?? 0) + 1 db.prepare( `INSERT INTO queued_messages (${COLUMNS}) - VALUES (?, ?, ?, ?, ?, ?, ?, 'waiting', NULL, NULL, NULL, NULL, NULL, NULL)` + VALUES (?, ?, ?, ?, ?, ?, ?, 'waiting', NULL, NULL, NULL, NULL, NULL, NULL, ?, ?, ?)` ).run( input.sessionId, input.messageId, @@ -81,7 +92,10 @@ export function insertQueuedMessage( JSON.stringify(input.body), input.fingerprint, input.now, - input.hostInstance + input.hostInstance, + input.carriedFrom ?? null, + input.queuedAt.epoch, + input.queuedAt.sequence ) return { sessionId: input.sessionId, @@ -97,7 +111,9 @@ export function insertQueuedMessage( returnedRejection: null, settledAt: null, settledByOp: null, - consumedAs: null + consumedAs: null, + carriedFrom: input.carriedFrom ?? null, + queuedAt: input.queuedAt } } @@ -276,6 +292,9 @@ function toStoredRow(row: unknown): QueuedMessageRow | null { settled_at: number | null settled_by_op: string | null consumed_as: string | null + carried_from: string | null + queued_epoch: string | null + queued_sequence: number | null } let body: AgentJournalMessageItem try { @@ -309,7 +328,12 @@ function toStoredRow(row: unknown): QueuedMessageRow | null { returnedRejection: storedRejection(record.returned_rejection), settledAt: record.settled_at, settledByOp: record.settled_by_op, - consumedAs: record.consumed_as + consumedAs: record.consumed_as, + carriedFrom: record.carried_from, + queuedAt: + record.queued_epoch !== null && typeof record.queued_sequence === 'number' + ? { epoch: record.queued_epoch, sequence: record.queued_sequence } + : null } } diff --git a/src/main/native-chat/agent-session-wire/agent-session-journal-batch.ts b/src/main/native-chat/agent-session-wire/agent-session-journal-batch.ts index b70a18a8386..42af5f83042 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-journal-batch.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-journal-batch.ts @@ -14,7 +14,10 @@ import type { } from '../../../shared/agent-session-journal-types' import type { AgentSessionJournalBatch } from '../../../shared/agent-session-wire' import { findSequenceGap } from '../agent-session-journal/journal-cursor' -import type { JournalRow } from '../agent-session-journal/journal-row-schema' +import { + isJournalStopOrResumeRow, + type JournalRow +} from '../agent-session-journal/journal-row-schema' export type JournalBatchProjection = | { ok: true; batch: AgentSessionJournalBatch } @@ -50,6 +53,10 @@ export function projectJournalBatch(input: { } continue } + if (isJournalStopOrResumeRow(row)) { + // Host-only: no item; the queue pause it feeds is published beside the list. + continue + } if (row.kind === 'item' || row.kind === 'tombstone') { touchedItemIds.add( input.canonicalItemId?.(row.itemId) ?? aliases.get(row.itemId) ?? row.itemId diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts index fd8f9cfe596..6f808baa629 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts @@ -18,7 +18,7 @@ import { } from './structured-agent-session-mutation-context' import type { StructuredAgentSessionCaller } from './structured-agent-session-host-types' import type { MutationPlan } from './structured-agent-session-mutation-plans' -import { runStopWithQueuePause } from './structured-agent-session-queued-stop' +import { runRecordedStop, stopReachesUnrecordedWork } from './structured-agent-session-queued-stop' import { openForWrite, structuredAgentSessionFailureWordsContext @@ -52,10 +52,11 @@ export function mutateWithChatStop( // Set by the Stop's step only when its provider's session ends; a replay leaves it unset. let windDown: StructuredAgentSessionStopWindDown | undefined const named = turnId !== undefined ? { turnId } : {} - // Stop's queue step, the same for every client: once the Stop takes effect the queue is paused. - // The cards stay published; nothing is withdrawn and no text ever rides the answer. + const stopEvent = { reason: 'user-stop' as const, caller: caller.callerKey, ...named } + // The same for every client: once the Stop takes effect its event is written, and the queue's + // pause follows from it. The cards stay published; no text rides the answer. const stop = (ctx: AgentSessionTurnContext): Promise => - runStopWithQueuePause(ctx, async (tookEffect) => { + runRecordedStop(ctx, stopEvent, async (tookEffect) => { // Stop withdraws every queued SUBMISSION first, whatever the start or the child is doing. const withdrawn = await ctx.journal.rejectQueuedSubmissions( ctx.fence, @@ -78,7 +79,10 @@ export function mutateWithChatStop( } return { ok: true, value: { ...named, cancelled: withdrawn.length > 0 } } } - await tookEffect() + // Awaited until journal appends are synchronous; then issued here, and a `finally` awaits it. + if (withdrawn.length > 0 || (await stopReachesUnrecordedWork(ctx, turnId))) { + await tookEffect() + } return performCancel( { ...ctx, failureTextContext: structuredAgentSessionFailureWordsContext(record) }, { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-queued-stop.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-queued-stop.test.ts index 2531450fd22..5e4ac75b066 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-queued-stop.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-queued-stop.test.ts @@ -113,7 +113,7 @@ function eventually(assertion: () => unknown): Promise { } function envelope( - method: 'agentSession.send' | 'agentSession.cancel', + method: 'agentSession.send' | 'agentSession.cancel' | 'agentSession.queuedMessageSend', fields: Record ) { return { @@ -318,3 +318,62 @@ it('withdraws a host-queued follow-up but leaves a newer turn running when the S ) expect(rows).not.toContain('The provider had already finished this turn.') }, 15_000) + +it('a card sent now into the running turn comes back paused when Stop withdraws it, and is not sent again', async () => { + const connection = claude.connections[0]! + const turnId = await openFirstTurn(connection) + const body = hostTestMessage('And then this.') + const delivery = 'queue-if-active' as const + const queuedSend = await host.send(CALLER, { + envelope: envelope('agentSession.send', { body, delivery }), + body, + delivery, + userSend: true + }) + if (!queuedSend.ok || !('queued' in queuedSend.value)) { + throw new Error('expected a queued receipt') + } + const cardId = queuedSend.value.queued.messageId + const sentNow = await host.queuedMessageSend(CALLER, { + envelope: envelope('agentSession.queuedMessageSend', { messageId: cardId }), + messageId: cardId + }) + expect(sentNow).toMatchObject({ ok: true }) + // Folded into the running turn: Claude holds it until that turn ends. + await eventually(() => expect(connection.sent).toHaveLength(2)) + queued.push(String(connection.sent.at(-1)!.uuid)) + const sends = async () => + (await host.journalSnapshot(SESSION)).submissions + .filter((entry) => entry.queuedMessageId === cardId) + .map((entry) => ({ origin: entry.origin, state: entry.dispatchState, reason: entry.reason })) + await eventually(async () => expect((await sends())[0]?.state).toBe('pending')) + + expect(await stop(turnId)).toMatchObject({ ok: true, value: { cancelled: true } }) + connection.handlers.onMessage?.({ + type: 'result', + subtype: 'error_during_execution', + session_id: PROVIDER_SESSION_ID, + uuid: 'interrupted-result' + }) + + // Inside the test's budget, so a re-send fails on this diff rather than the timeout. + await vi.waitFor(async () => { + const page = await host.history({ sessionId: SESSION, direction: 'tail' }) + expect({ + pause: page.ok ? (page.page.queuePause ?? null) : 'history refused', + cards: page.ok ? (page.page.queuedMessages ?? []).map((card) => card.state) : [], + sends: await sends() + }).toEqual({ + pause: { reason: 'stopped' }, + cards: ['waiting'], + sends: [{ origin: 'client', state: 'rejected', reason: DISPATCH_REJECTED_CANCELLED }] + }) + }, 5_000) + // A drain ignoring the pause re-sends only after the stopped turn ends: watch past that. + await eventually(async () => expect(await liveTurnId()).toBeNull()) + await new Promise((resolve) => setTimeout(resolve, 2_500)) + expect(connection.sent).toHaveLength(2) + expect(await sends()).toEqual([ + { origin: 'client', state: 'rejected', reason: DISPATCH_REJECTED_CANCELLED } + ]) +}, 20_000) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-ends-session.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-ends-session.test.ts index 845647f3c0a..ddef0ed0bb6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-ends-session.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-ends-session.test.ts @@ -214,6 +214,23 @@ function stop(turnId?: string) { return host.cancel(CALLER, { envelope: envelope('agentSession.cancel', fields), ...fields }) } +/** How many person's Stop events the journal holds when the child's close begins. */ +function stopEventsAtClose(connection: FakeConnection): () => number | undefined { + let atClose: number | undefined + const close = connection.close + connection.close = async () => { + const journal = host.collaboratorsForTests().sessions.get(SESSION)?.journal + const since = journal?.readSince({ epoch: journal.epoch, sequence: 0 }) + atClose ??= since?.ok + ? since.rows.filter( + (row) => row.kind === 'tombstone' && row.stopEvent?.reason === 'user-stop' + ).length + : -1 + return close() + } + return () => atClose +} + /** Resolves once everything queued on the session's lane so far has run: a Stop's second step. */ function laneDrained(): Promise { return host['tasks'].serialize(SESSION, async () => {}) @@ -261,6 +278,7 @@ async function statusTexts(): Promise { it('answers on the interrupt, ends the child once the stopped turn ends, and rests at that turn', async () => { const connection = claude.connections[0]! + const eventsAtClose = stopEventsAtClose(connection) await openTurn(connection) await expect(stop()).resolves.toMatchObject({ ok: true, value: { cancelled: true } }) @@ -274,6 +292,7 @@ it('answers on the interrupt, ends the child once the stopped turn ends, and res expect(Date.now() - ended).toBeLessThan(CLAUDE_STOP_GRACE_MS / 2) expect(connection.closed).toBe(true) + expect(eventsAtClose()).toBe(1) expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') expect(await turnOutcome()).toBe('cancellation') // The resume point is the stopped turn's own, so the next send continues after it. @@ -346,6 +365,7 @@ it('reads a Stop pressed before Claude echoed the send as interrupted, not as a it('ends the child once the grace runs out when Claude says nothing after a Stop before the echo', async () => { const connection = claude.connections[0]! + const eventsAtClose = stopEventsAtClose(connection) claude.routes.interrupt = () => ({ still_queued: [], cancelled: [] }) const clientMessageId = await sendUnechoed(connection) @@ -355,6 +375,7 @@ it('ends the child once the grace runs out when Claude says nothing after a Stop // As before the wait: the send Claude never answered is doubt once its child ends. expect(connection.closed).toBe(true) + expect(eventsAtClose()).toBe(1) expect(Date.now() - asked).toBeLessThan(CLAUDE_STOP_GRACE_MS + 1_500) expect(await dispatch(clientMessageId)).toMatchObject({ state: 'unknown' }) }, 15_000) @@ -386,6 +407,7 @@ it('ends the child at once when Claude refuses the interrupt, and says only that throw new ClaudeControlRequestError('interrupt', 'Claude did not answer the interrupt.') } const connection = claude.connections[0]! + const eventsAtClose = stopEventsAtClose(connection) await openTurn(connection) const asked = Date.now() @@ -395,6 +417,7 @@ it('ends the child at once when Claude refuses the interrupt, and says only that expect(Date.now() - asked).toBeLessThan(CLAUDE_STOP_GRACE_MS / 2) expect(connection.closed).toBe(true) + expect(eventsAtClose()).toBe(1) expect(await turnOutcome()).toBe('cancellation') const texts = await statusTexts() expect(texts).toContain('Cancellation requested.') @@ -406,12 +429,14 @@ it('ends the child when the interrupt fails, with no unconfirmed row', async () throw new Error('control request lost') } const connection = claude.connections[0]! + const eventsAtClose = stopEventsAtClose(connection) await openTurn(connection) await expect(stop()).resolves.toMatchObject({ ok: true, value: { cancelled: true } }) await laneDrained() expect(connection.closed).toBe(true) + expect(eventsAtClose()).toBe(1) expect(await turnOutcome()).toBe('cancellation') expect(await statusTexts()).toEqual(['Cancellation requested.']) }) @@ -419,6 +444,7 @@ it('ends the child when the interrupt fails, with no unconfirmed row', async () it('ends the child within the grace when Claude never answers the interrupt', async () => { claude.routes.interrupt = NEVER_ANSWERS const connection = claude.connections[0]! + const eventsAtClose = stopEventsAtClose(connection) await openTurn(connection) const asked = Date.now() @@ -426,6 +452,7 @@ it('ends the child within the grace when Claude never answers the interrupt', as await laneDrained() expect(connection.closed).toBe(true) + expect(eventsAtClose()).toBe(1) expect(Date.now() - asked).toBeLessThan(CLAUDE_STOP_GRACE_MS + 1_500) expect(await turnOutcome()).toBe('cancellation') expect(await statusTexts()).toEqual(['Cancellation requested.']) @@ -632,6 +659,7 @@ it.each([ claude.routes.interrupt = interrupt } const connection = claude.connections[0]! + const eventsAtClose = stopEventsAtClose(connection) const ended = await openTurn(connection) frame(connection, { type: 'result', subtype: 'success', is_error: false, uuid: 'result-1' }) await eventually(async () => @@ -649,6 +677,7 @@ it.each([ expect(connection.calls.some((call) => call.subtype === 'interrupt')).toBe(true) expect(connection.closed).toBe(true) + expect(eventsAtClose()).toBe(1) expect(await statusTexts()).toEqual(['Cancellation requested.']) }, 15_000 @@ -747,6 +776,7 @@ it('dismisses an approval card on its Cancel with the Deny reply, and the turn a it("ends a question card's Cancel the way the chat's Stop does, and the next send resumes", async () => { const connection = claude.connections[0]! + const eventsAtClose = stopEventsAtClose(connection) const turnId = await openTurn(connection) const request = new AbortController() const { answered, card } = await ask( @@ -772,6 +802,7 @@ it("ends a question card's Cancel the way the chat's Stop does, and the next sen await laneDrained() expect(connection.closed).toBe(true) + expect(eventsAtClose()).toBe(1) expect(await turnOutcome()).toBe('cancellation') // The child's end takes Claude's request with it: no reply raced the interrupt, and nothing // wrote over the user's cancel. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts index e93b44a6b08..d6a6819679e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts @@ -380,6 +380,53 @@ describe('a Codex Stop whose interrupt failed', () => { expect((await journalRows()).turns).toEqual(['completed', 'running']) }) + it('holds a card queued before it while the child it ends goes, its event written first', async () => { + await runningTurn() + const body = hostTestMessage('queued before the Stop') + const delivery = 'queue-if-active' as const + const queued = await host.send(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: 1, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body, delivery } + }) + }, + body, + delivery + }) + if (!queued.ok || !('queued' in queued.value)) { + throw new Error(`expected a queued receipt: ${JSON.stringify(queued)}`) + } + const cardId = queued.value.queued.messageId + codex.routes['turn/interrupt'] = () => { + throw interruptFailure('internal error') + } + + expect(await stop()).toMatchObject({ ok: true, value: { cancelled: true } }) + await host.flushStreamedEvents(SESSION) + await new Promise((resolve) => setTimeout(resolve, 250)) + + expect(childEndedByStop()).toBe(true) + const page = await host.history({ sessionId: SESSION, direction: 'tail' }) + expect(page.ok && page.page.queuePause).toEqual({ reason: 'stopped' }) + expect( + (await host.journalSnapshot(SESSION)).submissions.filter( + (entry) => entry.queuedMessageId === cardId + ) + ).toEqual([]) + const journal = host['sessions'].get(SESSION)!.journal + const since = journal.readSince({ epoch: journal.epoch, sequence: 0 }) + const rows = since.ok ? since.rows : [] + const stopAt = rows.find((row) => row.kind === 'tombstone' && row.stopEvent)?.seq + // The turn's end, from the child's end or Codex's own frame, in whatever row carries it. + const endAt = rows.find((row) => JSON.stringify(row).includes('"state":"interrupted"'))?.seq + expect(stopAt).toBeLessThan(endAt ?? 0) + }) + it('leaves a child that exited during the interrupt to its exit', async () => { await runningTurn() codex.routes['turn/interrupt'] = () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts index a9a1089cc1c..ead8d440004 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts @@ -36,7 +36,11 @@ export function eventually(assertion: () => void | Promise): Promise export type QueuedMessageTestRig = Awaited> -export async function createQueuedMessageTestRig() { +/** `restartable`: a child started for a chat whose chain already names a thread resumes it, so a + * chat whose child closed or died can start another. `starting`: every child stays starting. */ +export async function createQueuedMessageTestRig( + options: { restartable?: true; starting?: true } = {} +) { const root = await mkdtemp(join(tmpdir(), 'orca-queued-messages-')) resetHostTestOperationIds() // Admitted: the message is written and unanswered, so the session owes work @@ -52,45 +56,57 @@ export async function createQueuedMessageTestRig() { state: 'accepted' as const, providerIdentity: null })) + const cancelTurn: Mock = vi.fn(async () => ({ + cancelled: true + })) + const closeSession: Mock> = vi.fn( + async () => true + ) let events: StructuredAgentSessionEventSink | undefined const store = await openTestAgentSessionRecordStore(root) - const host = new StructuredAgentSessionHost({ - logger: createStructuredAgentSessionLogger(), - store, - adapter: { - acquire: async ({ fence, spawnToken, events: sink }) => { - events = sink - return { - process: { - hostId: 'local', - pid: 4242, - processStartTimeMs: 1_700_000_000_000, - spawnToken - }, - acquisitionGeneration: 'generation-1', - link: { - linkId: `link-${fence}`, - handle: { provider: 'codex' as const, threadId: THREAD }, - origin: 'created' as const, - mintedAtFence: fence, - observedAt: NOW + const makeHost = () => + new StructuredAgentSessionHost({ + logger: createStructuredAgentSessionLogger(), + store, + adapter: { + acquire: async ({ identity, fence, spawnToken, events: sink }) => { + events = sink + const resumes = + options.restartable === true && + (store.getRecord(identity.sessionId)?.providerHandleChain.length ?? 0) > 0 + return { + process: { + hostId: 'local', + pid: 4242, + processStartTimeMs: 1_700_000_000_000, + spawnToken + }, + acquisitionGeneration: 'generation-1', + ...(options.starting ? { providerChildPhase: 'starting' as const } : {}), + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex' as const, threadId: THREAD }, + origin: resumes ? ('resumed' as const) : ('created' as const), + mintedAtFence: fence, + observedAt: NOW + } } - } + }, + dispatch, + awaitStarted, + closeSession, + releaseAcquisition: vi.fn(async () => true), + compact, + cancelTurn, + answerPrompt: vi.fn(async () => undefined), + setOption: vi.fn(async () => undefined) }, - dispatch, - awaitStarted, - closeSession: vi.fn(async () => true), - releaseAcquisition: vi.fn(async () => true), - compact, - cancelTurn: vi.fn(async () => ({ cancelled: true })), - answerPrompt: vi.fn(async () => undefined), - setOption: vi.fn(async () => undefined) - }, - journalDatabase: openTestJournalHostDatabase(root), - claimKeyId: 'key-1', - mintSpawnToken: () => 'spawn-1', - now: () => NOW - }) + journalDatabase: openTestJournalHostDatabase(root), + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-1', + now: () => NOW + }) + let host = makeHost() expect(await host.attach(QUEUED_RIG_CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) @@ -251,6 +267,12 @@ export async function createQueuedMessageTestRig() { rotateStructuredAgentSessionHostInstanceForTests() } + /** A host process that dies with no close: a new host opens the same state directory. */ + function crashRestartHostProcess(): void { + rotateStructuredAgentSessionHostInstanceForTests() + host = makeHost() + } + /** The queue's published pause: null when it sends on its own. */ async function queuePause(sessionId = SESSION): Promise { const page = await host.history({ sessionId, direction: 'tail' }) @@ -274,8 +296,12 @@ export async function createQueuedMessageTestRig() { return { root, store, - host, + get host() { + return host + }, dispatch, + cancelTurn, + closeSession, awaitStarted, compact, finishCompact, @@ -292,6 +318,7 @@ export async function createQueuedMessageTestRig() { settleAccepted, settleRejected, restartHostProcess, + crashRestartHostProcess, queuePause, resume, dispose diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.test.ts index 33de806ff3b..b972c56877e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.test.ts @@ -14,7 +14,10 @@ import { import { ConversationCommandParams } from '../../../shared/rpc-contract/structured-agent-session-params' import { AgentSessionJournal } from '../agent-session-journal/journal-store' import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' -import { rotateStructuredAgentSessionHostInstanceForTests } from './structured-agent-session-queued-pause' +import { + rotateStructuredAgentSessionHostInstanceForTests, + structuredQueuePauses +} from './structured-agent-session-queued-pause' import { createQueuedMessageTestRig, eventually, @@ -512,19 +515,23 @@ describe('Stop and Delete', () => { }) }) - it('a Stop whose pause record fails still interrupts; only the pause is lost, and it is reported', async () => { + it('a Stop whose event fails to write still interrupts; only the pause is lost, and it is reported', async () => { await workingSend() const queued = await send('kept by the stop', 'queue-if-active').result if (!queued.ok || !('queued' in queued.value)) { throw new Error('expected a queued receipt') } const record = vi - .spyOn(JournalQueuedMessages.prototype, 'recordPause') + .spyOn(AgentSessionJournal.prototype, 'appendStopEvent') .mockRejectedValueOnce(new Error('disk full')) const warned = vi.spyOn(console, 'warn').mockImplementation(() => undefined) try { expect(await stop()).toMatchObject({ ok: true, value: { cancelled: true } }) - expect(warned).toHaveBeenCalledWith(expect.stringContaining('queue pause'), expect.anything()) + expect(rig.cancelTurn).toHaveBeenCalledTimes(1) + expect(warned).toHaveBeenCalledWith( + expect.stringContaining("Stop's event row"), + expect.anything() + ) } finally { record.mockRestore() warned.mockRestore() @@ -674,7 +681,7 @@ describe('/clear', () => { } expect(await drafts(replacementId)).toHaveLength(0) const journal = host.collaboratorsForTests().sessions.get(replacementId)?.journal - expect(journal?.queuedMessages.pause()).toBeNull() + expect(journal && structuredQueuePauses(journal)).toEqual([]) }) it('a returned card carries over as a plain waiting draft on the paused replacement', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index 856134f04a9..7d0eb8581e6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -28,8 +28,9 @@ import type { QueuedMessageRow } from '../agent-session-journal/queued-message-t import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { structuredAgentSessionHostInstance, - structuredQueuePause + structuredQueuePauses } from './structured-agent-session-queued-pause' +import { nextSendableQueuedCard } from '../agent-session-journal/queued-message-pause' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' /** Budget at accept, in the send schema's own unit (`Buffer.byteLength` of the @@ -58,27 +59,18 @@ export function pendingPromptExists(journal: Pick + journal: Pick ): QueuedMessageRow | null { const rows = journal.queuedMessages.list() // Nothing waiting costs no pause derivation: this runs on every journal publish. - if (!rows.some((row) => row.state === 'waiting') || structuredQueuePause(journal) !== null) { + if (!rows.some((row) => row.state === 'waiting')) { return null } - for (const row of rows) { - if (row.state === 'returned') { - // A returned card blocks everything after it until the user acts. - return null - } - if (row.state === 'waiting' && row.holdReason === null) { - return row - } - } - return null + return nextSendableQueuedCard(structuredQueuePauses(journal), rows) } /** @@ -367,12 +359,13 @@ export class StructuredAgentSessionQueuedMessageDrain { messageId: next.messageId, expect: 'waiting', settledByOp: null, - hostInstance: structuredAgentSessionHostInstance() + hostInstance: structuredAgentSessionHostInstance(), + yieldsToPause: { hostInstance: structuredAgentSessionHostInstance() } } ) } catch (error) { if (error instanceof QueuedMessageNotConsumableError) { - // Lost a race with a Send-now or Delete; their transition stands. + // Lost a race with a Send-now, a Delete or a Stop; their transition stands. return } // Pre-consume failure: the draft stays waiting, held with the marker on diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts index 288848a96a6..ee32b3648a9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts @@ -80,12 +80,12 @@ export async function withdrawQueuedMessagesForOperation( * cards stay visible where the user now is. The replacement's queue starts * paused ('cleared'), lifted exactly like a Stop's: the cards were written for the context /clear just * discarded, so they wait for the user's next turn there, or Resume, rather than - * sending into the fresh context unasked. Each card lands with the pause in one - * transaction, so the drain never sees a carried card unpaused and no pause is - * left over an empty queue if an insert fails. Runs after the clear commits, - * opening the replacement's conversation only when there are drafts to carry; - * the source rows are then tombstoned. Bookkeeping around the clear: a failure, - * or a crash before the carry, leaves the cards on the superseded source — whose + * sending into the fresh context unasked. Each card records the conversation it + * came from, which IS that pause, so the drain never sees a carried card unpaused + * and no pause outlives the cards. Runs after the clear commits, opening the + * replacement's conversation only when there are drafts to carry; the source + * rows are then tombstoned. Bookkeeping around the clear: a failure, or a crash + * before the carry, leaves the cards on the superseded source — whose * supersession fence already blocks the drain — reported, never gating the * clear. A crash between the copy and the tombstone leaves both, which the * fence also makes harmless: nothing is lost and nothing runs. @@ -117,7 +117,7 @@ export async function carryQueuedMessagesToClearReplacement( body: row.body, fingerprint: queuedMessageFingerprint(input.replacementSessionId, row.body), hostInstance: structuredAgentSessionHostInstance(), - pausedBy: 'cleared' + carriedFrom: ctx.sessionId }) } await withdrawQueuedMessagesForOperation(ctx.journal, { @@ -323,11 +323,11 @@ export function resumeStructuredAgentQueue( method: 'agentSession.queuedMessagesResume', fields: {}, conversationWrite: true, - // The lift notifies through the journal's commit listener, which publishes the - // cleared pause and wakes the drain. + // The Resume row notifies through the journal's commit listener, which publishes the + // lifted pause and wakes the drain. run: async (ctx) => ({ ok: true, - value: { resumed: await resumeStructuredQueue(ctx.journal) } + value: { resumed: await resumeStructuredQueue(ctx.journal, ctx.fence) } }), // Like Stop's replay: the Resume already ran, so this one lifts nothing. replay: () => ({ resumed: false }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause-lift.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause-lift.test.ts index 63a12313bf6..0760a55c264 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause-lift.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause-lift.test.ts @@ -16,9 +16,10 @@ import { type QueuedMessageTestRig } from './structured-agent-session-queued-message-rig.test-fixture' import { sameQueuePause } from './structured-agent-session-queued-publication' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' import { structuredAgentSessionHostInstance, - structuredQueuePause + structuredQueuePauses } from './structured-agent-session-queued-pause' let rig: QueuedMessageTestRig @@ -188,42 +189,24 @@ describe('the pause read', () => { } const scan = vi.spyOn(journal, 'submissions') // Read on every publish, per subscriber: it must not walk the submissions. - expect(structuredQueuePause(journal)).toEqual({ reason: 'stopped' }) + expect(structuredQueuePauses(journal)).toMatchObject([{ reason: 'stopped' }]) expect(scan).not.toHaveBeenCalled() scan.mockRestore() - const before = journal.queuedMessages.latestPersonTurnSequence() const mail = rig.send('coordinator mail', undefined, { internal: true }) await mail.result await rig.settleAccepted(mail.id, 'mail') - // Orca's own turn moves nothing; a person's does, and lifts the pause. - expect(journal.queuedMessages.latestPersonTurnSequence()).toBe(before) + // Orca's own turn lifts nothing; a person's does. + expect(structuredQueuePauses(journal)).toMatchObject([{ reason: 'stopped' }]) const next = rig.send('user starts a new turn') await next.result await rig.settleAccepted(next.id, 'next') - expect(journal.queuedMessages.latestPersonTurnSequence()).toBe( - journal.submission(next.id)?.acceptedSequence - ) + expect(structuredQueuePauses(journal)).toEqual([]) await eventually(async () => expect(await rig.handoff(draftId)).toBeDefined()) }) }) -describe('a pause is over the cards it paused', () => { - it('a Stop over an empty queue pauses nothing: a card typed during a later mail turn drains', async () => { - const working = await rig.workingSend() - await rig.stop() - await rig.settleAccepted(working, 'stopped') - const mail = rig.send('coordinator mail', undefined, { internal: true }) - await mail.result - await eventually(async () => - expect((await rig.submission(mail.id))?.handedOverAt).toBeDefined() - ) - const followUp = await queuedDraft('typed during the mail turn') - await rig.settleAccepted(mail.id, 'mail') - await eventually(async () => expect(await rig.handoff(followUp)).toBeDefined()) - expect(await rig.queuePause()).toBeNull() - }) - - it('a Stop over an empty queue pauses nothing: a correction typed before the turn ends drains', async () => { +describe('a card queued after a Stop is a new instruction', () => { + it('a correction typed after a Stop over an empty queue sends when the stopped turn ends', async () => { const working = await rig.workingSend() await rig.stop() const correction = await queuedDraft('typed right after the stop') @@ -231,13 +214,16 @@ describe('a pause is over the cards it paused', () => { await eventually(async () => expect(await rig.handoff(correction)).toBeDefined()) }) - it('deleting the last paused card ends the pause, so a card typed later is not held by it', async () => { + it('a card sent now before the Stop and taken anyway lifts nothing, and holds nothing typed later', async () => { const working = await rig.workingSend() - const only = await queuedDraft('paused, then deleted') + const sentId = await queuedDraft('sent now into the turn') + await rig.sendNow(sentId) + await handedOver(sentId) await rig.stop() + // Nothing waits, so nothing is published; the pause is still derived. + expect(await rig.queuePause()).toBeNull() + await rig.settleAccepted(await rig.handoffId(sentId), 'sent-now') await rig.settleAccepted(working, 'stopped') - expect(await rig.queuePause()).toEqual({ reason: 'stopped' }) - expect(await rig.deleteQueued(only)).toMatchObject({ ok: true, value: { deleted: true } }) const mail = rig.send('coordinator mail', undefined, { internal: true }) await mail.result await eventually(async () => @@ -246,11 +232,36 @@ describe('a pause is over the cards it paused', () => { const later = await queuedDraft('typed during the mail turn') await rig.settleAccepted(mail.id, 'mail') await eventually(async () => expect(await rig.handoff(later)).toBeDefined()) + const journal = rig.host.collaboratorsForTests().sessions.get(HOST_TEST_SESSION)?.journal + expect(journal && structuredQueuePauses(journal)).toMatchObject([{ reason: 'stopped' }]) + }) + + it("deleting the last paused card hides the pause; a person's next turn is what ends it", async () => { + const working = await rig.workingSend() + const only = await queuedDraft('paused, then deleted') + await rig.stop() + await rig.settleAccepted(working, 'stopped') + expect(await rig.queuePause()).toEqual({ reason: 'stopped' }) + expect(await rig.deleteQueued(only)).toMatchObject({ ok: true, value: { deleted: true } }) + expect(await rig.queuePause()).toBeNull() + const journal = rig.host.collaboratorsForTests().sessions.get(HOST_TEST_SESSION)?.journal + if (!journal) { + throw new Error('expected the conversation open') + } + // Hidden, not ended: with nothing to hold, no card can show the lift, so read the Stop itself. + expect(structuredQueuePauses(journal)).toMatchObject([{ reason: 'stopped' }]) + const next = await rig.workingSend() + const later = await queuedDraft('typed during the next turn') + expect(structuredQueuePauses(journal)).toMatchObject([{ reason: 'stopped' }]) + // That send is a person's turn after the Stop: once it starts, the Stop is over. + await rig.settleAccepted(next, 'next') + expect(structuredQueuePauses(journal)).toEqual([]) + await eventually(async () => expect(await rig.handoff(later)).toBeDefined()) }) }) describe('a pause only over cards Resume could send', () => { - it('a Stop that leaves only a returned card publishes no pause and keeps no fact', async () => { + it('a Stop that leaves only a returned card publishes no pause', async () => { const working = await rig.workingSend() const draftId = await queuedDraft('refused before the stop') await rig.settleAccepted(working, 'a') @@ -265,7 +276,7 @@ describe('a pause only over cards Resume could send', () => { await rig.settleAccepted(next, 'stopped') expect(await rig.queuePause()).toBeNull() const journal = rig.host.collaboratorsForTests().sessions.get(HOST_TEST_SESSION)?.journal - expect(journal?.queuedMessages.pause()).toBeNull() + expect(journal && structuredQueuePauses(journal)).toMatchObject([{ reason: 'stopped' }]) }) it('a returned card blocking the paused cards hides the pause but keeps it; deleting that card shows it again, and only Resume sends', async () => { @@ -290,7 +301,7 @@ describe('a pause only over cards Resume could send', () => { } // Resume would send nothing past the returned card, so no header offers it; the pause stays. expect(await rig.queuePause()).toBeNull() - expect(journal.queuedMessages.pause()).toMatchObject({ reason: 'stopped' }) + expect(structuredQueuePauses(journal)).toMatchObject([{ reason: 'stopped' }]) // Deleting the blocking card shows the pause again: the card behind it does not send unasked. expect(await rig.deleteQueued(refusedId)).toMatchObject({ ok: true, value: { deleted: true } }) await expectPaused(behindId) @@ -378,11 +389,10 @@ describe('a card handed off after a restart', () => { structuredAgentSessionHostInstance() ) // With the Stop's pause gone, nothing else holds it: no restart happened since it was sent. - await journal.queuedMessages.liftPause({ - stop: journal.queuedMessages.pause(), - adoptInto: null - }) - expect(structuredQueuePause(journal)).toBeNull() + await journal.appendQueueResume( + structuredAgentSessionConversationFence(rig.store, HOST_TEST_SESSION) + ) + expect(structuredQueuePauses(journal)).toEqual([]) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts index f96716d0304..7b85ede6fe9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts @@ -1,20 +1,10 @@ -// Whether the queue is paused, and why — DERIVED, never stored as a flag. The -// queue is paused when: -// - 'stopped': the user's last Stop took effect (its recorded journal -// position) and no turn a person asked for has started since — or -// 'cleared', the same for a /clear's replacement, whose carried cards -// start paused; or -// - 'restarted': a waiting draft was written by another host process and no -// turn a person asked for has started since this conversation opened. -// A person's turn is a submission whose recorded origin is `client` (a send over -// the client send RPC, or a card they sent now) that the provider accepted. -// Orchestration mail, a restart continuation, a host-sent launch prompt and the -// queue's own drain are `host` and never lift it. An explicit Resume lifts any. +// Whether the queue is paused, and why — derived from the journal and the cards +// (`queued-message-pause.ts`), never stored. A Stop's event and a Resume are journal +// rows; an explicit Resume lifts any pause. import { randomUUID } from 'node:crypto' -import type { AgentSessionQueuePause } from '../../../shared/agent-session-wire' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import type { QueuePauseFact } from '../agent-session-journal/queued-message-pause-table' +import type { DerivedQueuePause } from '../agent-session-journal/queued-message-pause' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' /** A per-process id, minted once per host process like the runtime's own @@ -32,80 +22,50 @@ export function rotateStructuredAgentSessionHostInstanceForTests(): string { return hostInstance } -type PauseJournal = Pick +type PauseJournal = Pick -// Both read the reducer's latest accepted person turn (its submission row), so a -// derivation on every publish costs no scan of the submissions. - -function stopEnded(journal: PauseJournal, stop: QueuePauseFact): boolean { - const latest = journal.queuedMessages.latestPersonTurnSequence() - // Sent after the Stop: a send made before it no longer lifts it, even if its turn starts later. - return stop.epoch !== journal.cursor().epoch ? latest > 0 : latest > stop.sequence -} - -function restartPending(journal: PauseJournal): boolean { - return journal.queuedMessages - .list() - .some((row) => row.state === 'waiting' && row.hostInstance !== hostInstance) -} - -function restartEnded(journal: PauseJournal): boolean { - const latest = journal.queuedMessages.latestPersonTurnSequence() - return latest > 0 && !journal.wroteBeforeOpen(latest) -} - -/** The queue's pause, derived; null when the queue sends on its own. */ -export function structuredQueuePause(journal: PauseJournal): AgentSessionQueuePause | null { - const stop = journal.queuedMessages.pause() - if (stop && !stopEnded(journal, stop)) { - return { reason: stop.reason } - } - if (restartPending(journal) && !restartEnded(journal)) { - return { reason: 'restarted' } - } - return null +/** The queue's pauses in force, derived; none when the queue sends on its own. */ +export function structuredQueuePauses(journal: PauseJournal): DerivedQueuePause[] { + return journal.queuedMessages.pauses(hostInstance) } /** - * Every journal publish: retire what a person's started turn already ended — the - * Stop fact it superseded, and a restart's rows, adopted into this instance. The - * derivation already reads them as lifted; the write keeps that answer when the - * handle reopens (the restart's "since this conversation opened" moves) and spares - * later derivations the submission scan. Bookkeeping: a failure is reported. + * Every journal publish: a restart's rows are adopted into this instance once a person's turn + * started. The derivation already reads them as lifted; the write keeps that answer when the + * handle reopens (its "since this conversation opened" moves). Bookkeeping: a failure is reported. */ -export async function retireEndedQueuePause( +export async function adoptEndedRestartPause( sessionId: string, journal: PauseJournal, logger: StructuredAgentSessionLogger ): Promise { try { - const stop = journal.queuedMessages.pause() - const retireStop = stop !== null && stopEnded(journal, stop) ? stop : null - const adopt = restartPending(journal) && restartEnded(journal) - if (retireStop === null && !adopt) { - return + const { queuedMessages } = journal + const restarted = queuedMessages + .list() + .some((row) => row.state === 'waiting' && row.hostInstance !== hostInstance) + if (restarted && queuedMessages.restartEnded()) { + await queuedMessages.adopt(hostInstance) } - await journal.queuedMessages.liftPause({ - stop: retireStop, - adoptInto: adopt ? hostInstance : null - }) } catch (error) { - logger.warn('retiring a queue pause a started turn ended failed', { - scope: 'queue-pause-retirement', + logger.warn("adopting a restart's queued cards after a started turn failed", { + scope: 'queue-pause-adoption', sessionId, error }) } } -/** Resume: ends whichever pause holds the queue. Returns whether it was paused. */ -export async function resumeStructuredQueue(journal: PauseJournal): Promise { - if (structuredQueuePause(journal) === null) { +/** Resume: a journal row that ends a Stop's or a /clear's pause, and adoption of a restart's + * rows. Returns whether the queue was paused. */ +export async function resumeStructuredQueue( + journal: Pick, + fence: number +): Promise { + if (structuredQueuePauses(journal).length === 0) { return false } - await journal.queuedMessages.liftPause({ - stop: journal.queuedMessages.pause(), - adoptInto: hostInstance - }) + await journal.appendQueueResume(fence) + await journal.queuedMessages.adopt(hostInstance) return true } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts index 10b1f75fc3e..1fa46616da1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts @@ -9,8 +9,8 @@ import { type AgentSessionQueuePause } from '../../../shared/agent-session-wire' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { hasResumableQueuedMessage } from '../agent-session-journal/queued-message-pause-table' -import { structuredQueuePause } from './structured-agent-session-queued-pause' +import { resumableQueuePause } from '../agent-session-journal/queued-message-pause' +import { structuredQueuePauses } from './structured-agent-session-queued-pause' export type QueuePublication = { queuedMessages: AgentSessionQueuedMessage[] @@ -81,11 +81,11 @@ export function sameQueuePause( export function readQueuePublication(journal: AgentSessionJournal): QueuePublication { const queuedMessages = readPublishedQueuedMessages(journal) - // Read per emit: the pause also turns on submissions (a person's turn starting). - // Kept over any card it holds back, but shown only over one Resume would send, so its - // header never offers to send nothing; deleting a blocking returned card shows it again. - const pausable = hasResumableQueuedMessage(journal.queuedMessages.list()) - const queuePause = pausable ? structuredQueuePause(journal) : null + // Read per emit: the pause also turns on submissions (a person's turn starting). Shown only + // over a card Resume would send, so its header never offers to send nothing; deleting a + // blocking returned card shows it again. + const pause = resumableQueuePause(structuredQueuePauses(journal), journal.queuedMessages.list()) + const queuePause = pause ? { reason: pause.reason } : null const previous = publications.get(journal) if ( previous && diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop-row.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop-row.test.ts new file mode 100644 index 00000000000..aa2c93ba15e --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop-row.test.ts @@ -0,0 +1,292 @@ +// Stop writes one event row before it interrupts, and the queue's pause is derived from it: +// through the real host, the cards queued before a Stop wait, a card queued after it sends +// normally but never ahead of them, a withdrawn card comes back under it, a crash keeps it, it +// never hides a restart's pause, and no stored pause is ever written. + +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { agentSessionFailureFact } from '../../../shared/agent-session-failure' +import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' +import Database from '../../sqlite/sync-database' +import { journalDatabasePath } from '../agent-session-journal/journal-host-database' +import { + HOST_TEST_SESSION, + hostTestMessage, + hostTestOperationId +} from './structured-agent-session-host-test-data' +import { + createQueuedMessageTestRig, + eventually, + QUEUED_RIG_CALLER, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { structuredQueuePauses } from './structured-agent-session-queued-pause' + +let rig: QueuedMessageTestRig + +beforeEach(async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) +}) + +afterEach(() => rig.dispose()) + +async function queuedDraft(text: string): Promise { + const queued = await rig.send(text, 'queue-if-active').result + if (!queued.ok || !('queued' in queued.value)) { + throw new Error('expected a queued receipt') + } + return queued.value.queued.messageId +} + +function journal(sessionId = HOST_TEST_SESSION) { + const open = rig.host.collaboratorsForTests().sessions.get(sessionId)?.journal + if (!open) { + throw new Error('expected the conversation open') + } + return open +} + +/** No drain step converts the cards, and the published pause names `reason`. */ +async function expectHeld(reason: string, ...draftIds: string[]): Promise { + await new Promise((resolve) => setTimeout(resolve, 250)) + for (const draftId of draftIds) { + expect(await rig.handoff(draftId)).toBeUndefined() + } + expect(await rig.queuePause()).toEqual({ reason }) +} + +async function mailTurn(): Promise { + const mail = rig.send('coordinator mail', undefined, { internal: true }) + await mail.result + await eventually(async () => expect((await rig.submission(mail.id))?.handedOverAt).toBeDefined()) + return mail.id +} + +function withdraw(clientMessageId: string) { + return rig.host.settleLateDispatch({ + sessionId: HOST_TEST_SESSION, + clientMessageId, + state: 'rejected', + ...agentSessionFailureWords(agentSessionFailureFact('cancelled'), { surface: 'rejection' }) + }) +} + +/** Tables that could store a pause: none, the journal rows are the only record. */ +function pauseTables(): number { + const db = new Database(journalDatabasePath(rig.root), { readonly: true }) + try { + return db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE '%pause%'") + .all().length + } finally { + db.close() + } +} + +describe("Stop's event", () => { + it('is written before the interrupt reaches the agent', async () => { + await rig.workingSend() + let pausedAtInterrupt: unknown = 'not interrupted' + rig.cancelTurn.mockImplementationOnce(async () => { + pausedAtInterrupt = structuredQueuePauses(journal())[0]?.reason ?? null + return { cancelled: true } + }) + expect(await rig.stop()).toMatchObject({ ok: true, value: { cancelled: true } }) + expect(pausedAtInterrupt).toBe('stopped') + }) + + it('over an EMPTY queue holds nothing: a card queued during a later mail turn sends normally', async () => { + const working = await rig.workingSend() + await rig.stop() + await rig.settleAccepted(working, 'stopped') + const mail = await mailTurn() + const typed = await queuedDraft('typed during the mail turn') + expect(await rig.queuePause()).toBeNull() + await rig.settleAccepted(mail, 'mail') + await eventually(async () => expect(await rig.handoff(typed)).toBeDefined()) + }) + + it('a card queued after the Stop waits behind the cards it holds, then all send in order', async () => { + const working = await rig.workingSend() + const held = await queuedDraft('queued before the stop') + await rig.stop() + await rig.settleAccepted(working, 'stopped') + const mail = await mailTurn() + const later = await queuedDraft('queued during the mail turn') + await rig.settleAccepted(mail, 'mail') + // The queue never reorders: the newer card waits behind the held one, with no caption of its own. + await expectHeld('stopped', held, later) + expect(await rig.drafts()).toEqual([ + { messageId: held, state: 'waiting' }, + { messageId: later, state: 'waiting' } + ]) + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + await eventually(async () => expect(await rig.handoff(held)).toBeDefined()) + expect(await rig.handoff(later)).toBeUndefined() + await eventually(async () => expect((await rig.handoff(held))?.handedOverAt).toBeDefined()) + await rig.settleAccepted(await rig.handoffId(held), 'held') + await eventually(async () => expect(await rig.handoff(later)).toBeDefined()) + }) + + it("a person's accepted turn lifts it; a host turn and a later Stop do not", async () => { + const working = await rig.workingSend() + const first = await queuedDraft('first') + await rig.stop() + await rig.settleAccepted(working, 'stopped') + await rig.settleAccepted(await mailTurn(), 'mail') + await expectHeld('stopped', first) + const person = rig.send('the person asks for a turn') + await person.result + // A later Stop supersedes: the send made before it no longer lifts anything. + await rig.stop() + await rig.settleAccepted(person.id, 'person') + await expectHeld('stopped', first) + const after = rig.send('asked after the second Stop') + await after.result + await rig.settleAccepted(after.id, 'after') + await eventually(async () => expect(await rig.handoff(first)).toBeDefined()) + }) + + it('a card the Stop withdrew comes back once, under the pause, and is never sent again on its own', async () => { + const working = await rig.workingSend() + const sentId = await queuedDraft('sent now into the turn') + await rig.sendNow(sentId) + await eventually(async () => expect((await rig.handoff(sentId))?.handedOverAt).toBeDefined()) + const handoffId = await rig.handoffId(sentId) + await rig.stop() + await withdraw(handoffId) + await withdraw(handoffId) + await rig.settleAccepted(working, 'stopped') + await new Promise((resolve) => setTimeout(resolve, 250)) + expect(await rig.queuePause()).toEqual({ reason: 'stopped' }) + expect(await rig.drafts()).toEqual([{ messageId: sentId, state: 'waiting' }]) + const sends = (await rig.host.journalSnapshot(HOST_TEST_SESSION)).submissions.filter( + (entry) => entry.queuedMessageId === sentId + ) + expect(sends.map((entry) => [entry.origin, entry.dispatchState])).toEqual([ + ['client', 'rejected'] + ]) + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + await eventually(async () => expect((await rig.handoff(sentId))?.origin).toBe('host')) + }) + + it('survives a host crash: the next open marks the hand-off unknown, the pause stays, Resume sends', async () => { + await rig.workingSend() + const sentId = await queuedDraft('sent now into the turn') + const waiting = await queuedDraft('waiting behind it') + await rig.sendNow(sentId) + await eventually(async () => expect((await rig.handoff(sentId))?.handedOverAt).toBeDefined()) + await rig.stop() + // The process dies with no close; a new host opens the same state directory. + rig.crashRestartHostProcess() + expect((await rig.handoff(sentId))?.dispatchState).toBe('unknown') + await expectHeld('stopped', waiting) + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + await eventually(async () => expect(await rig.handoff(waiting)).toBeDefined()) + }) +}) + +describe("a Stop never hides a restart's pause", () => { + it('a card queued after a Stop over an empty queue, before a restart, waits restarted', async () => { + const working = await rig.workingSend() + await rig.stop() + await rig.settleAccepted(working, 'stopped') + const mail = await mailTurn() + const typed = await queuedDraft('typed during the mail turn') + await rig.restartHostProcess() + await rig.settleAccepted(mail, 'mail') + await expectHeld('restarted', typed) + }) + + it("a card queued during the queue's own send after a Stop, before a restart, waits restarted", async () => { + const working = await rig.workingSend() + await rig.stop() + const correction = await queuedDraft('typed while the interrupt lands') + await rig.settleAccepted(working, 'stopped') + // The queue's own send is not a person's turn: the Stop stays in force, holding nothing. + await eventually(async () => + expect((await rig.handoff(correction))?.handedOverAt).toBeDefined() + ) + const typed = await queuedDraft('typed during that send') + await rig.restartHostProcess() + await rig.settleAccepted(await rig.handoffId(correction), 'correction') + await expectHeld('restarted', typed) + }) +}) + +describe("a /clear's carried cards", () => { + function clear() { + const fields = { command: 'clear' as const } + return rig.host.conversationCommand(QUEUED_RIG_CALLER, { + envelope: rig.envelope(fields, 'agentSession.conversationCommand', hostTestOperationId()), + ...fields + }) + } + + it("wait 'cleared' on the replacement until a person's turn there", async () => { + const working = await rig.workingSend() + const carried = await queuedDraft('written for the old context') + await rig.stop() + await rig.settleAccepted(working, 'stopped') + const cleared = await clear() + const replacementId = cleared.ok ? cleared.value.replacementSessionId : undefined + if (!replacementId) { + throw new Error(`expected a replacement session: ${JSON.stringify(cleared)}`) + } + expect(await rig.queuePause(replacementId)).toEqual({ reason: 'cleared' }) + // Idle there, so the person's send goes straight out rather than queueing. + const text = hostTestMessage('hi') + const person = rig.host.send(QUEUED_RIG_CALLER, { + envelope: rig.envelope( + { body: text }, + 'agentSession.send', + hostTestOperationId(), + replacementId + ), + body: text, + userSend: true + }) + const sent = await person + if (!sent.ok || !('submission' in sent.value)) { + throw new Error(`expected an immediate send: ${JSON.stringify(sent)}`) + } + await eventually(async () => + expect( + (await rig.host.journalSnapshot(replacementId)).submissions.find( + (entry) => entry.clientMessageId === sent.value.clientMessageId + )?.handedOverAt + ).toBeDefined() + ) + await rig.host.settleLateDispatch({ + sessionId: replacementId, + clientMessageId: sent.value.clientMessageId, + providerIdentity: { provider: 'codex', threadId: 'thread-1', turnId: 'turn-hi', ordinal: 0 } + }) + expect(journal(replacementId).queuedMessages.list()[0]?.messageId).toBe(carried) + expect(structuredQueuePauses(journal(replacementId))).toEqual([]) + }) +}) + +describe('no stored pause', () => { + it('is read or written across Stop, Resume and /clear', async () => { + const working = await rig.workingSend() + const draftId = await queuedDraft('paused') + await rig.stop() + await rig.settleAccepted(working, 'stopped') + await expectHeld('stopped', draftId) + expect(pauseTables()).toBe(0) + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + expect(pauseTables()).toBe(0) + await eventually(async () => expect((await rig.handoff(draftId))?.handedOverAt).toBeDefined()) + await queuedDraft('carried') + await rig.stop() + await rig.settleAccepted(await rig.handoffId(draftId), 'drained') + const fields = { command: 'clear' as const } + const cleared = await rig.host.conversationCommand(QUEUED_RIG_CALLER, { + envelope: rig.envelope(fields, 'agentSession.conversationCommand', hostTestOperationId()), + ...fields + }) + const replacementId = cleared.ok ? cleared.value.replacementSessionId : undefined + expect(replacementId && (await rig.queuePause(replacementId))).toEqual({ reason: 'cleared' }) + expect(pauseTables()).toBe(0) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts index c715f5accd9..316793751c5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts @@ -1,17 +1,23 @@ -// Stop's pause on the queue. A Stop never withdraws a draft and no text ever -// travels back over the wire: it records WHERE in the journal it took effect, -// and the queue is paused from there (`structured-agent-session-queued-pause.ts` -// derives it) until a turn a person asked for starts, or they Resume. The cards -// stay published, and Send-now sends one card without lifting the pause for the -// rest until that card's turn starts. The record is bookkeeping: a failure is -// reported and never gates the interrupt. +// A Stop's event and the queue. A Stop never withdraws a draft and no text ever +// travels back over the wire: where it takes effect it appends ONE Stop event +// (`JournalStopEvent`), and the queue's pause is derived from it +// (`queued-message-pause.ts`) until a turn a person asked for is sent after it, or +// they Resume. The cards stay published, and Send-now sends one card without lifting +// the pause for the rest until that card's turn starts. The event is bookkeeping: a +// failure is reported and never gates the interrupt. +import type { JournalStopEvent } from '../agent-session-journal/journal-row-schema' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { isUnsettledQueuedMessage, type QueuedMessageRow } from '../agent-session-journal/queued-message-table' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' +import { isMainAgentWorkingOnceFlushed } from './structured-agent-session-turns-cancel' +import { + structuredAgentSessionStopNamesTurnNotLive, + structuredAgentSessionStoppedTurnId +} from './structured-agent-session-turn-stop-notes' /** The one unsettled-card predicate /clear's carry and the budget share: * waiting or returned. Pending/unknown/accepted deliveries stay outside it. */ @@ -20,44 +26,71 @@ export function unsettledQueuedMessages(journal: AgentSessionJournal): QueuedMes } /** - * Runs a Stop and records its queue pause at the point it takes effect — after - * it withdrew the queued sends, as it reaches the agent, or, reaching no agent, - * once it withdrew something — and only over cards it then holds back. The Stop - * calls `tookEffect` there. A Stop that throws before then changed nothing and - * recorded nothing, so there is nothing to undo; one that fails after it keeps - * the pause, since the interrupt may have landed. A draft whose hand-off the - * Stop withdrew is back to waiting in its own place, under this same pause. The - * drain cannot slip a draft in between: it runs on the same serialized lane as - * the Stop. + * Runs a Stop, which calls `tookEffect` where it takes effect: after it withdrew the queued + * sends, and BEFORE the interrupt or anything that ends the child (the at-start stop, a running + * command's stop, a kill after the interrupt), or, reaching no agent, once it withdrew something. + * That writes the Stop's event, whatever the queue holds, so a card its interrupt later withdraws + * comes back to waiting under the pause, and whatever ends the child finds the event already + * written. A Stop that throws before then, or stops nothing (`stopReachesUnrecordedWork`), changed + * nothing and writes nothing. + * The drain cannot slip a card in between: the Stop runs on the drain's serialized lane. */ -export async function runStopWithQueuePause( +export async function runRecordedStop( ctx: AgentSessionTurnContext, + /** `turnId` absent: the turn running when the Stop takes effect, if any. */ + event: Omit, stop: (tookEffect: () => Promise) => Promise> ): Promise> { - let attempted = false - return stop(async () => { - if (attempted) { - return - } - attempted = true - const { queuedMessages } = ctx.journal - // A hand-off this Stop's withdrawal sent back may not have caught up yet (its hook - // was skipped): heal it first, as the drain would, so the pause sees it waiting. + const skipped = (error: unknown): void => report(ctx, 'event row', error) + return stop(() => { try { - if (queuedMessages.settlementOwed()) { - await queuedMessages.settleOwed() - } + const turnId = structuredAgentSessionStoppedTurnId(ctx.journal, event.turnId) ?? undefined + return ctx.journal + .appendStopEvent({ ...event, ...(turnId ? { turnId } : {}) }, ctx.fence) + .then(() => undefined, skipped) } catch (error) { - report(ctx, 'owed settlement', error) + // A throw before the append is queued is reported too: the Stop still interrupts. + skipped(error) + return Promise.resolve() } - // Recorded only over a card it holds back — judged in its own transaction, which - // still counts an owed return to waiting if that heal failed. - await queuedMessages - .recordPause('stopped') - .catch((error: unknown) => report(ctx, 'queue pause', error)) }) } +/** + * Whether a Stop reaching a running agent stops anything no Stop event records yet. Not when it + * names a turn already over (a late Stop from a phone), nor when it repeats the Stop still in force + * with nothing sent since, on the same turn or one that opened after a Stop pressed before any + * turn showed: a card queued between the presses then sends normally, as after one Stop. + */ +export async function stopReachesUnrecordedWork( + ctx: Pick, + namedTurnId: string | undefined +): Promise { + const live = ctx.journal.activeTurnId() + // No turn published yet while the agent works: the named one may still be opening. + if ( + structuredAgentSessionStopNamesTurnNotLive(namedTurnId, live) && + (live !== null || !(await isMainAgentWorkingOnceFlushed(ctx))) + ) { + return false + } + const inForce = ctx.journal.queuedMessages.userStopInForce() + if (inForce === null) { + return true + } + // Sent after that Stop and not refused, even if its fate is unknown: this interrupt may send it + // back to waiting, so this Stop must hold it. A send with no sequence is an older host's. + const sentSince = ctx.journal + .submissions() + .some( + (entry) => + entry.dispatchState !== 'rejected' && + entry.acceptedSequence !== undefined && + entry.acceptedSequence > inForce.sequence + ) + return sentSince || structuredAgentSessionStopNamesTurnNotLive(inForce.event.turnId, live) +} + function report(ctx: AgentSessionTurnContext, step: string, error: unknown): void { ctx.logger.warn(`Stop's ${step} failed`, { scope: 'stop-queued-bookkeeping', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts index f3861f97166..1717f2e7ec3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts @@ -9,7 +9,7 @@ import type { } from './structured-agent-session-host-types' import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' import { StructuredAgentSessionQueuedMessageDrain } from './structured-agent-session-queued-messages' -import { retireEndedQueuePause } from './structured-agent-session-queued-pause' +import { adoptEndedRestartPause } from './structured-agent-session-queued-pause' import { deleteQueuedStructuredAgentMessage, resumeStructuredAgentQueue, @@ -41,12 +41,12 @@ export function wireStructuredAgentSessionQueuedMessages( drain, /** Every journal publish: turn, submission, prompt, command and Stop * settlements are all commits, and each re-derives the drain's gates — - * and retires a queue pause a person's started turn already ended. */ + * and adopts a restart's cards once a person's turn started. */ onJournalActivity: (sessionId: string) => { sessions.touch(sessionId) const journal = sessions.get(sessionId)?.journal if (journal && !journal.isReadOnly) { - void retireEndedQueuePause(sessionId, journal, context().deps.logger) + void adoptEndedRestartPause(sessionId, journal, context().deps.logger) } drain.schedule(sessionId) }, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-withdrawn-draft.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-withdrawn-draft.test.ts index 403cb08a405..7ae2baada74 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-withdrawn-draft.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-withdrawn-draft.test.ts @@ -170,7 +170,8 @@ async function stopWithSkippedSettlement(): Promise<{ a: string; working: string it("a Stop whose withdrawal's settlement was skipped still pauses the card it sent back", async () => { const { a } = await stopWithSkippedSettlement() - expect(await rig.queuePause()).toEqual({ reason: 'stopped' }) + // The drain heals the skipped return; the Stop's row already pauses whatever comes back. + await eventually(async () => expect(await rig.queuePause()).toEqual({ reason: 'stopped' })) await new Promise((resolve) => setTimeout(resolve, 250)) // Healed back to waiting, but the Stop's pause holds it: it does not send. expect(await rig.drafts()).toEqual([{ messageId: a, state: 'waiting' }]) @@ -183,27 +184,3 @@ it("a Stop whose withdrawal's settlement was skipped still pauses the card it se ).toHaveLength(2) ) }) - -it('the pause is recorded even when healing the skipped settlement fails, since the card is still owed', async () => { - const heal = vi - .spyOn(JournalQueuedMessages.prototype, 'settleOwed') - .mockRejectedValueOnce(new Error('disk full')) - try { - const { a } = await stopWithSkippedSettlement() - const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)?.journal - expect(journal?.queuedMessages.pause()).toMatchObject({ reason: 'stopped' }) - // The drain heals it later; the pause recorded over the owed card holds it then. - await eventually(async () => - expect(await rig.drafts()).toEqual([{ messageId: a, state: 'waiting' }]) - ) - await new Promise((resolve) => setTimeout(resolve, 250)) - expect(await rig.queuePause()).toEqual({ reason: 'stopped' }) - expect( - (await rig.host.journalSnapshot(SESSION)).submissions.filter( - (entry) => entry.queuedMessageId === a - ) - ).toHaveLength(1) - } finally { - heal.mockRestore() - } -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-repeated-stop.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-repeated-stop.test.ts index 111671350b0..c0cc13125ca 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-repeated-stop.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-repeated-stop.test.ts @@ -81,6 +81,16 @@ describe('a Stop pressed again', () => { expect(await statusRows()).toEqual([REQUESTED]) }) + it('naming no turn, rewrites the row of the turn running, as a Stop naming it does', async () => { + await attach() + await turn('running') + const unnamed = () => host.cancel(CALLER, { envelope: envelope('agentSession.cancel', {}) }) + expect(await unnamed()).toMatchObject({ ok: true, value: { cancelled: true } }) + expect(await unnamed()).toMatchObject({ ok: true, value: { cancelled: true } }) + expect(await stopTurn()).toMatchObject({ ok: true, value: { cancelled: true } }) + expect(await statusRows()).toEqual([REQUESTED]) + }) + it('writes nothing when neither Stop found anything to stop', async () => { await attach() await turn('running') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts index 51ab878e2c8..4fa2811472a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts @@ -476,6 +476,38 @@ describe('startup restore of chats still in their per-chat files', () => { expect(importCount()).toBe(1) }) + it("copies a chat before a Stop's event, which lands before the turn's end and the kill after it", async () => { + const rows = await seedLegacyChat('chat-a') + const { sessions } = await restore(['chat-a']) + const journal = sessions.get('chat-a')!.journal + expect(journal.importPending).toBe(true) + const scope = { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + + // In the Stop's order: its event, then the stopped turn's end, then a row the kill writes. + const [stopped, ended, killed] = await Promise.all([ + journal.appendStopEvent({ reason: 'user-stop' }, 1), + journal.appendItem( + { provider: 'codex', threadId: 'thread-chat-a', turnId: 't', ordinal: 9 }, + { kind: 'turn', turnId: 't', state: 'interrupted', startedAt: 1 }, + scope + ), + journal.appendItem( + { provider: 'codex', threadId: 'thread-chat-a', turnId: 't', ordinal: 10 }, + { kind: 'status', text: 'the agent ended' }, + scope + ) + ]) + + expect(readTestJournalRows(hostDb(), 'chat-a', rows[0]!.epoch).slice(0, rows.length)).toEqual( + rows + ) + expect([stopped.sequence, ended.cursor.sequence, killed.cursor.sequence]).toEqual([ + rows.length + 1, + rows.length + 2, + rows.length + 3 + ]) + }) + it("copies a chat before its first queued message, which lands as that chat's draft", async () => { const rows = await seedLegacyChat('chat-a') const { sessions } = await restore(['chat-a']) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-cause.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-cause.ts index 8b3078df8df..58a36546a04 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-cause.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-cause.ts @@ -1,4 +1,5 @@ -/** Why a provider child ended. In memory only: never journaled, persisted or sent. */ +/** Why a provider child ended. In memory only, except `user-stop`, the one arm a Stop event + * journals so far. */ export type StructuredAgentSessionChildEndCause = | 'user-stop' /** The user closed this chat: its tab, its launch, or a `/clear` that replaces it. */ @@ -8,7 +9,8 @@ export type StructuredAgentSessionChildEndCause = | 'attach-failed' | 'evict' -/** Why the host asked a child to stop. The adapter carries it onto the `ended` it settles with. */ +/** Why the host asked a child to stop. The adapter carries it onto the `ended` it settles with, + * and a Stop event persists it (`JournalStopEvent.reason`), so never rename an arm. */ export type StructuredAgentSessionStopCause = Extract< StructuredAgentSessionChildEndCause, 'user-stop' | 'user-close' | 'host-stop' | 'evict' diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event.test.ts new file mode 100644 index 00000000000..1ceff1b63a4 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event.test.ts @@ -0,0 +1,293 @@ +// A Stop's event, through the real host: written in the Stop's serialized step once it takes +// effect, before the interrupt and before anything that ends the child, naming the turn and who +// asked; never by a Stop that stopped nothing. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { agentSessionFailureFact } from '../../../shared/agent-session-failure' +import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' +import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' +import type { JournalStopEvent } from '../agent-session-journal/journal-row-schema' +import { HOST_TEST_SESSION, hostTestOperationId } from './structured-agent-session-host-test-data' +import { + createQueuedMessageTestRig, + eventually, + QUEUED_RIG_CALLER, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' + +let rig: QueuedMessageTestRig + +afterEach(() => rig.dispose()) + +function journal() { + const open = rig.host.collaboratorsForTests().sessions.get(HOST_TEST_SESSION)?.journal + if (!open) { + throw new Error('expected the conversation open') + } + return open +} + +/** Every Stop event in the live epoch, oldest first. */ +function stopEvents(): JournalStopEvent[] { + const since = journal().readSince({ epoch: journal().epoch, sequence: 0 }) + if (!since.ok) { + throw new Error(`expected rows, got reset ${since.reset}`) + } + return since.rows.flatMap((row) => + row.kind === 'tombstone' && row.stopEvent ? [row.stopEvent] : [] + ) +} + +async function queuedDraft(text: string): Promise { + const queued = await rig.send(text, 'queue-if-active').result + if (!queued.ok || !('queued' in queued.value)) { + throw new Error(`expected a queued receipt: ${JSON.stringify(queued)}`) + } + return queued.value.queued.messageId +} + +/** The provider's turn row for the working send, which may land after a Stop. */ +async function turnRow(turnId: string, state: 'running' | 'interrupted') { + return journal().appendItem( + { provider: 'codex', threadId: 'thread-1', turnId, ordinal: 999 }, + { kind: 'turn', turnId, state, startedAt: 1 }, + { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) +} + +function withdraw(clientMessageId: string) { + return rig.host.settleLateDispatch({ + sessionId: HOST_TEST_SESSION, + clientMessageId, + state: 'rejected', + ...agentSessionFailureWords(agentSessionFailureFact('cancelled'), { surface: 'rejection' }) + }) +} + +/** Holds every start until the returned release. */ +function holdStart(): () => void { + let release: () => void = () => undefined + rig.awaitStarted.mockImplementation( + () => new Promise((resolve) => (release = () => resolve(undefined))) + ) + return () => release() +} + +describe("a Stop's event", () => { + it('reaches the journal before the interrupt, naming the turn it stopped and who asked', async () => { + rig = await createQueuedMessageTestRig() + await rig.workingSend() + let atInterrupt: JournalStopEvent[] = [] + rig.cancelTurn.mockImplementationOnce(async () => { + atInterrupt = stopEvents() + return { cancelled: true } + }) + const fields = { turnId: 'turn-named' } + const stopped = await rig.host.cancel(QUEUED_RIG_CALLER, { + envelope: rig.envelope(fields, 'agentSession.cancel', hostTestOperationId()), + ...fields + }) + expect(stopped).toMatchObject({ ok: true }) + expect(rig.cancelTurn).toHaveBeenCalledTimes(1) + expect(atInterrupt).toEqual([ + { + reason: 'user-stop', + turnId: 'turn-named', + caller: QUEUED_RIG_CALLER.callerKey, + at: expect.any(Number) + } + ]) + }) + + it("lands before the rows the interrupt causes: the stopped turn's end comes after it", async () => { + rig = await createQueuedMessageTestRig() + await rig.workingSend() + let turnEnd: number | undefined + rig.cancelTurn.mockImplementationOnce(async () => { + // As a provider answers an interrupt: the stopped turn's end, journaled before it returns. + turnEnd = (await turnRow('turn-1', 'interrupted')).cursor.sequence + return { cancelled: true } + }) + await rig.stop() + const since = journal().readSince({ epoch: journal().epoch, sequence: 0 }) + const stopRow = since.ok + ? since.rows.find((row) => row.kind === 'tombstone' && row.stopEvent) + : undefined + expect(stopRow?.seq).toBeLessThan(turnEnd ?? 0) + }) + + it('a write that throws before it is queued is reported, and the Stop still interrupts', async () => { + rig = await createQueuedMessageTestRig() + await rig.workingSend() + const warned = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.spyOn(journal(), 'appendStopEvent').mockImplementation(() => { + throw new Error('the journal threw') + }) + expect(await rig.stop()).toMatchObject({ ok: true, value: { cancelled: true } }) + expect(rig.cancelTurn).toHaveBeenCalledTimes(1) + expect(warned).toHaveBeenCalledWith( + "[agent-session] stop-queued-bookkeeping: Stop's event row failed", + expect.objectContaining({ step: 'event row', error: new Error('the journal threw') }) + ) + warned.mockRestore() + }) + + it('at an agent still starting, reaches the journal before the start is ended, and holds a card queued before it', async () => { + rig = await createQueuedMessageTestRig({ starting: true, restartable: true }) + const release = holdStart() + rig.send('work on this') + const held = await queuedDraft('queued while it starts') + let atEnd: JournalStopEvent[] | undefined + rig.closeSession.mockImplementationOnce(async () => { + atEnd = stopEvents() + return true + }) + expect(await rig.stop()).toMatchObject({ ok: true, value: { cancelled: true } }) + release() + await new Promise((resolve) => setTimeout(resolve, 250)) + expect(await rig.handoff(held)).toBeUndefined() + expect(await rig.queuePause()).toEqual({ reason: 'stopped' }) + expect(rig.cancelTurn).not.toHaveBeenCalled() + expect(rig.closeSession).toHaveBeenCalledTimes(1) + expect(atEnd).toEqual([ + { reason: 'user-stop', caller: QUEUED_RIG_CALLER.callerKey, at: expect.any(Number) } + ]) + }) + + it('is written by an idle Stop only when it withdrew a send', async () => { + rig = await createQueuedMessageTestRig() + expect(await rig.stop()).toMatchObject({ ok: true, value: { cancelled: false } }) + expect(stopEvents()).toEqual([]) + const release = holdStart() + const waiting = rig.send('waits for the start') + await eventually(async () => expect(await rig.submission(waiting.id)).toBeDefined()) + expect(await rig.stop()).toMatchObject({ ok: true, value: { cancelled: true } }) + release() + expect(rig.cancelTurn).not.toHaveBeenCalled() + expect(stopEvents()).toEqual([ + { reason: 'user-stop', caller: QUEUED_RIG_CALLER.callerKey, at: expect.any(Number) } + ]) + }) + + it('a second press while the first interrupt lands writes nothing: a card queued between them sends normally', async () => { + rig = await createQueuedMessageTestRig() + const working = await rig.workingSend() + expect(await rig.stop()).toMatchObject({ ok: true, value: { cancelled: true } }) + const between = await queuedDraft('queued between the presses') + expect(await rig.stop()).toMatchObject({ ok: true }) + expect(stopEvents()).toHaveLength(1) + await rig.settleAccepted(working, 'stopped') + await eventually(async () => expect(await rig.handoff(between)).toBeDefined()) + }) + + it('a second press once the turn shows, after a first before it did, writes nothing: a card queued between sends', async () => { + rig = await createQueuedMessageTestRig() + const working = await rig.workingSend() + await rig.stop() + const between = await queuedDraft('queued between the presses') + await turnRow('turn-1', 'running') + await rig.stop() + expect(stopEvents()).toHaveLength(1) + await rig.settleAccepted(working, 'stopped') + await turnRow('turn-1', 'interrupted') + await eventually(async () => expect(await rig.handoff(between)).toBeDefined()) + }) + + it('a second press after a card sent into the turn settled unknown writes again', async () => { + rig = await createQueuedMessageTestRig() + await rig.workingSend() + await rig.stop() + const steered = await queuedDraft('sent into the turn between the presses') + await rig.sendNow(steered) + await eventually(async () => expect((await rig.handoff(steered))?.handedOverAt).toBeDefined()) + await rig.host.settleLateDispatch({ + sessionId: HOST_TEST_SESSION, + clientMessageId: await rig.handoffId(steered), + state: 'unknown', + reason: 'the provider never answered' + }) + await rig.stop() + expect(stopEvents()).toHaveLength(2) + }) + + it('a second press after a card was sent into the turn writes again, and holds that card', async () => { + rig = await createQueuedMessageTestRig() + const working = await rig.workingSend() + await rig.stop() + const steered = await queuedDraft('sent into the turn between the presses') + await rig.sendNow(steered) + await eventually(async () => expect((await rig.handoff(steered))?.handedOverAt).toBeDefined()) + await rig.stop() + expect(stopEvents()).toHaveLength(2) + await withdraw(await rig.handoffId(steered)) + await rig.settleAccepted(working, 'stopped') + await new Promise((resolve) => setTimeout(resolve, 250)) + expect(await rig.drafts()).toEqual([{ messageId: steered, state: 'waiting' }]) + expect(await rig.queuePause()).toEqual({ reason: 'stopped' }) + }) + + it('naming no turn, names the turn running when it takes effect', async () => { + rig = await createQueuedMessageTestRig() + await rig.workingSend() + await turnRow('turn-1', 'running') + await rig.stop() + expect(stopEvents()).toMatchObject([{ reason: 'user-stop', turnId: 'turn-1' }]) + }) + + it('names a turn already over, as a late Stop from a phone does: writes nothing', async () => { + rig = await createQueuedMessageTestRig() + rig.cancelTurn.mockResolvedValueOnce({ cancelled: false }) + const fields = { turnId: 'turn-already-over' } + const stopped = await rig.host.cancel(QUEUED_RIG_CALLER, { + envelope: rig.envelope(fields, 'agentSession.cancel', hostTestOperationId()), + ...fields + }) + expect(stopped).toMatchObject({ ok: true, value: { cancelled: false } }) + expect(stopEvents()).toEqual([]) + }) + + it('names a turn that ended while the next card is sent but shows no turn yet: writes, and holds that card', async () => { + rig = await createQueuedMessageTestRig() + const working = await rig.workingSend() + await turnRow('turn-1', 'running') + const next = await queuedDraft('sent when turn-1 ends') + await rig.settleAccepted(working, 'working') + await turnRow('turn-1', 'interrupted') + await eventually(async () => expect((await rig.handoff(next))?.handedOverAt).toBeDefined()) + expect(journal().activeTurnId()).toBeNull() + const fields = { turnId: 'turn-1' } + await rig.host.cancel(QUEUED_RIG_CALLER, { + envelope: rig.envelope(fields, 'agentSession.cancel', hostTestOperationId()), + ...fields + }) + expect(stopEvents()).toHaveLength(1) + await withdraw(await rig.handoffId(next)) + await new Promise((resolve) => setTimeout(resolve, 250)) + expect(await rig.drafts()).toEqual([{ messageId: next, state: 'waiting' }]) + expect(await rig.queuePause()).toEqual({ reason: 'stopped' }) + }) + + it("holds a card when it lands between the queue's pick and its claim", async () => { + rig = await createQueuedMessageTestRig() + const working = await rig.workingSend() + const draftId = await queuedDraft('picked by the drain') + const open = journal() + const appendSubmission = open.appendSubmission.bind(open) + let injected = false + // Stop and the drain share one serialized lane, so this interleaving is forced: a pause + // written after the drain chose the card must still hold it in the claim's transaction. + vi.spyOn(open, 'appendSubmission').mockImplementation(async (input, consume) => { + if (input.origin === 'host' && consume?.messageId === draftId && !injected) { + injected = true + await open.appendStopEvent({ reason: 'user-stop' }, input.fence) + } + return appendSubmission(input, consume) + }) + await rig.settleAccepted(working, 'working') + await eventually(() => expect(injected).toBe(true)) + await new Promise((resolve) => setTimeout(resolve, 250)) + expect(await rig.handoff(draftId)).toBeUndefined() + expect(await rig.drafts()).toEqual([{ messageId: draftId, state: 'waiting' }]) + expect(await rig.queuePause()).toEqual({ reason: 'stopped' }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turn-stop-notes.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turn-stop-notes.ts index 676b3179765..0f86c053ad6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turn-stop-notes.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turn-stop-notes.ts @@ -1,5 +1,6 @@ -// A Stop's note sits on the turn it stopped, found by the turn's id whether it still runs or has -// ended, and keyed by that turn, so a repeated Stop rewrites the one row instead of adding one. +// The turn a Stop is about, read once for its event and its note. The note sits on that turn, found +// by the turn's id whether it still runs or has ended, and keyed by it, so a repeated Stop rewrites +// the one row instead of adding one. import type { AgentJournalTurnScope } from '../../../shared/agent-session-journal-types' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' @@ -7,6 +8,22 @@ import type { AgentSessionJournal } from '../agent-session-journal/journal-store export const STOP_NOTE_CANCELLATION_REQUESTED = 'Cancellation requested.' +/** The turn a Stop is about: the one it named, else the one running when it is read. */ +export function structuredAgentSessionStoppedTurnId( + journal: Pick, + namedTurnId: string | undefined +): string | null { + return namedTurnId ?? journal.activeTurnId() +} + +/** A Stop names a turn the journal does not show running, as a phone does once that turn ended. */ +export function structuredAgentSessionStopNamesTurnNotLive( + namedTurnId: string | undefined, + liveTurnId: string | null +): boolean { + return namedTurnId !== undefined && namedTurnId !== liveTurnId +} + /** The scope of the turn `turnId` names, running or ended; null when the journal has no such turn. */ export function structuredAgentSessionNamedTurnScope( journal: Pick, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts index 57aa06299fc..30809bb5cd2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts @@ -17,7 +17,9 @@ import { } from './structured-agent-session-prompt-state' import { STOP_NOTE_CANCELLATION_REQUESTED, - structuredAgentSessionNamedTurnScope + structuredAgentSessionNamedTurnScope, + structuredAgentSessionStopNamesTurnNotLive, + structuredAgentSessionStoppedTurnId } from './structured-agent-session-turn-stop-notes' import { isStructuredAgentSessionMainAgentWorking } from '../../../shared/structured-agent-session-main-agent-working' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' @@ -143,11 +145,14 @@ export async function performCancel( // turn for, would not interrupt, or was already asked to stop, ends with its child; that child's // dead-generation settlement writes the command's verdict. const liveTurnId = ctx.journal.activeTurnId() + // Read with the live turn, before the cancel settles it: the note is keyed by this turn. + const stoppedTurnId = structuredAgentSessionStoppedTurnId(ctx.journal, input.turnId) + const namesTurnNotLive = structuredAgentSessionStopNamesTurnNotLive(input.turnId, liveTurnId) const runningCommand = input.stopChild !== undefined && liveTurnId !== null && isStructuredAgentSessionCommandTurnId(liveTurnId) && - (input.turnId === undefined || input.turnId === liveTurnId) + !namesTurnNotLive const stoppedBefore = runningCommand && structuredAgentSessionCommandWasStopped(ctx.journal, liveTurnId) // Read while the child is live: a provider whose Stop is a session boundary loses it next. @@ -211,10 +216,7 @@ export async function performCancel( } // A Stop naming a turn that has since ended keeps the session only when the provider declined // it: an interrupt, answered or not, can stop a follow-up whose turn has not opened. - if ( - endsSession && - (input.turnId === undefined || input.turnId === liveTurnId || taken !== false) - ) { + if (endsSession && (!namesTurnNotLive || taken !== false)) { // An interrupt the provider took is worth waiting on, turn row or not: a Stop before the echo // has none, and the echo still opens the turn the Stop interrupted. input.endSession?.({ waitsForProvider: taken === true, stoppedAt }) @@ -232,7 +234,7 @@ export async function performCancel( interruptFailed && input.stopChild && // An unnamed Stop meant the turn the journal showed when it was sent. - (await stillRunsStoppedTurn(ctx, input.turnId ?? liveTurnId)) + (await stillRunsStoppedTurn(ctx, stoppedTurnId)) ) { // The interrupt failed and the turn runs on: only the child's end stops it. let ended: boolean @@ -264,7 +266,7 @@ export async function performCancel( } // Keyed by the turn it stopped, so another Stop of that turn rewrites this row, never adds one. await ctx.journal.appendItem( - structuredAgentSessionStopNoteIdentity(input.turnId ?? liveTurnId ?? input.clientOperationId), + structuredAgentSessionStopNoteIdentity(stoppedTurnId ?? input.clientOperationId), note, { fence: ctx.fence, turnScope } ) diff --git a/src/main/runtime/structured-agent-session-codex-turn-end-settlement.test.ts b/src/main/runtime/structured-agent-session-codex-turn-end-settlement.test.ts index 6545ffb2ecf..36ab233a915 100644 --- a/src/main/runtime/structured-agent-session-codex-turn-end-settlement.test.ts +++ b/src/main/runtime/structured-agent-session-codex-turn-end-settlement.test.ts @@ -289,6 +289,64 @@ describe('a Codex send its turn ended without taking it', () => { }) }) +describe('a queued card sent now into the turn a Stop ends', () => { + async function handoffs(messageId: string): Promise { + return (await settled()).submissions.filter((entry) => entry.queuedMessageId === messageId) + } + + /** Each hand-off of the card, as who sent it and how it settled. */ + async function sends(messageId: string) { + return (await handoffs(messageId)).map((entry) => ({ + origin: entry.origin, + verdict: verdictOf([entry], entry.clientMessageId) + })) + } + + async function queue() { + const page = await host.history({ sessionId: SESSION, direction: 'tail' }) + if (!page.ok) { + throw new Error('history refused') + } + return { + pause: page.page.queuePause ?? null, + cards: (page.page.queuedMessages ?? []).map(({ messageId, state }) => ({ messageId, state })) + } + } + + it('comes back as a paused waiting card, and nothing sends it again', async () => { + const opening = await send('look around') + await vi.waitFor(() => expect(answers).toBe(1)) + turns.start() + turns.echo(opening) + const { messageId: cardId } = await queueThenSendNow('and check the tests') + // Steered into the running turn, with no echo yet. + await vi.waitFor(() => expect(steers).toBe(1)) + const [steered] = await handoffs(cardId) + expect(steered?.dispatchState).toBe('pending') + + await stop('turn-1') + + await vi.waitFor(async () => { + const [withdrawn] = await handoffs(cardId) + expect(verdictOf([withdrawn!], withdrawn!.clientMessageId)).toBe('withdrawn') + }) + await vi.waitFor( + async () => + expect({ ...(await queue()), sends: await sends(cardId) }).toEqual({ + pause: { reason: 'stopped' }, + cards: [{ messageId: cardId, state: 'waiting' }], + sends: [{ origin: 'client', verdict: 'withdrawn' }] + }), + { timeout: 5_000 } + ) + // A drain ignoring the pause re-sends only after the stopped turn ends: watch past that. + await vi.waitFor(() => expect(turns.turnId).toBeNull()) + await new Promise((resolve) => setTimeout(resolve, 2_500)) + expect(steers + answers).toBe(2) + expect(await sends(cardId)).toEqual([{ origin: 'client', verdict: 'withdrawn' }]) + }, 20_000) +}) + describe('a Codex before 0.148, which names a steered start falsely', () => { it('withdraws a send made while a turn runs when a Stop ends it, and then takes /compact', async () => { turns = codexTurnLifecycleFake( diff --git a/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts b/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts new file mode 100644 index 00000000000..0a9b5ea5f35 --- /dev/null +++ b/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts @@ -0,0 +1,224 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, test } from 'vitest' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemIdentity, + type AgentSessionJournalIdentity +} from '../../../src/shared/agent-session-journal-types' +import Database from '../../../src/main/sqlite/sync-database' +import { journalDatabasePath } from '../../../src/main/native-chat/agent-session-journal/journal-host-database' +import { + createTrackedJournalOpener, + liveTestJournalRows +} from '../../../src/main/native-chat/agent-session-journal/journal-host-database-test-support' +import type { JournalRow } from '../../../src/main/native-chat/agent-session-journal/journal-row-schema' +import { importReleaseCheckoutModule, materializeReleaseCheckout } from './release-checkout' + +// A release that knows neither the Stop event nor the Resume marker: an unknown row kind would +// make it delete the journal from that row on, so both ride a tombstone it already reads. +const BASELINE_REF = 'v1.4.218' +const JOURNAL = 'src/main/native-chat/agent-session-journal' +// A main build that shares this one's host database and schema version, so a downgrade to it opens +// the journal writable. No release tag has that database yet; move to the first one that does. +const WRITABLE_BASELINE_REF = '3727100cc9dbcea6201f8a3e506676a3c4b53b18' + +const IDENTITY: AgentSessionJournalIdentity = { + sessionId: 'session-downgrade', + workspaceId: 'ws-1', + hostId: 'host-1', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: 'thread-1' } +} + +function item(ordinal: number): AgentJournalItemIdentity { + return { provider: 'codex', threadId: 'thread-1', turnId: 'turn-1', ordinal } +} + +/** A function the pinned release exports, typed as the caller calls it. */ +function releaseExport(module: Record, name: string): T { + const value = module[name] + if (typeof value !== 'function') { + throw new Error(`the pinned release exports no ${name}`) + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a function the pinned release exports; each caller names the signature it calls, and a changed one fails the test. + return value as T +} + +type OldReplay = { + state: { items: Map } + readOnly: boolean + corrupt: boolean + malformedRows: number + truncateFrom?: number +} + +test("an older build keeps every row around a Stop's event and a Resume, and folds the rows after them", async () => { + const directory = mkdtempSync(join(tmpdir(), 'orca-stop-event-downgrade-')) + const journals = createTrackedJournalOpener() + try { + // This build: history, a person's Stop, a Resume, then more history. + const journal = await journals.open({ identity: IDENTITY, stateDirectory: directory }) + const append = (ordinal: number, text: string) => + journal.appendItem( + item(ordinal), + { kind: 'status', text }, + { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + await append(0, 'before the Stop') + const beforeMarks = journal.cursor() + await journal.appendStopEvent({ reason: 'user-stop', turnId: 'turn-1', caller: 'client-1' }, 1) + await journal.appendQueueResume(1) + const afterMarks = journal.cursor() + await append(1, 'after the Stop') + const since = journal.readSince({ epoch: journal.epoch, sequence: 0 }) + if (!since.ok) { + throw new Error(`expected rows, got reset ${since.reset}`) + } + const rows: JournalRow[] = since.rows + + // The older build, after a downgrade, replays the same rows from its own database. + const checkout = await materializeReleaseCheckout(BASELINE_REF) + const [database, table, open, reducer, batch] = await Promise.all( + [ + `${JOURNAL}/journal-database.ts`, + `${JOURNAL}/journal-row-table.ts`, + `${JOURNAL}/journal-open.ts`, + `${JOURNAL}/journal-reducer.ts`, + 'src/main/native-chat/agent-session-wire/agent-session-journal-batch.ts' + ].map((path) => importReleaseCheckoutModule(checkout, path)) + ) + const openJournalDatabase = releaseExport<(path: string) => { db: { close: () => void } }>( + database, + 'openJournalDatabase' + ) + const upsertJournalSessionRow = releaseExport< + (...args: [unknown, string, string, number]) => void + >(table, 'upsertJournalSessionRow') + const insertJournalRow = releaseExport<(...args: [unknown, string, JournalRow]) => void>( + table, + 'insertJournalRow' + ) + const replayJournal = releaseExport<(...args: [unknown, boolean, string]) => OldReplay | null>( + open, + 'replayJournal' + ) + const renderJournalState = releaseExport<(state: unknown) => unknown>( + reducer, + 'renderJournalState' + ) + const projectJournalBatch = releaseExport< + (input: { rows: readonly JournalRow[]; snapshot: unknown; afterSequence: number }) => { + ok: boolean + batch?: { items: unknown[]; removedItemIds: string[] } + } + >(batch, 'projectJournalBatch') + + const { db } = openJournalDatabase(join(directory, 'older-build-journal.sqlite')) + try { + upsertJournalSessionRow(db, IDENTITY.sessionId, journal.epoch, 1) + for (const row of rows) { + insertJournalRow(db, IDENTITY.sessionId, row) + } + const replayed = replayJournal(db, false, IDENTITY.sessionId) + expect(replayed).toMatchObject({ readOnly: false, corrupt: false, malformedRows: 0 }) + expect(replayed?.truncateFrom).toBeUndefined() + expect([...(replayed?.state.items.keys() ?? [])]).toHaveLength(2) + + // An older client is sent only ids no item uses, removed. + const projected = projectJournalBatch({ + rows: rows.filter( + (row) => row.seq > beforeMarks.sequence && row.seq <= afterMarks.sequence + ), + snapshot: renderJournalState(replayed?.state), + afterSequence: beforeMarks.sequence + }) + expect(projected.ok).toBe(true) + expect(projected.batch?.items).toEqual([]) + expect(projected.batch?.removedItemIds).toHaveLength(2) + const liveIds = new Set(journal.snapshot().items.map((entry) => entry.itemId)) + expect(projected.batch?.removedItemIds.some((id) => liveIds.has(id))).toBe(false) + } finally { + db.close() + } + } finally { + await journals.closeAll() + rmSync(directory, { recursive: true, force: true }) + } +}) + +type OlderJournal = { + isReadOnly: boolean + cursor: () => { epoch: string; sequence: number } + snapshot: () => { items: { itemId: string }[] } + appendItem: (...args: [AgentJournalItemIdentity, unknown, unknown]) => Promise +} + +type OlderOpener = { + open: (options: { + identity: AgentSessionJournalIdentity + stateDirectory: string + }) => Promise + closeAll: () => Promise +} + +function storedRows(directory: string): string[] { + const db = new Database(journalDatabasePath(directory), { readonly: true }) + try { + return liveTestJournalRows(db, IDENTITY.sessionId).map((row) => row.rowJson) + } finally { + db.close() + } +} + +test("an older build opens this build's journal writable and appends to it; the pause survives the round trip", async () => { + const directory = mkdtempSync(join(tmpdir(), 'orca-stop-event-writable-downgrade-')) + const journals = createTrackedJournalOpener() + const itemIds = (journal: Pick) => + journal.snapshot().items.map((entry) => entry.itemId) + try { + const journal = await journals.open({ identity: IDENTITY, stateDirectory: directory }) + const scope = { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + await journal.appendItem(item(0), { kind: 'status', text: 'before the Stop' }, scope) + await journal.appendStopEvent({ reason: 'user-stop', turnId: 'turn-1', caller: 'client-1' }, 1) + await journal.appendItem(item(1), { kind: 'status', text: 'after the Stop' }, scope) + const wrote = { cursor: journal.cursor(), items: itemIds(journal) } + expect(journal.queuedMessages.pauses('host-a').map((pause) => pause.reason)).toEqual([ + 'stopped' + ]) + await journals.closeAll() + const rowsBefore = storedRows(directory) + + const checkout = await materializeReleaseCheckout(WRITABLE_BASELINE_REF) + const support = await importReleaseCheckoutModule( + checkout, + `${JOURNAL}/journal-host-database-test-support.ts` + ) + const older = releaseExport<() => OlderOpener>(support, 'createTrackedJournalOpener')() + try { + const downgraded = await older.open({ identity: IDENTITY, stateDirectory: directory }) + expect(downgraded.isReadOnly).toBe(false) + expect(downgraded.cursor()).toEqual(wrote.cursor) + expect(itemIds(downgraded)).toEqual(wrote.items) + await downgraded.appendItem(item(2), { kind: 'status', text: 'the older build' }, scope) + } finally { + await older.closeAll() + } + const rowsAfter = storedRows(directory) + expect(rowsAfter.slice(0, rowsBefore.length)).toEqual(rowsBefore) + expect(rowsAfter).toHaveLength(rowsBefore.length + 1) + + // Upgraded again: the older build's row folds, and the person's Stop still pauses the queue. + const upgraded = await journals.open({ identity: IDENTITY, stateDirectory: directory }) + expect(upgraded.isReadOnly).toBe(false) + expect(upgraded.cursor().sequence).toBe(wrote.cursor.sequence + 1) + expect(itemIds(upgraded)).toEqual([...wrote.items, 'codex:thread-1:turn-1:2']) + expect(upgraded.queuedMessages.pauses('host-a').map((pause) => pause.reason)).toEqual([ + 'stopped' + ]) + } finally { + await journals.closeAll() + rmSync(directory, { recursive: true, force: true }) + } +}, 120_000)