diff --git a/config/tsconfig.node.json b/config/tsconfig.node.json index b029a12c3d2..f460d89a058 100644 --- a/config/tsconfig.node.json +++ b/config/tsconfig.node.json @@ -26,6 +26,7 @@ "../src/renderer/src/components/native-chat/structured-agent-question-projection.ts", "../src/renderer/src/components/native-chat/native-chat-subagent-delegation.ts", "../src/renderer/src/components/native-chat/structured-agent-session-pending-sends.ts", + "../src/renderer/src/components/native-chat/structured-agent-session-queue-request.ts", "../src/renderer/src/components/native-chat/native-chat-composer-primary-action.ts", "../src/renderer/src/components/terminal-pane/legacy-relay-bridge-panes.ts", "../src/preload/**/*", diff --git a/src/main/acp/acp-structured-launch-resolution.test.ts b/src/main/acp/acp-structured-launch-resolution.test.ts index 913f5ea6cd2..d8d8086ee47 100644 --- a/src/main/acp/acp-structured-launch-resolution.test.ts +++ b/src/main/acp/acp-structured-launch-resolution.test.ts @@ -20,6 +20,7 @@ import { } from '../native-chat/agent-session-journal/journal-host-database-test-support' import type { JournalLoad } from '../native-chat/agent-session-journal/journal-open' import { publishJournalOutcomeEpoch } from '../native-chat/agent-session-journal/journal-outcome-write' +import { JOURNAL_OUTCOME_FORMAT } from '../native-chat/agent-session-journal/journal-outcome-format' import type { JournalRow } from '../native-chat/agent-session-journal/journal-row-schema' import { createJournalReducerState } from '../native-chat/agent-session-journal/journal-reducer' import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key' @@ -226,8 +227,13 @@ describe('ACP launch resolution', () => { // The turn on acp-1 is found, so the exchange is saved; the loss has no warning yet. expect(await readDecisions()).toEqual({ unsaved: false, losses: [lostKey] }) // A newer build's records are unreadable here, never a missing chat. - database.db.prepare('UPDATE journal_outcome_epochs SET format = 2').run() - expect(restoreJournal(database.db, identity.sessionId)?.newer).toBeTruthy() + database.db + .prepare('UPDATE journal_outcome_epochs SET format = ?') + .run(JOURNAL_OUTCOME_FORMAT + 1) + const newer = restoreJournal(database.db, identity.sessionId) + expect(newer).not.toBeNull() + expect(newer?.newer).toStrictEqual({ sequence: 1 }) + expect(newer?.damage).toBeNull() expect(await readDecisions()).toEqual({ unsaved: false, losses: [] }) } finally { await closeTestJournalHostDatabase(root) diff --git a/src/main/native-chat/agent-session-journal/journal-commit.ts b/src/main/native-chat/agent-session-journal/journal-commit.ts new file mode 100644 index 00000000000..9b70093c46b --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-commit.ts @@ -0,0 +1,12 @@ +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import type { JournalReducerState } from './journal-reducer' + +export function retireCoveredJournalFloor( + state: JournalReducerState, + reopenFloor: AgentJournalCursor | null +): AgentJournalCursor | null { + return reopenFloor?.epoch === state.epoch && + state.queuePauseMarks.reopenedSequence >= reopenFloor.sequence + ? null + : reopenFloor +} diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts index c39a1fbbbad..6efcdd91cde 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts @@ -1,3 +1,4 @@ +import { journalInputOrder } from './journal-input-order' import { journalOutcomeSource } from './journal-outcome-format' import { publishJournalOutcomeEpoch } from './journal-outcome-write' import { @@ -53,6 +54,8 @@ import { AgentSessionJournalError, assertJournalFence } from './journal-write-gu import type { JournalOperationReceipt } from './journal-row-writer' import type { JournalAttachmentClaim } from './journal-submission-hook' +import { copyJournalNativeState } from './journal-input-conversion' +import { journalInputAcceptance, journalQueueRebase } from './journal-input-provenance' export type JournalReplacementItem = AgentJournalProducerLinkage & { echoOf?: JournalItemEcho['echoOf'] @@ -88,7 +91,13 @@ export function replaceJournalEpoch(input: { onPublished: (loaded: JournalLoad) => void }): void { const epoch = input.mintEpoch() - const state = createJournalReducerState(input.identity.sessionId, epoch) + const state = createJournalReducerState(input.identity.sessionId, epoch, (id) => + journalInputAcceptance(input.database.db, input.identity.sessionId, id) + ) + if (input.previous) { + copyJournalNativeState(input.previous, state) + } + journalInputOrder(state).beginReconstruction() const providerHandle = agentSessionJournalProviderHandle(input.identity) const previousOrigin = input.previous?.historyOrigin const restate = @@ -96,6 +105,9 @@ export function replaceJournalEpoch(input: { input.reason !== 'session_created' && previousOrigin && sameJournalConversation(previousOrigin.providerHandle, providerHandle) + const queued = [...(input.previous?.submissions.values() ?? [])] + .filter(isQueuedAgentJournalSubmission) + .sort((a, b) => a.submittedSequence! - b.submittedSequence!) const epochRow: JournalRow = { kind: 'epoch', reason: @@ -133,23 +145,12 @@ export function replaceJournalEpoch(input: { applyJournalRow(state, row) rows.push(row) } - for (const { messageId, change } of input.queue.cards) { - if (change) { - const row = buildJournalQueueRow( - state, - messageId, - change, - state.lastSequence + 1, - input.now() - ) - applyJournalRow(state, row) - rows.push(row) - } + const appendQueue = (messageId: string, change: Parameters[2]) => { + const row = buildJournalQueueRow(state, messageId, change, state.lastSequence + 1, input.now()) + applyJournalRow(state, row) + rows.push(row) } - // Accepted-but-unhanded messages must survive provider history replacement. - const queued = [...(input.previous?.submissions.values() ?? [])] - .filter(isQueuedAgentJournalSubmission) - .sort((a, b) => (a.acceptedSequence ?? 0) - (b.acceptedSequence ?? 0)) + const rebuilt = new Map() for (const submission of queued) { const body = input.previous?.items.get( agentJournalSubmissionKey(submission.clientMessageId) @@ -168,6 +169,34 @@ export function replaceJournalEpoch(input: { }) applyJournalRow(state, row) rows.push(row) + rebuilt.set(submission.clientMessageId, row.seq) + } + const rebase = journalQueueRebase( + input.database.db, + input.identity.sessionId, + epoch, + input.previous?.epoch, + queued, + rebuilt + ) + for (const { messageId, change } of input.queue.cards) { + if (!change) { + continue + } + const prior = input.previous?.queue.cards.get(messageId) + const queuedAt = change.op === 'queued' ? rebase(change.queuedAt) : null + appendQueue( + messageId, + change.op === 'queued' + ? { + ...change, + queuedAt, + ...(prior?.state === 'dispatched' && prior.consumedAs + ? { attempt: prior.consumedAs } + : {}) + } + : change + ) } const { lifted, liveStop, reopened } = input.queuePause const place = () => ({ state, seq: state.lastSequence + 1, fence: input.fence, ts: input.now() }) @@ -192,6 +221,8 @@ export function replaceJournalEpoch(input: { rows.push(row) } + journalInputOrder(state).restore(state) + journalInputOrder(state).finishReconstruction(state) const { sessionId } = input.identity const receipt = input.receipt?.({ epoch, sequence: state.lastSequence }) const claimsCommitted: (() => void)[] = [] @@ -225,7 +256,7 @@ export function replaceJournalEpoch(input: { input.queue.receipt.write(db) publishJournalSessionEpoch(db, input.identity, epoch) receipt?.write(db) - return publishJournalOutcomeEpoch(db, sessionId, state.epoch, rows) + return publishJournalOutcomeEpoch(db, sessionId, state.epoch, rows, input.previous) }, false, (publication) => { diff --git a/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts b/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts index 93d11ac3a22..edf1daa6134 100644 --- a/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts +++ b/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts @@ -1,3 +1,4 @@ +import type { AgentSessionJournalProviderHandle } from '../../../shared/agent-session-journal-types' // Share one connection per test profile to match production host ownership. import { resolve } from 'node:path' @@ -197,6 +198,44 @@ export function deleteTestJournalRow(db: Database.Database, sessionId: string, s ) } +/** Unannotated submissions and terminal rows emitted by the legacy writer in one source epoch. */ +export function writeTestLegacyTerminalGaps( + db: Database.Database, + sessionId: string, + base: { + epoch: string + sequence: number + fence: number + providerHandle: AgentSessionJournalProviderHandle + }, + n: number +): number { + let seq = base.sequence + const row = () => ({ v: 3, epoch: base.epoch, seq: ++seq, fence: base.fence, ts: 1 }) + for (let i = 0; i < n; i++) { + const id = `terminal-${i}` + insertTestJournalRow(db, sessionId, { + ...row(), + kind: 'submission', + clientMessageId: id, + payloadFingerprint: id, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: id }] }, + providerHandle: base.providerHandle, + handoverRecorded: true, + origin: 'client' + }) + insertTestJournalRow(db, sessionId, { + ...row(), + kind: 'dispatch', + clientMessageId: id, + state: 'rejected', + providerItemId: null, + reason: 'not sent' + }) + } + return seq +} + /** Read current canonical contributions in commit order for persisted-outcome assertions. */ export function readTestJournalOutcomes( db: Database.Database, diff --git a/src/main/native-chat/agent-session-journal/journal-input-card-index.ts b/src/main/native-chat/agent-session-journal/journal-input-card-index.ts new file mode 100644 index 00000000000..37cdfac5f23 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-input-card-index.ts @@ -0,0 +1,279 @@ +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import type { JournalSubmissionRow } from './journal-row-schema' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import type { JournalReducerState } from './journal-reducer' +import type { QueuedMessageRow } from './queued-message-table' +import type { JournalInputPlace } from './journal-input-order' +import { + queuePauseHolding, + journalQueueStopHolds, + type DerivedQueuePause +} from './queued-message-pause' +import { + JournalInputPlaceIndex, + mergeInputPlaces, + type InputPlaceQuery +} from './journal-input-place-index' +import { + type JournalInputLineageOwner, + type JournalInputCard, + queryInputContext +} from './journal-input-lineage' +import { JournalInputDomainIndex } from './journal-input-domain-index' + +export class JournalInputCardIndex { + private index: JournalInputPlaceIndex | undefined + private currentCoordinates: JournalInputPlaceIndex | undefined + private domains: JournalInputDomainIndex | undefined + private readonly indexed = new Map< + string, + { + card: QueuedMessageRow + actionable: boolean + handoff: boolean + source?: JournalInputCard + } + >() + // Derived current membership only; the prepared transition owns introduction values. + private readonly unintroduced = new Set() + private overlayRows: readonly QueuedMessageRow[] = [] + private overlays = new Map() + overlay(state: JournalReducerState, rows: readonly QueuedMessageRow[]): void { + if (rows === this.overlayRows || (rows.length === 0 && this.overlays.size === 0)) { + return + } + this.ensure(state) + const next = new Map(rows.map((card) => [card.messageId, card])) + for (const card of this.overlays.values()) { + if (!next.has(card.messageId) && !state.queue.cards.has(card.messageId)) { + this.indexCard(state, { ...card, state: 'withdrawn' }) + this.unintroduced.delete(card.messageId) + } + } + this.overlayRows = rows + this.overlays = next + for (const card of rows) { + if (!state.queue.cards.has(card.messageId)) { + if (!this.lineage.cards.has(card.messageId)) { + this.unintroduced.add(card.messageId) + } + this.indexCard(state, card) + } + } + } + introduceOverlays(change: (card: QueuedMessageRow) => void): void { + for (const id of this.unintroduced) { + change(this.overlays.get(id)!) + } + this.unintroduced.clear() + } + private pauseKey: readonly unknown[] = [] + private floor: { epoch: string; sequence: number } | null = null + private pauses: readonly DerivedQueuePause[] = [] + private cleared = new Set() + constructor(private readonly lineage: JournalInputLineageOwner) {} + hasOverlay(id: string): boolean { + return this.overlays.has(id) + } + command(): QueuedMessageRow | undefined { + return this.currentCoordinates?.allPlaces().command + } + releaseHistory(): void { + this.domains = undefined + } + head(): QueuedMessageRow | undefined { + return this.index?.allPlaces().head + } + paused(card: QueuedMessageRow): boolean { + return ( + card.state === 'waiting' && + card.holdReason === null && + (this.cleared.has(card.messageId) || Boolean(queuePauseHolding(this.pauses, card))) + ) + } + admission( + state: JournalReducerState, + row: JournalSubmissionRow, + originalAcceptance?: (id: string) => AgentJournalCursor | undefined + ): JournalInputPlace { + this.ensure(state) + const coordinate = { epoch: row.epoch, sequence: row.seq } + const context = row.inputOrderContext ?? { + source: coordinate, + basis: 'legacy' as const, + cut: this.lineage.at(row.seq) + } + const original = + context.basis === 'legacy' ? originalAcceptance?.(row.clientMessageId) : undefined + const place: JournalInputPlace = { coordinate, context, ...(original ? { original } : {}) } + const exposed = row.inputOrderContext?.basis === 'legacy' ? this.command() : undefined + const selected = + row.inputOrderContext?.basis === 'legacy' + ? exposed && this.relation(exposed, place) + ? exposed + : undefined + : !row.inputOrderContext + ? this.queryCurrent(place).command + : this.query(place).command + return { ...place, ...(selected ? { witnessCardId: selected.messageId } : {}) } + } + + queryCurrent(place: JournalInputPlace): InputPlaceQuery { + return queryInputContext( + place.context, + place.original, + (coordinate) => this.currentCoordinates!.query(coordinate.epoch, coordinate.sequence), + () => this.currentCoordinates!.allPlaces(), + mergeInputPlaces + ) + } + + query(place: JournalInputPlace): InputPlaceQuery { + if (place.context.basis === 'observed') { + return this.index!.query('introduction', place.context.cut) + } + if (!this.domains) { + this.domains = new JournalInputDomainIndex() + for (const { card, source, actionable, handoff } of this.indexed.values()) { + const known = this.lineage.cards.get(card.messageId) ?? source + if (known) { + this.domains.set(card, known, actionable, handoff) + } + } + } + return this.domains.query(place.context, place.original) + } + + ensure(state: JournalReducerState, floor?: { epoch: string; sequence: number } | null): void { + if (floor !== undefined) { + this.floor = floor + } + floor = this.floor + const marks = state.queuePauseMarks + const lifted = Math.max(state.latestAcceptedTurnSequence, marks.resumedSequence) + const reopened = Math.max( + marks.reopenedSequence, + floor?.epoch === state.epoch ? floor.sequence : 0 + ) + const pauseInputs = [ + journalQueueStopHolds(marks, state.latestAcceptedTurnSequence) ? marks.latestStop : null, + reopened > lifted ? reopened : 0, + marks.cleared && !marks.cleared.lifted && marks.cleared.sequence > lifted + ? marks.cleared + : null + ] + if (!this.index || pauseInputs.some((value, i) => value !== this.pauseKey[i])) { + this.pauseKey = pauseInputs + this.cleared = new Set(pauseInputs[2] ? marks.cleared?.messageIds : []) + this.index = new JournalInputPlaceIndex('introduction') + this.currentCoordinates = new JournalInputPlaceIndex(state.epoch) + this.domains = undefined + this.indexed.clear() + this.pauses = [ + ...(pauseInputs[0] && marks.latestStop + ? [ + { + reason: 'stopped' as const, + since: { epoch: state.epoch, sequence: marks.latestStop.sequence } + } + ] + : []), + ...(pauseInputs[1] + ? [ + { + reason: 'restarted' as const, + since: { epoch: state.epoch, sequence: reopened } + } + ] + : []) + ] + for (const card of this.overlays.values()) { + if (!state.queue.cards.has(card.messageId)) { + this.indexCard(state, card) + } + } + for (const card of state.queue.cards.values()) { + this.indexCard(state, card) + } + } + } + + queueChanged( + state: JournalReducerState, + prior: QueuedMessageRow | undefined, + card: QueuedMessageRow + ): void { + this.ensure(state) + const sameCoordinate = + prior?.queuedAt?.epoch === card.queuedAt?.epoch && + prior?.queuedAt?.sequence === card.queuedAt?.sequence + if ( + prior && + prior.position === card.position && + prior.state === card.state && + prior.holdReason === card.holdReason && + prior.consumedAs === card.consumedAs && + sameCoordinate && + Boolean(prior.body.command) === Boolean(card.body.command) + ) { + return + } + this.lineage.change(card.messageId, card.queuedAt, state.lastSequence) + this.indexCard(state, card) + } + + indexCard(state: JournalReducerState, card: QueuedMessageRow): void { + const attempt = card.consumedAs ? state.submissions.get(card.consumedAs) : undefined + const cleared = this.cleared.has(card.messageId) + const actionable = + !cleared && + card.state === 'waiting' && + card.holdReason === null && + !queuePauseHolding(this.pauses, card) + const handoff = attempt !== undefined && isQueuedAgentJournalSubmission(attempt) + const lineage = this.lineage.cards.get(card.messageId) ?? { + introduced: this.lineage.at(state.lastSequence), + baseline: { coordinate: card.queuedAt } + } + this.currentCoordinates!.set(card, actionable, handoff) + this.index!.set(card, actionable, handoff, { + epoch: 'introduction', + sequence: lineage.introduced + }) + if (this.domains && lineage) { + this.domains.set(card, lineage, actionable, handoff) + } + if (card.state === 'withdrawn') { + this.indexed.delete(card.messageId) + return + } + this.indexed.set(card.messageId, { + card, + actionable, + handoff, + ...(!this.lineage.cards.has(card.messageId) ? { source: lineage } : {}) + }) + } + + relation(card: QueuedMessageRow, place: JournalInputPlace): boolean { + const lineage = + this.lineage.cards.get(card.messageId) ?? this.indexed.get(card.messageId)?.source + if (!lineage || lineage.introduced >= place.context.cut) { + return false + } + if (place.context.basis === 'observed') { + return true + } + if (!lineage.baseline) { + return false + } + const coordinate = lineage.baseline.coordinate + return queryInputContext( + place.context, + place.original, + (boundary) => coordinate?.epoch === boundary.epoch && coordinate.sequence < boundary.sequence, + () => true, + (a, b) => a || b + ) + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-input-conversion.ts b/src/main/native-chat/agent-session-journal/journal-input-conversion.ts new file mode 100644 index 00000000000..89e003e2681 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-input-conversion.ts @@ -0,0 +1,108 @@ +import { readJournalLegacyCards } from './journal-queue-import' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import type { JournalReducerState } from './journal-reducer' +import { initializeJournalInputOrder, journalInputOrder } from './journal-input-order' +import type { JournalOutcomeFold } from './journal-outcome-codec' +import type { JournalInputCard, JournalInputEntry } from './journal-input-lineage' +import type Database from '../../sqlite/sync-database' +import { journalInputAcceptance } from './journal-input-provenance' + +export type JournalNativePrefix = { + entries: Map + cards: Map + base: number +} + +/** Marker-5 ordinary keys remain frozen; native evidence is lifted only after tail overrides. */ +export function detachJournalNativePrefix(state?: JournalReducerState): JournalNativePrefix { + const prefix = { + entries: new Map(state?.inputEntries), + cards: new Map(state?.inputCards), + base: state ? journalInputOrder(state).lineage.at(0) : 0 + } + state?.inputEntries.clear() + state?.inputCards.clear() + return prefix +} + +export function restoreJournalNativeState( + db: Database.Database, + state: JournalReducerState, + format: number, + fold: JournalOutcomeFold, + prefix: JournalNativePrefix +): void { + if (format < 2) { + for (const [id, own] of prefix.entries) { + if ( + state.submissions.get(id)?.dispatchState === 'pending' && + !fold.replaced('submissions', id) + ) { + state.inputEntries.set(id, own) + } + } + for (const [id, card] of prefix.cards) { + if (!state.queue.cards.has(id) && !fold.replaced('queue.cards', id)) { + state.inputCards.set(id, card) + } + } + for (const [id, card] of state.queue.cards) { + const known = !fold.replaced('queue.cards', id) ? prefix.cards.get(id) : undefined + const introducedAt = fold.introduction('queue.cards', id) + if (known) { + state.inputCards.set(id, known) + } else if (introducedAt !== undefined) { + state.inputCards.set(id, { + introduced: prefix.base + introducedAt, + baseline: { coordinate: card.queuedAt } + }) + } + } + for (const [id, entry] of state.submissions) { + const sequence = entry.acceptedSequence ?? entry.submittedSequence + if ( + !state.inputEntries.has(id) && + isQueuedAgentJournalSubmission(entry) && + !entry.queuedMessageId && + sequence !== undefined + ) { + state.inputEntries.set(id, { + context: { + source: { epoch: state.epoch, sequence }, + basis: 'legacy', + cut: prefix.base + sequence + } + }) + } + } + } + const owner = initializeJournalInputOrder(state, (id) => + journalInputAcceptance(db, state.sessionId, id) + ) + owner.beginReconstruction() + owner.restore(state) + owner.overlay(state, readJournalLegacyCards(db, state)) + owner.finishReconstruction(state) +} + +/** Replacement carries facts directly; restatement is not a new admission or introduction. */ +export function copyJournalNativeState( + previous: JournalReducerState, + state: JournalReducerState +): void { + for (const [id, own] of previous.inputEntries) { + state.inputEntries.set(id, { + ...own, + context: { ...own.context, source: { ...own.context.source } } + }) + } + for (const [id, card] of previous.inputCards) { + state.inputCards.set(id, { + ...card, + ...(card.baseline + ? { baseline: { coordinate: card.baseline.coordinate && { ...card.baseline.coordinate } } } + : {}) + }) + } + journalInputOrder(state).lineage.restoreFrontier(state.lastSequence, state.inputEntries.values()) +} diff --git a/src/main/native-chat/agent-session-journal/journal-input-domain-index.ts b/src/main/native-chat/agent-session-journal/journal-input-domain-index.ts new file mode 100644 index 00000000000..7b9579f1a5d --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-input-domain-index.ts @@ -0,0 +1,85 @@ +import { + queryInputContext, + type JournalInputCard, + type JournalInputContext +} from './journal-input-lineage' +import { + JournalInputPlaceIndex, + mergeInputPlaces, + type InputPlaceQuery +} from './journal-input-place-index' +import type { QueuedMessageRow } from './queued-message-table' +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' + +type Node = { index: JournalInputPlaceIndex; left?: Node; right?: Node } +const LIMIT = Number.MAX_SAFE_INTEGER + +/** A fixed-width introduction prefix, with coordinate summaries on each of its 53 levels. */ +export class JournalInputDomainIndex { + private readonly root: Node = { index: new JournalInputPlaceIndex('') } + + set( + card: QueuedMessageRow, + source: JournalInputCard, + actionable: boolean, + handoff: boolean + ): void { + if (!source.baseline) { + return + } + let node = this.root, + low = 0, + high = LIMIT + while (true) { + node.index.set(card, actionable, handoff, source.baseline.coordinate) + if (high - low <= 1) { + break + } + const mid = low + Math.floor((high - low) / 2) + if (source.introduced < mid) { + node = node.left ??= { index: new JournalInputPlaceIndex('') } + high = mid + } else { + node = node.right ??= { index: new JournalInputPlaceIndex('') } + low = mid + } + } + } + + query(context: JournalInputContext, original?: AgentJournalCursor): InputPlaceQuery { + let result: InputPlaceQuery = { position: -Infinity } + const append = (node: Node) => { + result = mergeInputPlaces( + result, + queryInputContext( + context, + original, + (coordinate) => node.index.query(coordinate.epoch, coordinate.sequence), + () => node.index.allPlaces(), + mergeInputPlaces + ) + ) + } + let node: Node | undefined = this.root, + low = 0, + high = LIMIT + while (node && low < context.cut) { + if (high <= context.cut) { + append(node) + break + } + const mid = low + Math.floor((high - low) / 2) + if (context.cut <= mid) { + node = node.left + high = mid + } else { + if (node.left) { + append(node.left) + } + node = node.right + low = mid + } + } + return result + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-input-index-probe.test-support.ts b/src/main/native-chat/agent-session-journal/journal-input-index-probe.test-support.ts new file mode 100644 index 00000000000..67e7642873d --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-input-index-probe.test-support.ts @@ -0,0 +1,74 @@ +import { vi } from 'vitest' +import { JournalInputLineageOwner } from './journal-input-lineage' +/** Instrument real index nodes without charging the test's discovery walk as production work. */ +export function journalInputIndexProbe( + keys: readonly string[], + count: { allocated: () => void; read: () => void } +): ((node: unknown) => void) & { restore: () => void } { + const seen = new WeakMap>() + const originals: { node: object; key: string; descriptor?: PropertyDescriptor }[] = [] + const inspect = (node: unknown): void => { + if (typeof node !== 'object' || node === null) { + return + } + const prior = seen.get(node) + if (prior) { + inspect(prior.get('left')) + inspect(prior.get('right')) + return + } + const values = new Map() + seen.set(node, values) + count.allocated() + for (const key of keys) { + const descriptor = Object.getOwnPropertyDescriptor(node, key) + originals.push({ node, key, descriptor }) + values.set(key, descriptor?.value) + if (key === 'left' || key === 'right') { + inspect(values.get(key)) + } + Object.defineProperty(node, key, { + configurable: true, + get: () => { + count.read() + return values.get(key) + }, + set: (next) => { + values.set(key, next) + } + }) + } + } + return Object.assign(inspect, { + restore: () => { + for (const { node, key, descriptor } of originals) { + const value = seen.get(node)?.get(key) + if (descriptor || value !== undefined) { + Object.defineProperty(node, key, { + ...descriptor, + value, + writable: true, + configurable: true + }) + } else { + Reflect.deleteProperty(node, key) + } + } + originals.length = 0 + } + }) +} + +export function journalInputHistoryProbe() { + let visits = 0 + const release = JournalInputLineageOwner.prototype.releaseHistory + const spy = vi + .spyOn(JournalInputLineageOwner.prototype, 'releaseHistory') + .mockImplementation(function (this: JournalInputLineageOwner, keep) { + release.call(this, (id) => { + visits++ + return keep(id) + }) + }) + return { spy, visits: () => visits } +} diff --git a/src/main/native-chat/agent-session-journal/journal-input-legacy-baseline.test.ts b/src/main/native-chat/agent-session-journal/journal-input-legacy-baseline.test.ts new file mode 100644 index 00000000000..431e8c8247b --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-input-legacy-baseline.test.ts @@ -0,0 +1,296 @@ +import { journalInputOrder } from './journal-input-order' +import { + journalInputIndexProbe, + journalInputHistoryProbe +} from './journal-input-index-probe.test-support' +import { JournalInputDomainIndex } from './journal-input-domain-index' +import { JournalInputPlaceIndex } from './journal-input-place-index' +import { insertCommandReceiptIfAbsent } from './command-receipt-table' +import { writeCommandReceiptTestRecord } from './command-receipt-test-support' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, it, vi } from 'vitest' +import { + claudeProviderHandle, + agentSessionJournalProviderHandle +} from '../../../shared/agent-session-provider-handle-encoding' +import type { JournalRow } from './journal-row-schema' +import type { JournalReducerState } from './journal-reducer' +import type { AgentJournalMessageItem } from '../../../shared/agent-session-journal-types' +import { holdUnsentSends } from './journal-unsent-send-hold' +import { + createTrackedJournalOpener, + loadTestJournal, + openTestJournalHostDatabase, + readTestJournalOutcomes, + insertTestJournalRow, + writeTestLegacyTerminalGaps +} from './journal-host-database-test-support' +const message = (text: string): AgentJournalMessageItem => ({ + kind: 'message', + role: 'user', + blocks: [{ type: 'text', text }] +}) + +it.each([10, 100])( + 'indexes the initial legacy writer shape without promoting foreign/null comparison N=%i', + async (n) => { + const root = await mkdtemp(join(tmpdir(), 'orca-legacy-baseline-')) + const journals = createTrackedJournalOpener({ legacyInputs: true }) + const identity = { + sessionId: 'baseline', + workspaceId: 'folder', + hostId: 'ssh', + agent: 'claude' as const, + providerHandle: claudeProviderHandle('thread', null) + } + let journal = await journals.open({ identity, stateDirectory: root }) + const database = openTestJournalHostDatabase(root) + const cost = { sets: 0, queries: 0, nodes: 0, reads: 0 } + const inspect = journalInputIndexProbe(['index', 'left', 'right'], { + allocated: () => { + cost.nodes++ + }, + read: () => { + cost.reads++ + } + }) + let inspecting = true + const set = JournalInputDomainIndex.prototype.set + const domainSets = vi + .spyOn(JournalInputDomainIndex.prototype, 'set') + .mockImplementation(function (this: JournalInputDomainIndex, ...args) { + cost.sets++ + if (inspecting) { + inspect(this['root']) + } + set.apply(this, args) + if (inspecting) { + inspect(this['root']) + } + }) + const query = JournalInputDomainIndex.prototype.query + const domainQueries = vi + .spyOn(JournalInputDomainIndex.prototype, 'query') + .mockImplementation(function (this: JournalInputDomainIndex, ...args) { + cost.queries++ + return query.apply(this, args) + }) + const indexSets = vi.spyOn(JournalInputPlaceIndex.prototype, 'set') + const release = journalInputHistoryProbe() + try { + const { epoch, sequence } = journal.cursor() + let seq = sequence + await journal.close() + const base = () => ({ v: 3, epoch, seq: ++seq, fence: 0, ts: 1 }) + const queue = ( + id: string, + queuedAt: { epoch: string; sequence: number } | null, + command = false + ) => + insertTestJournalRow(database.db, identity.sessionId, { + ...base(), + kind: 'queue', + messageId: id, + change: { + op: 'queued', + body: command ? { ...message('/compact'), command: { name: 'compact' } } : message(id), + position: id === 'C' ? 1 : id === 'D' ? 2 : 3, + queuedAt + } + }) + queue('C', { epoch, sequence: seq }, true) + queue('F', { epoch: 'original-foreign', sequence: 1 }) + queue('N', null) + database.transaction((db) => writeCommandReceiptTestRecord(db, identity.sessionId)) + const ids = Array.from({ length: n }, (_, i) => `M${i + 1}`) + for (let i = 0; i < n; i++) { + queue(`ordinary-${i}`, { epoch, sequence: seq }) + } + queue('D', { epoch: 'original-foreign', sequence: 1 }) + seq = writeTestLegacyTerminalGaps( + database.db, + identity.sessionId, + { + epoch, + sequence: seq, + fence: 0, + providerHandle: agentSessionJournalProviderHandle(identity) + }, + n + ) + for (const id of ids) { + const target: JournalRow = { + ...base(), + kind: 'submission', + clientMessageId: id, + payloadFingerprint: id, + body: message(id), + providerHandle: agentSessionJournalProviderHandle(identity), + handoverRecorded: true, + origin: 'client' + } + insertTestJournalRow(database.db, identity.sessionId, target) + if (id !== 'M2') { + database.transaction((db) => + insertCommandReceiptIfAbsent( + db, + { kind: 'global' }, + { + operationId: id, + sessionId: identity.sessionId, + callerKey: 'fixture', + method: 'agentSession.send', + fingerprint: id, + status: 'accepted', + acceptedAt: 1, + result: { kind: 'journal-row', epoch: 'original-foreign', sequence: 100 } + } + ) + ) + } + } + queue('later-F', { epoch: 'original-foreign', sequence: 1 }) + journal = await journals.open({ identity, stateDirectory: root }) + expect(release.spy).not.toHaveBeenCalled() + const check = (state: JournalReducerState) => { + for (const card of ['F', 'D']) { + expect( + ['M1', 'M2', 'M3'].map((id) => + journalInputOrder(state).precedes( + state.queue.cards.get(card)!, + state.submissions.get(id)! + ) + ) + ).toEqual([true, false, true]) + } + for (const card of ['N', 'later-F']) { + expect( + ['M1', 'M2', 'M3'].map((id) => + journalInputOrder(state).precedes( + state.queue.cards.get(card)!, + state.submissions.get(id)! + ) + ) + ).toEqual([false, false, false]) + } + expect(['M1', 'M2', 'M3'].every((id) => journalInputOrder(state).forced(id))).toBe(true) + } + check(journal.queuedMessages.displayState()) + for (const id of ids) { + expect( + journal.inputOrder.predecessors(journal.queuedMessages.displayState(), id).position + ).toBe(3) + } + // Accessor probes count the old raw fold only; remove them before keyed transaction drafts. + const rawCost = { ...cost } + expect(rawCost.nodes).toBeGreaterThan(0) + expect(rawCost.nodes).toBeLessThanOrEqual(54 * rawCost.sets) + expect(rawCost.reads).toBeLessThanOrEqual(4 * 54 * (rawCost.sets + rawCost.queries)) + inspect.restore() + inspecting = false + for (let pass = 0; pass < 3; pass++) { + domainSets.mockClear() + domainQueries.mockClear() + indexSets.mockClear() + if (pass) { + await journal.replaceEpochItems('unreconcilable_prefix', 0, []) + } + check(journal.queuedMessages.displayState()) + const replay = loadTestJournal(root, identity.sessionId)! + expect(replay.damage).toBeNull() + check(replay.state) + if (pass) { + const outcomes = readTestJournalOutcomes(database.db, identity.sessionId) + const rows = [...new Set(outcomes.map((value) => value.seq))] + expect(rows.length).toBeLessThanOrEqual(2 * n + 12) + const baseline = [...journal.queuedMessages.displayState().inputCards.values()].filter( + (card) => card.baseline !== undefined + ).length + expect(baseline).toBeLessThanOrEqual(n + 6) + for (const id of ids) { + journal.inputOrder.predecessors(journal.queuedMessages.displayState(), id) + journalInputOrder(replay.state).predecessors(replay.state, id) + } + expect(domainQueries).toHaveBeenCalledTimes(2 * n) + // Each independent fold adds at most one command-witness transition beyond the frozen baseline; each update traverses at most 54 introduction nodes. + expect(domainSets.mock.calls.length).toBeLessThanOrEqual(4 * (n + 6) + 2 * rows.length) + expect(indexSets.mock.calls.length).toBeLessThanOrEqual( + 54 * domainSets.mock.calls.length + 8 * rows.length + ) + console.info( + 'legacy-normalized-output', + JSON.stringify({ + n, + pass, + rows: rows.length, + baseline, + domainSets: domainSets.mock.calls.length, + domainQueries: domainQueries.mock.calls.length, + indexSets: indexSets.mock.calls.length + }) + ) + } + } + domainSets.mockClear() + domainQueries.mockClear() + indexSets.mockClear() + await holdUnsentSends(journal, { fence: 0, hold: { cause: 'hostRestarted' } }) + expect(domainQueries).toHaveBeenCalledTimes(n) + // One finite-rank move per current card and one keep per input, each updating its one frozen coordinate through at most 54 introduction nodes. + expect(domainSets.mock.calls.length).toBeLessThanOrEqual(16 * (n + 6)) + expect(indexSets.mock.calls.length).toBeLessThanOrEqual( + 54 * domainSets.mock.calls.length + 16 * (n + 6) + ) + console.info( + 'legacy-recovery-index', + JSON.stringify({ + n, + domainSets: domainSets.mock.calls.length, + domainQueries: domainQueries.mock.calls.length, + indexSets: indexSets.mock.calls.length + }) + ) + for (const id of ids) { + expect(journal.submission(id)).toMatchObject({ + dispatchState: 'rejected', + keptAsQueuedMessageId: id + }) + } + const ranks = journal.queuedMessages + .list() + .filter((card) => card.state === 'waiting') + .map((card) => card.position) + expect( + ranks.every((rank, i) => Number.isFinite(rank) && (i === 0 || rank > ranks[i - 1])) + ).toBe(true) + console.info('frozen-baseline-raw-nodes', JSON.stringify({ n, ...rawCost })) + expect(release.spy).toHaveBeenCalledOnce() + expect(release.visits()).toBe(2 * n + 5) + const finished = loadTestJournal(root, identity.sessionId)! + expect(finished.damage).toBeNull() + expect(release.spy).toHaveBeenCalledTimes(2) + expect(release.visits()).toBe(2 * (2 * n + 5)) + expect( + [...journalInputOrder(finished.state).lineage.cards.values()].every( + (card) => !card.baseline + ) + ).toBe(true) + expect(journal.inputOrder['cards']['domains']).toBeUndefined() + expect(journal.queuedMessages.displayState().inputEntries.size).toBe(0) + expect(finished.state.inputEntries.size).toBe(0) + expect( + [...journal.inputOrder.lineage.cards.values()].every((card) => card.baseline === undefined) + ).toBe(true) + } finally { + inspect.restore() + domainSets.mockRestore() + domainQueries.mockRestore() + indexSets.mockRestore() + release.spy.mockRestore() + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } + } +) diff --git a/src/main/native-chat/agent-session-journal/journal-input-lineage.ts b/src/main/native-chat/agent-session-journal/journal-input-lineage.ts new file mode 100644 index 00000000000..3fffb798f85 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-input-lineage.ts @@ -0,0 +1,127 @@ +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' + +export type JournalInputContext = { + source: AgentJournalCursor + basis: 'observed' | 'legacy' + cut: number +} +export type JournalInputCard = { + introduced: number + baseline?: { coordinate: AgentJournalCursor | null } +} +export type JournalCardLineage = JournalInputCard & { messageId: string } +export type JournalInputEntry = { context: JournalInputContext; witnessCardId?: string } +export type JournalInputLineage = { clock: number; cards: JournalCardLineage[] } + +const object = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value) +const ordinal = (value: unknown): value is number => + Number.isSafeInteger(value) && Number(value) >= 0 +export function isInputCoordinate(value: unknown): value is AgentJournalCursor { + return ( + object(value) && + typeof value.epoch === 'string' && + value.epoch.length > 0 && + ordinal(value.sequence) + ) +} +export function isInputContext(value: unknown): value is JournalInputContext { + return ( + object(value) && + isInputCoordinate(value.source) && + ordinal(value.cut) && + (value.basis === 'observed' || value.basis === 'legacy') + ) +} +export function isInputLineage(value: unknown): value is JournalInputLineage { + if (!object(value) || !ordinal(value.clock) || !Array.isArray(value.cards)) { + return false + } + const clock = value.clock + const ids = new Set() + return value.cards.every((card) => { + if ( + !object(card) || + typeof card.messageId !== 'string' || + !card.messageId || + ids.has(card.messageId) || + !ordinal(card.introduced) || + card.introduced > clock + ) { + return false + } + ids.add(card.messageId) + return ( + card.baseline === undefined || + (object(card.baseline) && + (card.baseline.coordinate === null || isInputCoordinate(card.baseline.coordinate))) + ) + }) +} + +export function queryInputContext( + context: JournalInputContext, + original: AgentJournalCursor | undefined, + before: (coordinate: AgentJournalCursor) => T, + all: () => T, + merge: (left: T, right: T) => T +): T { + if (context.basis === 'observed' || original?.epoch === context.source.epoch) { + return all() + } + const source = before(context.source) + return original ? merge(source, before(original)) : source +} + +/** Initial source coordinates and introduction cuts never follow physical restatement rows. */ +export class JournalInputLineageOwner { + constructor(readonly cards: Map) {} + private base = 0 + clock = 0 + + load(lineage: JournalInputLineage): void { + this.base = lineage.clock + this.clock = lineage.clock + for (const card of lineage.cards) { + this.cards.set(card.messageId, { + introduced: card.introduced, + ...(card.baseline ? { baseline: card.baseline } : {}) + }) + } + } + restoreFrontier(sequence: number, entries: Iterable): void { + let frontier = 0 + for (const entry of entries) { + frontier = Math.max(frontier, entry.context.cut) + } + for (const card of this.cards.values()) { + frontier = Math.max(frontier, card.introduced) + } + this.clock = frontier + this.base = frontier - sequence + } + at(sequence: number): number { + return this.base + sequence + } + observe(sequence: number): void { + this.clock = Math.max(this.clock, this.at(sequence)) + } + + change(messageId: string, coordinate: AgentJournalCursor | null, sequence: number): void { + if (!this.cards.has(messageId)) { + this.cards.set(messageId, { + introduced: this.at(sequence), + baseline: { coordinate } + }) + } + } + releaseHistory(keep: (id: string) => boolean): void { + for (const [id, card] of this.cards) { + if (!keep(id)) { + this.cards.delete(id) + } else { + delete card.baseline + } + } + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-input-order-state.ts b/src/main/native-chat/agent-session-journal/journal-input-order-state.ts new file mode 100644 index 00000000000..63f2cea2980 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-input-order-state.ts @@ -0,0 +1,84 @@ +import type { + AgentJournalCursor, + AgentJournalSubmission +} from '../../../shared/agent-session-journal-types' +import type { JournalReducerState } from './journal-reducer' +import type { JournalInputEntry } from './journal-input-lineage' +import type { JournalRow } from './journal-row-schema' + +export type JournalInputPlace = JournalInputEntry & { + coordinate: AgentJournalCursor + original?: AgentJournalCursor +} + +export function readJournalInputPlace( + state: JournalReducerState, + id: string, + originals: Map, + originalAcceptance?: (id: string) => AgentJournalCursor | undefined +): JournalInputPlace | undefined { + const own = state.inputEntries.get(id) + const entry = state.submissions.get(id) + if (!own || !entry) { + return undefined + } + if (own.context.basis === 'legacy' && !originals.has(id)) { + originals.set(id, originalAcceptance?.(id)) + } + const original = originals.get(id) + return { + ...own, + coordinate: { + epoch: state.epoch, + sequence: entry.acceptedSequence ?? entry.submittedSequence ?? 0 + }, + ...(original ? { original } : {}) + } +} +export function currentJournalInputIdentity( + state: JournalReducerState, + id: string, + hasOverlay: () => boolean +): boolean { + const card = state.queue.cards.get(id) + return card + ? card.state === 'waiting' || + card.state === 'returned' || + (card.state === 'dispatched' && + card.consumedAs !== null && + state.submissions.get(card.consumedAs)?.dispatchState === 'pending') + : hasOverlay() +} + +export function restoreJournalInputPlaces(state: JournalReducerState): number { + let legacyPlaces = 0 + for (const [id, own] of state.inputEntries) { + if (state.submissions.get(id)?.dispatchState !== 'pending') { + state.inputEntries.delete(id) + } else if (own.context.basis === 'legacy') { + legacyPlaces++ + } + } + return legacyPlaces +} + +export function journalInputOrderChanged( + state: JournalReducerState, + row: JournalRow, + readCount: (key: 'receiptCount' | 'queueRevision') => number +): boolean { + return ( + row.kind === 'submission' || + row.kind === 'dispatch' || + row.kind === 'tombstone' || + readCount('receiptCount') !== state.receipts.size || + readCount('queueRevision') !== state.queue.revision + ) +} + +export function isJournalInputOverride( + entry: AgentJournalSubmission, + recoveryPending: boolean +): boolean { + return !recoveryPending && entry.origin === 'client' && entry.queuedMessageId !== undefined +} diff --git a/src/main/native-chat/agent-session-journal/journal-input-order.test.ts b/src/main/native-chat/agent-session-journal/journal-input-order.test.ts new file mode 100644 index 00000000000..36821e08c6b --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-input-order.test.ts @@ -0,0 +1,566 @@ +import { journalInputOrder } from './journal-input-order' +import { insertCommandReceiptIfAbsent, readCommandReceipt } from './command-receipt-table' +import { writeCommandReceiptTestRecord } from './command-receipt-test-support' +import type { JournalRow } from './journal-row-schema' +import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' +import type { AgentSessionJournal } from './journal-store' +import type { JournalReducerState } from './journal-reducer' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, it, vi } from 'vitest' +import { + agentSessionJournalProviderHandle, + claudeProviderHandle +} from '../../../shared/agent-session-provider-handle-encoding' +import type { AgentJournalMessageItem } from '../../../shared/agent-session-journal-types' +import { holdUnsentSends } from './journal-unsent-send-hold' +import { + createTrackedJournalOpener, + loadTestJournal, + openTestJournalHostDatabase, + readTestJournalOutcomes, + insertTestJournalRow +} from './journal-host-database-test-support' +import { + selectedStructuredSubmission, + selectStructuredAgentSessionInput +} from '../agent-session-wire/structured-agent-session-input-selection' + +const message = (text: string): AgentJournalMessageItem => ({ + kind: 'message', + role: 'user', + blocks: [{ type: 'text', text }] +}) + +function knownPredecessors(owner: AgentSessionJournal | JournalReducerState, id: string): string[] { + const state = 'queuedMessages' in owner ? owner.queuedMessages.displayState() : owner + const submission = state.submissions.get(id) + return submission + ? [...state.queue.cards.values()] + .filter((card) => journalInputOrder(state).precedes(card, submission)) + .map((card) => card.messageId) + : [] +} + +it('preserves the forced queue place through command removal, two epoch replacements, replay and recovery', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-input-order-')) + const journals = createTrackedJournalOpener() + const identity = { + sessionId: 'order', + workspaceId: 'folder', + hostId: 'ssh', + agent: 'claude' as const, + providerHandle: claudeProviderHandle('thread', null) + } + const journal = await journals.open({ identity, stateDirectory: root }) + try { + const c = await journal.queuedMessages.insert({ + messageId: 'C', + body: { ...message('/compact'), command: { name: 'compact' } } + }) + await journal.queuedMessages.insert({ messageId: 'D', body: message('earlier card') }) + await journal.appendSubmission({ + clientMessageId: 'M', + body: message('stale direct'), + payloadFingerprint: 'M', + fence: 0, + handoverRecorded: true, + origin: 'client' + }) + await journal.queuedMessages.insert({ messageId: 'later', body: message('later card') }) + await journal.queuedMessages.withdraw([c.messageId]) + expect(selectStructuredAgentSessionInput(journal)).toMatchObject({ + kind: 'card', + card: { messageId: 'D' } + }) + for (let round = 0; round < 2; round++) { + await journal.replaceEpochItems('unreconcilable_prefix', 0, []) + expect(knownPredecessors(journal, 'M')).toEqual(['D']) + expect(journal.inputOrder.forced('M')).toBe(true) + expect(journal.queuedMessages.get('C')).toBeNull() + expect(selectStructuredAgentSessionInput(journal)).toMatchObject({ + kind: 'card', + card: { messageId: 'D' } + }) + } + await journal.close() + const reopened = await journals.open({ identity, stateDirectory: root }) + expect(knownPredecessors(reopened, 'M')).toEqual(['D']) + expect(reopened.inputOrder.forced('M')).toBe(true) + await holdUnsentSends(reopened, { fence: 0, hold: { cause: 'hostRestarted' } }) + expect( + reopened.queuedMessages + .list() + .filter((card) => card.state === 'waiting') + .map((card) => card.messageId) + ).toEqual(['D', 'M', 'later']) + expect(reopened.submission('M')).toMatchObject({ + dispatchState: 'rejected', + keptAsQueuedMessageId: 'M' + }) + expect(selectedStructuredSubmission(reopened)).toBeUndefined() + } finally { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } +}) + +it.each(['resume', 'accepted'] as const)( + 'preserves failed reopen admission facts across replay/replacement and %s release', + async (release) => { + const root = await mkdtemp(join(tmpdir(), 'orca-input-order-reopen-')) + const journals = createTrackedJournalOpener() + const identity = { + sessionId: 'order', + workspaceId: 'folder', + hostId: 'ssh', + agent: 'claude' as const, + providerHandle: claudeProviderHandle('thread', null) + } + let journal = await journals.open({ identity, stateDirectory: root }) + const append = ( + id: string, + receipt?: Parameters[2], + fence = 1 + ) => + journal.appendSubmission( + { + clientMessageId: id, + body: message(id), + payloadFingerprint: id, + fence, + handoverRecorded: true, + origin: 'client', + options: { effort: 'high' } + }, + undefined, + receipt + ) + const accept = (id: string) => + journal.resolveDispatch({ + clientMessageId: id, + state: 'accepted', + providerIdentity: null, + fence: 1 + }) + try { + await journal.queuedMessages.insert({ + messageId: 'C', + body: { ...message('/compact'), command: { name: 'compact' } } + }) + await append('forced') + const mark = vi + .spyOn(journal, 'appendQueueReopen') + .mockRejectedValueOnce(new Error('disk full')) + await expect(journal.markQueueReopen(1)).rejects.toThrow('disk full') + mark.mockRestore() + const floor = journal.reopenFloor()! + const before = journal.cursor() + let committed = 0 + await expect( + append('M', { + write: () => { + throw new Error('receipt transaction fails') + }, + committed: () => { + committed++ + } + }) + ).rejects.toThrow('receipt transaction fails') + expect(journal.cursor()).toEqual(before) + expect(journal.submission('M')).toBeUndefined() + expect(journal.reopenFloor()).toEqual(floor) + expect( + loadTestJournal(root, identity.sessionId)!.state.queuePauseMarks.reopenedSequence + ).toBe(0) + expect(committed).toBe(0) + await expect(append('stale', undefined, 0)).rejects.toMatchObject({ + code: 'journal_stale_fence' + }) + expect(journal.cursor()).toEqual(before) + expect(journal.reopenFloor()).toEqual(floor) + if (release === 'resume') { + await journal.appendQueueResume(1) + } + let target: { epoch: string; sequence: number } | undefined + const cursor = await append('M', { + write: (_db, row) => { + if (row) { + target = { epoch: row.epoch, sequence: row.seq } + } + expect(row).toMatchObject({ + kind: 'submission', + clientMessageId: 'M', + payloadFingerprint: 'M', + options: { effort: 'high' }, + body: message('M') + }) + }, + committed: () => { + committed++ + } + }) + expect(target).toEqual(cursor) + expect(committed).toBe(1) + expect(journal.reopenFloor()).toBeNull() + const marks = readTestJournalOutcomes( + openTestJournalHostDatabase(root).db, + identity.sessionId + ).filter((record) => record.field === '$' && record.key === '"queuePauseMarks"') + expect( + marks.some( + (record) => + record.seq < cursor.sequence && + JSON.parse(record.json).reopenedSequence === floor.sequence + ) + ).toBe(true) + expect(knownPredecessors(journal, 'M')).toEqual(release === 'resume' ? ['C'] : []) + if (release === 'accepted') { + expect(journal.queuedMessages.pauses().map((p) => p.reason)).toEqual(['restarted']) + await append('N') + expect(knownPredecessors(journal, 'N')).toEqual([]) + await accept('N') + expect(journal.queuedMessages.pauses()).toEqual([]) + await append('ordinary-held') + // The accepted N released C; this later input genuinely belongs behind it. + expect(knownPredecessors(journal, 'ordinary-held')).toEqual(['C']) + } else { + expect(journal.queuedMessages.pauses()).toEqual([]) + } + const beforePrefixes = ['forced', 'M', 'ordinary-held'].map((id) => + knownPredecessors(journal, id) + ) + const replay = loadTestJournal(root, identity.sessionId)! + expect(replay.damage).toBeNull() + expect(replay.newer).toBeNull() + expect( + ['forced', 'M', 'ordinary-held'].map((id) => knownPredecessors(replay.state, id)) + ).toEqual(beforePrefixes) + const lateMark = vi + .spyOn(journal, 'appendQueueReopen') + .mockRejectedValueOnce(new Error('later reopen write fails')) + await expect(journal.markQueueReopen(1)).rejects.toThrow('later reopen write fails') + lateMark.mockRestore() + const foreignFloor = journal.reopenFloor()! + await journal.replaceEpochItems('unreconcilable_prefix', 1, []) + expect(journal.reopenFloor()).toEqual(foreignFloor) + expect(foreignFloor.epoch).not.toBe(journal.epoch) + const beforeForeign = journal.cursor() + await append('new-epoch') + expect(journal.cursor().sequence).toBe(beforeForeign.sequence + 1) + expect(knownPredecessors(journal, 'new-epoch')).toEqual([]) + expect(['forced', 'M', 'ordinary-held'].map((id) => knownPredecessors(journal, id))).toEqual( + beforePrefixes + ) + await journal.close() + journal = await journals.open({ identity, stateDirectory: root }) + expect(['forced', 'M', 'ordinary-held'].map((id) => knownPredecessors(journal, id))).toEqual( + beforePrefixes + ) + await accept('forced') + expect(knownPredecessors(journal, 'forced')).toEqual([]) + } finally { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } + } +) + +it('keeps the receipt callback live after replacement of a legacy null-boundary card', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-input-replacement-')) + const journals = createTrackedJournalOpener({ legacyInputs: true }) + const identity = { + sessionId: 'null-order', + workspaceId: 'folder', + hostId: 'ssh', + agent: 'claude' as const, + providerHandle: claudeProviderHandle('thread', null) + } + const body: AgentJournalMessageItem = { + kind: 'message', + role: 'user', + blocks: [{ type: 'text', text: '/compact' }], + command: { name: 'compact' } + } + let journal = await journals.open({ identity, stateDirectory: root }) + try { + const cursor = journal.cursor() + await journal.close() + insertTestJournalRow(openTestJournalHostDatabase(root).db, identity.sessionId, { + v: 3, + epoch: cursor.epoch, + seq: cursor.sequence + 1, + fence: 0, + ts: 1, + kind: 'queue', + messageId: 'legacy-C', + change: { op: 'queued', body, position: 1, queuedAt: null } + }) + journal = await journals.open({ identity, stateDirectory: root }) + expect(journal.queuedMessages.get('legacy-C')?.queuedAt).toBeNull() + await journal.replaceEpochItems('unreconcilable_prefix', 0, []) + expect(journal.queuedMessages.get('legacy-C')?.queuedAt).toBeNull() + await expect( + journal.appendSubmission({ + clientMessageId: 'M-after-replace', + body: { ...body, command: undefined }, + payloadFingerprint: 'M-after-replace', + fence: 0, + handoverRecorded: true, + origin: 'client' + }) + ).resolves.toMatchObject({ epoch: journal.epoch }) + expect(journal.submission('M-after-replace')).toMatchObject({ dispatchState: 'pending' }) + const replay = loadTestJournal(root, identity.sessionId)! + expect(replay.damage).toBeNull() + expect(replay.state.queue.cards.get('legacy-C')?.queuedAt).toBeNull() + expect(knownPredecessors(replay.state, 'M-after-replace')).toEqual( + knownPredecessors(journal, 'M-after-replace') + ) + } finally { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } +}) + +it.each(['legacy', 'observed'] as const)( + 'retains %s import evidence through two production replacements and replay', + async (basis) => { + const root = await mkdtemp(join(tmpdir(), 'orca-source-domain-')) + const journals = createTrackedJournalOpener({ legacyInputs: true }) + const identity = { + sessionId: 'domain', + workspaceId: 'folder', + hostId: 'ssh', + agent: 'claude' as const, + providerHandle: claudeProviderHandle('thread', null) + } + let journal = await journals.open({ identity, stateDirectory: root }) + const database = openTestJournalHostDatabase(root) + try { + const a = journal.cursor() + await journal.close() + let seq = a.sequence + const row = ( + fields: Pick, 'kind' | 'messageId' | 'change'> + ): JournalRow => ({ v: 3, epoch: a.epoch, seq: ++seq, fence: 0, ts: 1, ...fields }) + for (const [id, queuedAt] of [ + ['D', { epoch: 'foreign', sequence: 1 }], + ['N', null], + ['C', { epoch: a.epoch, sequence: seq }] + ] as const) { + insertTestJournalRow( + database.db, + identity.sessionId, + row({ + kind: 'queue', + messageId: id, + change: { + op: 'queued', + body: + id === 'C' ? { ...message('/compact'), command: { name: 'compact' } } : message(id), + position: seq, + queuedAt + } + }) + ) + } + database.transaction((db) => writeCommandReceiptTestRecord(db, identity.sessionId)) + journal = await journals.open({ identity, stateDirectory: root }) + const originals = new Map() + for (const id of ['M1', 'M2', 'M3']) { + const receipt = + id === 'M2' + ? undefined + : { + write: ( + db: Parameters[0], + target?: JournalRow + ) => { + expect(target).toMatchObject({ kind: 'submission', clientMessageId: id }) + const result = { + kind: 'journal-row' as const, + epoch: target!.epoch, + sequence: target!.seq + } + insertCommandReceiptIfAbsent( + db, + { kind: 'global' }, + { + operationId: id, + sessionId: identity.sessionId, + callerKey: 'fixture', + method: 'agentSession.send', + fingerprint: id, + status: 'accepted', + acceptedAt: 1, + result + } + ) + originals.set(id, JSON.stringify(result)) + }, + committed: () => {} + } + if (basis === 'observed') { + await journal.appendSubmission( + { + clientMessageId: id, + payloadFingerprint: id, + body: message(id), + handoverRecorded: true, + origin: 'client', + options: { effort: 'high' }, + fence: 0 + }, + undefined, + receipt + ) + } else { + await journal.close() + const target: JournalRow = { + kind: 'submission', + v: 3, + epoch: a.epoch, + seq: journal.cursor().sequence + 1, + ts: 1, + fence: 0, + clientMessageId: id, + payloadFingerprint: id, + body: message(id), + providerHandle: agentSessionJournalProviderHandle(identity), + handoverRecorded: true, + origin: 'client', + options: { effort: 'high' } + } + insertTestJournalRow(database.db, identity.sessionId, target) + if (receipt) { + database.transaction((db) => receipt.write(db, target)) + } + journal = await journals.open({ identity, stateDirectory: root }) + } + } + const expected = basis === 'legacy' ? [true, false, true] : [true, true, true] + const check = (state: JournalReducerState) => { + for (const card of ['D', 'N']) { + expect( + ['M1', 'M2', 'M3'].map((id) => + journalInputOrder(state).precedes( + state.queue.cards.get(card)!, + state.submissions.get(id)! + ) + ) + ).toEqual(expected) + } + expect(['M1', 'M2', 'M3'].map((id) => journalInputOrder(state).forced(id))).toEqual([ + true, + true, + true + ]) + expect( + ['M1', 'M2', 'M3'].map((id) => + journalInputOrder(state).precedes( + state.queue.cards.get('L')!, + state.submissions.get(id)! + ) + ) + ).toEqual([false, false, false]) + } + await journal.queuedMessages.insert({ messageId: 'L', body: message('later') }) + for (let pass = 0; pass < 3; pass++) { + if (pass) { + await journal.replaceEpochItems('unreconcilable_prefix', 0, []) + } + check(journal.queuedMessages.displayState()) + const replay = loadTestJournal(root, identity.sessionId)! + expect(replay.damage).toBeNull() + expect(replay.newer).toBeNull() + check(replay.state) + for (const id of ['M1', 'M3']) { + expect(JSON.stringify(readCommandReceipt(database.db, { kind: 'global' }, id))).toContain( + originals.get(id)! + ) + } + expect(readCommandReceipt(database.db, { kind: 'global' }, 'M2').verdict).toBe('absent') + expect(journal.itemBody(agentJournalSubmissionKey('M2'))).toEqual(message('M2')) + expect(journal.submission('M2')?.options).toEqual({ effort: 'high' }) + } + } finally { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } + } +) + +it('retries partial keeps from committed facts and carries rank moves on the first successful keep', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-partial-order-')) + const journals = createTrackedJournalOpener() + const identity = { + sessionId: 'partial', + workspaceId: 'folder', + hostId: 'ssh', + agent: 'claude' as const, + providerHandle: claudeProviderHandle('thread', null) + } + let journal = await journals.open({ identity, stateDirectory: root }) + try { + await journal.queuedMessages.insert({ + messageId: 'C', + body: { ...message('/compact'), command: { name: 'compact' } } + }) + await journal.queuedMessages.insert({ messageId: 'D', body: message('D') }) + for (const id of ['M1', 'M2']) { + await journal.appendSubmission({ + clientMessageId: id, + payloadFingerprint: id, + body: message(id), + handoverRecorded: true, + origin: 'client', + fence: 0 + }) + } + await journal.queuedMessages.insert({ messageId: 'L', body: message('L') }) + await journal.close() + journal = await journals.open({ identity, stateDirectory: root }) + const resolve = journal.resolveDispatch.bind(journal) + const fault = vi + .spyOn(journal, 'resolveDispatch') + .mockImplementation((...args) => + args[0].clientMessageId === 'M1' + ? Promise.reject(new Error('keep and fallback unavailable')) + : resolve(...args) + ) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + await expect( + holdUnsentSends(journal, { fence: 0, hold: { cause: 'hostRestarted' } }) + ).rejects.toThrow('settling unsent sends failed') + expect(journal.submission('M1')?.dispatchState).toBe('pending') + expect(journal.submission('M2')?.keptAsQueuedMessageId).toBe('M2') + expect(journal.queuedMessages.get('L')?.position).toBeGreaterThan( + journal.queuedMessages.get('M2')!.position + ) + fault.mockRestore() + warn.mockRestore() + await holdUnsentSends(journal, { fence: 0, hold: { cause: 'hostRestarted' } }) + const saved = journal.queuedMessages.list().filter((card) => card.state === 'waiting') + expect(saved.map((card) => card.messageId)).toEqual(['C', 'D', 'M1', 'M2', 'L']) + expect( + saved.every( + (card, i) => + Number.isFinite(card.position) && (i === 0 || card.position > saved[i - 1].position) + ) + ).toBe(true) + await journal.close() + journal = await journals.open({ identity, stateDirectory: root }) + await holdUnsentSends(journal, { fence: 0, hold: { cause: 'hostRestarted' } }) + expect( + journal.queuedMessages + .list() + .filter((card) => card.state === 'waiting') + .map((card) => card.messageId) + ).toEqual(saved.map((card) => card.messageId)) + } finally { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } +}) diff --git a/src/main/native-chat/agent-session-journal/journal-input-order.ts b/src/main/native-chat/agent-session-journal/journal-input-order.ts new file mode 100644 index 00000000000..2a19fa3237c --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-input-order.ts @@ -0,0 +1,323 @@ +import { + readJournalInputPlace, + restoreJournalInputPlaces, + currentJournalInputIdentity, + journalInputOrderChanged, + isJournalInputOverride +} from './journal-input-order-state' +export type { JournalInputPlace } from './journal-input-order-state' + +import { JournalSubmissionLinks } from './journal-submission-links' +import type { + AgentJournalCursor, + AgentJournalSubmission +} from '../../../shared/agent-session-journal-types' +import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import type { JournalReducerState } from './journal-reducer' +import type { JournalRow } from './journal-row-schema' +import type { QueuedMessageRow } from './queued-message-table' +import type { InputPlaceQuery } from './journal-input-place-index' +import { JournalInputLineageOwner } from './journal-input-lineage' +import { JournalInputCardIndex } from './journal-input-card-index' + +const owners = new WeakMap() +export function initializeJournalInputOrder( + state: JournalReducerState, + originalAcceptance?: (id: string) => AgentJournalCursor | undefined +): JournalInputOrder { + const owner = new JournalInputOrder(state, originalAcceptance) + owners.set(state, owner) + return owner +} +export function journalInputOrder(state: JournalReducerState): JournalInputOrder { + return owners.get(state) ?? initializeJournalInputOrder(state) +} +export function setJournalInputOrder(state: JournalReducerState, owner: JournalInputOrder): void { + owners.set(state, owner) +} + +/** Own admission places and shared command witnesses are reconstructed by the journal fold. */ +export class JournalInputOrder { + private readonly links = new JournalSubmissionLinks() + private reconstructing = false + private legacyPlaces = 0 + private readonly recoveryOwed = new Set() + private readonly originals = new Map() + private readonly unsettled = new Map() + private readonly queued = new Map() + private readonly entered = new Map() + private readonly overrides = new Map() + private pending: readonly AgentJournalSubmission[] | undefined + readonly lineage: JournalInputLineageOwner + private readonly cards: JournalInputCardIndex + private receiptCount = 0 + private queueRevision = 0 + private revision = 0 + + constructor( + private readonly state: JournalReducerState, + private readonly originalAcceptance?: (id: string) => AgentJournalCursor | undefined + ) { + this.lineage = new JournalInputLineageOwner(state.inputCards) + this.cards = new JournalInputCardIndex(this.lineage) + } + + beforeRow(state: JournalReducerState, row: JournalRow): void { + if (row.kind === 'dispatch' && row.keptAsQueuedMessageId) { + const place = this.place(row.clientMessageId) + if (place && !this.lineage.cards.has(row.keptAsQueuedMessageId)) { + this.lineage.cards.set(row.keptAsQueuedMessageId, { + introduced: place.context.cut, + baseline: { coordinate: place.coordinate } + }) + } + } + this.receiptCount = state.receipts.size + this.queueRevision = state.queue.revision + if (row.kind === 'epoch' && row.inputOrderLineage) { + this.lineage.load(row.inputOrderLineage) + } + this.lineage.observe(row.seq) + if (row.kind !== 'submission' || !row.handoverRecorded || row.queuedMessageId !== undefined) { + return + } + if (this.state.inputEntries.has(row.clientMessageId)) { + return + } + const place = this.cards.admission(state, row, this.originalAcceptance) + if (place.context.basis === 'legacy') { + this.legacyPlaces++ + } + this.state.inputEntries.set(row.clientMessageId, { + context: place.context, + ...(place.witnessCardId ? { witnessCardId: place.witnessCardId } : {}) + }) + } + + // A streaming prefix cannot establish the last legacy consumer. + beginReconstruction(): void { + this.reconstructing = true + this.legacyPlaces = 0 + for (const entry of this.state.inputEntries.values()) { + if (entry.context.basis === 'legacy') { + this.legacyPlaces++ + } + } + } + finishReconstruction(state: JournalReducerState): void { + this.reconstructing = false + if (this.legacyPlaces === 0) { + this.releaseHistory(state) + } + } + + restore(state: JournalReducerState): void { + this.legacyPlaces = restoreJournalInputPlaces(state) + this.cards.ensure(state) + for (const id of state.submissions.keys()) { + this.refresh(state, id) + } + this.lineage.restoreFrontier(state.lastSequence, state.inputEntries.values()) + } + + linkedVersion(): number { + return this.links.version() + } + linkedTo(id: string): Iterable { + return this.links.to(id) + } + overlay(state: JournalReducerState, rows: readonly QueuedMessageRow[]): void { + this.cards.overlay(state, rows) + } + introduceOverlays(state: JournalReducerState): void { + this.cards.introduceOverlays((card) => this.queueChanged(state, undefined, card)) + } + + markOpeningRecovery(): void { + for (const entry of this.unsettled.values()) { + if (isQueuedAgentJournalSubmission(entry)) { + this.markRecoveryPending(entry.clientMessageId) + } + } + } + + markRecoveryPending(id: string): void { + this.recoveryOwed.add(id) + const queued = this.queued.delete(id) + const override = this.overrides.delete(id) + if (queued || override) { + this.revision++ + } + } + recoveryPending(id: string): boolean { + return this.recoveryOwed.has(id) + } + carryRecovery(previous: JournalInputOrder): void { + for (const id of previous.recoveryOwed) { + if (this.queued.has(id)) { + this.markRecoveryPending(id) + } + } + } + + place(id: string): ReturnType { + return readJournalInputPlace(this.state, id, this.originals, this.originalAcceptance) + } + forced(id: string): boolean { + return this.state.inputEntries.get(id)?.witnessCardId !== undefined + } + firstQueued(): AgentJournalSubmission | undefined { + return this.queued.values().next().value + } + explicitSend(): AgentJournalSubmission | undefined { + return this.overrides.values().next().value + } + enteredSubmissions(): Iterable { + return this.entered.values() + } + unsettledSubmissions(): readonly AgentJournalSubmission[] { + return (this.pending ??= [...this.unsettled.values()]) + } + version(): number { + return this.revision + } + + head(state: JournalReducerState, floor: AgentJournalCursor | null): QueuedMessageRow | undefined { + this.cards.ensure(state, floor) + return this.cards.head() + } + paused( + state: JournalReducerState, + card: QueuedMessageRow, + floor: AgentJournalCursor | null + ): boolean { + this.cards.ensure(state, floor) + return this.cards.paused(card) + } + + predecessors(state: JournalReducerState, id: string): InputPlaceQuery { + this.cards.ensure(state) + const place = this.place(id) + return place?.witnessCardId ? this.cards.query(place) : { position: -Infinity } + } + + queueChanged( + state: JournalReducerState, + prior: QueuedMessageRow | undefined, + card: QueuedMessageRow + ): void { + this.cards.queueChanged(state, prior, card) + if (!this.reconstructing && this.legacyPlaces === 0) { + const source = this.lineage.cards.get(card.messageId) + if (source) { + delete source.baseline + } + } + if (card.state === 'withdrawn' && !this.reconstructing && this.legacyPlaces === 0) { + this.lineage.cards.delete(card.messageId) + } + } + + afterRow(state: JournalReducerState, row: JournalRow): void { + if (journalInputOrderChanged(state, row, (key) => this[key])) { + this.revision++ + } + if (row.kind === 'submission' || row.kind === 'dispatch') { + this.refresh(state, row.clientMessageId) + } else if (row.kind === 'tombstone' && row.retireSubmission) { + this.refreshItem(state, row.itemId) + } else if (row.kind === 'item') { + this.refreshItem(state, state.aliases.get(row.itemId) ?? row.itemId) + } else if (row.kind === 'lifecycle-batch') { + for (const mutation of row.mutations) { + this.refreshItem(state, state.aliases.get(mutation.itemId) ?? mutation.itemId) + } + } else if (row.kind === 'queue') { + const id = state.queue.cards.get(row.messageId)?.consumedAs + if (id) { + this.refresh(state, id) + } + } + } + + private refreshItem(state: JournalReducerState, itemId: string): void { + const identity = parseAgentJournalItemKey(itemId) + if (identity?.provider === 'orca') { + this.refresh(state, identity.clientMessageId) + } + } + + private refresh(state: JournalReducerState, id: string): void { + const entry = state.submissions.get(id) + this.links.refresh(id, entry) + if (entry?.dispatchState === 'pending' || entry?.dispatchState === 'unknown') { + if (this.unsettled.get(id) !== entry) { + this.unsettled.set(id, entry) + this.pending = undefined + this.revision++ + } + } else if (this.unsettled.delete(id)) { + this.pending = undefined + this.revision++ + } + if (entry?.dispatchState !== 'pending') { + this.releasePlace(state, id) + } + if (entry && isQueuedAgentJournalSubmission(entry)) { + if (!this.recoveryOwed.has(id)) { + this.queued.set(id, entry) + } + this.entered.delete(id) + if (isJournalInputOverride(entry, this.recoveryOwed.has(id))) { + this.overrides.set(id, entry) + } + } else { + this.recoveryOwed.delete(id) + this.queued.delete(id) + this.overrides.delete(id) + if (entry && (entry.dispatchState === 'pending' || entry.dispatchState === 'unknown')) { + this.entered.set(id, entry) + } else { + this.entered.delete(id) + } + } + if (entry?.queuedMessageId) { + const card = state.queue.cards.get(entry.queuedMessageId) + if (card?.consumedAs === id) { + this.cards.indexCard(state, card) + if ( + !this.reconstructing && + this.legacyPlaces === 0 && + !this.currentIdentity(state, card.messageId) + ) { + this.lineage.cards.delete(card.messageId) + } + } + } + } + + private currentIdentity(state: JournalReducerState, id: string): boolean { + return currentJournalInputIdentity(state, id, () => this.cards.hasOverlay(id)) + } + + private releasePlace(state: JournalReducerState, id: string): void { + const place = this.state.inputEntries.get(id) + if (place && this.state.inputEntries.delete(id)) { + this.originals.delete(id) + if (place.context.basis === 'legacy' && --this.legacyPlaces === 0 && !this.reconstructing) { + this.releaseHistory(state) + } + } + } + + private releaseHistory(state: JournalReducerState): void { + this.lineage.releaseHistory((id) => this.currentIdentity(state, id)) + this.cards.releaseHistory() + } + + precedes(card: QueuedMessageRow, submission: AgentJournalSubmission): boolean { + const place = this.place(submission.clientMessageId) + return Boolean(place?.witnessCardId && this.cards.relation(card, place)) + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-input-place-index.ts b/src/main/native-chat/agent-session-journal/journal-input-place-index.ts new file mode 100644 index 00000000000..f9107817aea --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-input-place-index.ts @@ -0,0 +1,169 @@ +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import type { QueuedMessageRow } from './queued-message-table' + +export type InputPlaceQuery = { + head?: QueuedMessageRow + command?: QueuedMessageRow + waiting?: QueuedMessageRow + handoff?: QueuedMessageRow + position: number +} +type Entry = InputPlaceQuery & { card: QueuedMessageRow; sequence: number; domain: string } +type Node = { + entry: Entry + height: number + result: InputPlaceQuery + left?: Node + right?: Node +} +const empty = (): InputPlaceQuery => ({ position: -Infinity }) +const first = (a?: QueuedMessageRow, b?: QueuedMessageRow) => + !a || (b && b.position < a.position) ? b : a +export function mergeInputPlaces(a: InputPlaceQuery, b: InputPlaceQuery): InputPlaceQuery { + return { + head: first(a.head, b.head), + command: first(a.command, b.command), + waiting: first(a.waiting, b.waiting), + handoff: first(a.handoff, b.handoff), + position: Math.max(a.position, b.position) + } +} +const height = (node?: Node) => node?.height ?? 0 +const compare = (a: Entry, b: Entry) => + a.sequence - b.sequence || a.card.messageId.localeCompare(b.card.messageId) +function update(node: Node): Node { + node.height = 1 + Math.max(height(node.left), height(node.right)) + node.result = mergeInputPlaces( + mergeInputPlaces(node.left?.result ?? empty(), node.entry), + node.right?.result ?? empty() + ) + return node +} +function left(node: Node): Node { + const next = node.right! + node.right = next.left + next.left = update(node) + return update(next) +} +function right(node: Node): Node { + const next = node.left! + node.left = next.right + next.right = update(node) + return update(next) +} +function balance(node: Node): Node { + update(node) + if (height(node.left) - height(node.right) > 1) { + if (height(node.left!.right) > height(node.left!.left)) { + node.left = left(node.left!) + } + return right(node) + } + if (height(node.right) - height(node.left) > 1) { + if (height(node.right!.left) > height(node.right!.right)) { + node.right = right(node.right!) + } + return left(node) + } + return node +} +function put(node: Node | undefined, entry: Entry, remove: boolean): Node | undefined { + if (!node) { + return remove ? undefined : { entry, height: 1, result: entry } + } + const order = compare(entry, node.entry) + if (order < 0) { + node.left = put(node.left, entry, remove) + } else if (order > 0) { + node.right = put(node.right, entry, remove) + } else if (!remove) { + node.entry = entry + } else { + if (!node.left) { + return node.right + } + if (!node.right) { + return node.left + } + let next = node.right + while (next.left) { + next = next.left + } + node.entry = next.entry + node.right = put(node.right, next.entry, true) + } + return balance(node) +} + +/** Current card coordinates only; subtree summaries answer admission/selection/recovery cuts. */ +export class JournalInputPlaceIndex { + private readonly epochs = new Map() + private all: Node | undefined + private readonly entries = new Map() + + constructor(_epoch: string) {} + + set( + card: QueuedMessageRow, + actionable: boolean, + handoff: boolean, + coordinate: AgentJournalCursor | null = card.queuedAt + ): void { + const prior = this.entries.get(card.messageId) + if (prior) { + this.change(prior, true) + } + this.entries.delete(card.messageId) + if (card.state === 'withdrawn') { + return + } + const entry: Entry = { + card, + sequence: coordinate?.sequence ?? 0, + domain: coordinate?.epoch ?? '', + position: card.position, + ...((card.state === 'waiting' && card.holdReason === null) || card.state === 'returned' + ? { head: card } + : {}), + ...(actionable ? { waiting: card, ...(card.body.command ? { command: card } : {}) } : {}), + ...(handoff ? { handoff: card } : {}) + } + this.entries.set(card.messageId, entry) + this.change(entry, false) + } + + private change(entry: Entry, remove: boolean): void { + const epoch = entry.domain + const root = put(this.epochs.get(epoch), entry, remove) + if (root) { + this.epochs.set(epoch, root) + } else { + this.epochs.delete(epoch) + } + this.all = put(this.all, entry, remove) + } + + isEmpty(): boolean { + return this.entries.size === 0 + } + + query(epoch: string, sequence = Infinity): InputPlaceQuery { + const before = (node?: Node): InputPlaceQuery => { + if (!node) { + return empty() + } + if (node.entry.sequence >= sequence) { + return before(node.left) + } + return mergeInputPlaces( + mergeInputPlaces(node.left?.result ?? empty(), node.entry), + before(node.right) + ) + } + return before(this.epochs.get(epoch)) + } + + allPlaces(): InputPlaceQuery { + return this.all?.result ?? empty() + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-input-provenance.ts b/src/main/native-chat/agent-session-journal/journal-input-provenance.ts new file mode 100644 index 00000000000..b7b09925e1e --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-input-provenance.ts @@ -0,0 +1,113 @@ +import type Database from '../../sqlite/sync-database' +import type { + AgentJournalCursor, + AgentJournalSubmission +} from '../../../shared/agent-session-journal-types' +import { readCommandReceipt } from './command-receipt-table' + +/** Only the original send's accepted coordinate can compare a retired enqueue boundary. */ +export function journalInputAcceptance( + db: Database.Database, + sessionId: string, + id: string +): AgentJournalCursor | undefined { + const read = readCommandReceipt(db, { kind: 'global' }, id) + if ( + read.verdict === 'readable' && + read.receipt.status === 'accepted' && + read.receipt.sessionId === sessionId && + read.receipt.method === 'agentSession.send' && + read.receipt.result.kind === 'journal-row' + ) { + return { epoch: read.receipt.result.epoch, sequence: read.receipt.result.sequence } + } + return undefined +} + +/** Reuse comparable admission coordinates; original receipts remain the provenance authority. */ +export function journalQueueRebase( + db: Database.Database, + sessionId: string, + epoch: string, + previousEpoch: string | undefined, + queued: readonly AgentJournalSubmission[], + rebuilt: ReadonlyMap +): (queuedAt: AgentJournalCursor | null) => AgentJournalCursor | null { + type Position = { sequence: number; rebased: number } + const comparableByEpoch = new Map() + let originalPositions: Map | undefined + const originals = () => { + if (!originalPositions) { + originalPositions = new Map() + for (const submission of queued) { + const original = journalInputAcceptance(db, sessionId, submission.clientMessageId) + if (original) { + const positions = originalPositions.get(original.epoch) ?? [] + positions.push({ + sequence: original.sequence, + rebased: rebuilt.get(submission.clientMessageId) ?? 1 + }) + originalPositions.set(original.epoch, positions) + } + } + } + return originalPositions + } + return (queuedAt) => { + if (queuedAt && previousEpoch !== undefined) { + const boundary = queuedAt + let comparable = comparableByEpoch.get(boundary.epoch) + if (!comparable) { + if (boundary.epoch === previousEpoch) { + const current = queued.flatMap((submission) => + submission.acceptedSequence === undefined + ? [] + : [ + { + sequence: submission.acceptedSequence, + rebased: rebuilt.get(submission.clientMessageId) ?? 1 + } + ] + ) + if (current.length === queued.length) { + comparable = current + } else { + // Legacy missing admission places retain only comparable original coordinates. + const missing = new Set( + queued + .filter((submission) => submission.acceptedSequence === undefined) + .map((submission) => rebuilt.get(submission.clientMessageId)) + ) + comparable = [ + ...current, + ...(originals().get(boundary.epoch) ?? []).filter((position) => + missing.has(position.rebased) + ) + ] + } + } else { + comparable = originals().get(boundary.epoch) ?? [] + } + comparable.sort((a, b) => a.sequence - b.sequence) + for (let i = 1; i < comparable.length; i++) { + comparable[i].rebased = Math.max(comparable[i].rebased, comparable[i - 1].rebased) + } + comparableByEpoch.set(boundary.epoch, comparable) + } + if (comparable.length === queued.length) { + let low = 0 + let high = comparable.length + while (low < high) { + const mid = (low + high) >>> 1 + if (comparable[mid].sequence <= boundary.sequence) { + low = mid + 1 + } else { + high = mid + } + } + queuedAt = { epoch, sequence: low === 0 ? 1 : comparable[low - 1].rebased } + } + } + return queuedAt + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-mixed-epoch-replacement.test.ts b/src/main/native-chat/agent-session-journal/journal-mixed-epoch-replacement.test.ts index 6f1a72ca78f..312ad155715 100644 --- a/src/main/native-chat/agent-session-journal/journal-mixed-epoch-replacement.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-mixed-epoch-replacement.test.ts @@ -9,11 +9,15 @@ import { type AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import { createTrackedJournalOpener, + insertTestJournalRow, liveTestJournalRows, - openTestJournalHostDatabase + openTestJournalHostDatabase, + seedLegacyTestJournal } from './journal-host-database-test-support' +import type { JournalRow } from './journal-row-schema' import { journalOutcomeSource } from './journal-outcome-format' import { sweepJournalOutcomePage } from './journal-outcome-cleanup' import type { AgentSessionJournal } from './journal-store' @@ -67,15 +71,69 @@ const recordsOf = (generation: number) => /** Old rows (items 0 and 1, a waiting card, a user Stop), then this build's records (item 0 * revised, item 2, a second card). */ async function mixedChat(): Promise { - const older = createTrackedJournalOpener({ legacyInputs: true }) - const earlier = await older.open({ identity: IDENTITY, stateDirectory: root }) - await earlier.appendItem(item(0), status('first'), scope) - await earlier.appendItem(item(1), status('second'), scope) - await earlier.queuedMessages.insert({ messageId: 'card-old', body: message('old card') }) - await earlier.appendStopEvent({ reason: 'user-stop' }, FENCE) - await older.closeAll() + // Freeze the old writer's rows before opening a current native coordinate owner. + const database = openTestJournalHostDatabase(root) + const epoch = 'old-input-epoch' + seedLegacyTestJournal(database, { identity: IDENTITY, mintEpoch: () => epoch, now: () => 1000 }) + const base = { v: 2, epoch, fence: FENCE, ts: 1000 } + const prefix = [ + ...[status('first'), status('second')].map((body, ordinal): JournalRow => ({ + ...base, + kind: 'item', + seq: ordinal + 2, + itemId: agentJournalItemKey(item(ordinal)), + revision: 1, + body, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + })), + { + ...base, + v: 4, + kind: 'queue', + seq: 4, + messageId: 'card-old', + change: { + op: 'queued', + body: message('old card'), + position: 0, + queuedAt: { epoch, sequence: 4 } + } + }, + { + ...base, + kind: 'tombstone', + seq: 5, + itemId: agentJournalItemKey({ provider: 'orca', clientMessageId: 'stop-event' }), + revision: 1, + stopEvent: { reason: 'user-stop', at: 1000 } + } + ] satisfies JournalRow[] + database.transaction((db) => { + for (const row of prefix) { + insertTestJournalRow(db, IDENTITY.sessionId, row) + } + }) + expect( + rowsOfEpoch(epoch) + .slice(1) + .map((row) => JSON.parse(String(row.row_json))) + ).toEqual(prefix) + expect(journalOutcomeSource(db(), IDENTITY.sessionId, epoch)).toBeNull() const journal = await journals.open({ identity: IDENTITY, stateDirectory: root }) + expect(seen(journal)).toEqual({ + items: [status('first'), status('second')], + cards: [['card-old', message('old card'), 'waiting']], + stop: 'user-stop' + }) await journal.appendItem(item(0), status('first, revised'), scope) + const converted = journalOutcomeSource(db(), IDENTITY.sessionId, epoch) + expect(converted).toMatchObject({ prefix: 5, format: 2 }) + expect( + db() + .prepare(`SELECT entry_key FROM journal_outcomes + WHERE generation = ? AND field = 'inputCards' AND deleted = 0`) + .all(converted!.generation) + ).toEqual([{ entry_key: JSON.stringify('card-old') }]) await journal.appendItem(item(2), status('third'), scope) await journal.queuedMessages.insert({ messageId: 'card-new', body: message('new card') }) const source = journalOutcomeSource(db(), IDENTITY.sessionId, journal.epoch) diff --git a/src/main/native-chat/agent-session-journal/journal-open.ts b/src/main/native-chat/agent-session-journal/journal-open.ts index 35a2d593291..82138bda6c9 100644 --- a/src/main/native-chat/agent-session-journal/journal-open.ts +++ b/src/main/native-chat/agent-session-journal/journal-open.ts @@ -1,3 +1,5 @@ +import { readJournalLegacyCards } from './journal-queue-import' +import { journalInputOrder } from './journal-input-order' // Replay live epochs without rewriting history; damage/newer rows fail load and remain on disk. import { journalOutcomeSource } from './journal-outcome-format' @@ -6,6 +8,7 @@ import { JournalOutcomeDecodeError } from './journal-outcome-codec' import { readJournalOutcomeGroups, type JournalReadRow } from './journal-read-row' import { AgentSessionJournalError } from './journal-write-guards' import type Database from '../../sqlite/sync-database' +import { journalInputAcceptance } from './journal-input-provenance' import { applyJournalRow, createJournalReducerState, @@ -38,7 +41,10 @@ export function replayJournal(db: Database.Database, sessionId: string): Journal if (epoch === null) { return null } - return foldJournalFromState(db, createJournalReducerState(sessionId, epoch)) + return foldJournalFromState( + db, + createJournalReducerState(sessionId, epoch, (id) => journalInputAcceptance(db, sessionId, id)) + ) } /** The read is synchronous, like main's replay: it blocks once for one chat's saved records (its @@ -49,7 +55,9 @@ export function restoreJournal(db: Database.Database, sessionId: string): Journa if (epoch === null) { return null } - const empty = createJournalReducerState(sessionId, epoch) + const empty = createJournalReducerState(sessionId, epoch, (id) => + journalInputAcceptance(db, sessionId, id) + ) try { const source = journalOutcomeSource(db, sessionId, epoch) if (!source) { @@ -57,7 +65,7 @@ export function restoreJournal(db: Database.Database, sessionId: string): Journa } let baseline: JournalReducerState | undefined if (source.prefix > 0) { - const prefix = foldJournalFromState(db, empty, source.prefix) + const prefix = foldJournalFromState(db, empty, source.prefix, true) if (prefix.newer || prefix.damage || prefix.state.lastSequence !== source.prefix) { const gap = { sequence: prefix.state.lastSequence + 1, cause: 'sequence-gap' as const } return { ...prefix, damage: prefix.newer ? null : (prefix.damage ?? gap) } @@ -82,10 +90,11 @@ export function restoreJournal(db: Database.Database, sessionId: string): Journa function foldJournalFromState( db: Database.Database, state: JournalReducerState, - through = Number.POSITIVE_INFINITY + through = Number.POSITIVE_INFINITY, + deferNativeCleanup = false ): JournalLoad { const { sessionId, epoch, lastSequence } = state - const fold = startJournalRowFold({ sessionId, epoch }, state) + const fold = startJournalRowFold({ sessionId, epoch, deferNativeCleanup }, state) for (const entry of iterateJournalEpochRows( db, sessionId, @@ -96,17 +105,27 @@ function foldJournalFromState( break } } + if (!deferNativeCleanup) { + journalInputOrder(state).overlay(state, readJournalLegacyCards(db, state)) + } return fold.finish() } /** untilDamage stops at the first damage, for callers that only ask whether a fold publishes. */ export function startJournalRowFold( - input: { sessionId: string; epoch: string; untilDamage?: boolean }, - state = createJournalReducerState(input.sessionId, input.epoch) + input: { + sessionId: string + epoch: string + untilDamage?: boolean + deferNativeCleanup?: boolean + originalAcceptance?: Parameters[2] + }, + state = createJournalReducerState(input.sessionId, input.epoch, input.originalAcceptance) ): { add: (entry: { seq: number; rowJson: string; ts?: number }) => boolean finish: () => JournalLoad } { + journalInputOrder(state).beginReconstruction() let expectedSequence = state.lastSequence + 1 let damage: JournalDamage | null = null let anchored = state.lastSequence > 0 @@ -149,6 +168,13 @@ export function startJournalRowFold( return true } const finish = (): JournalLoad => { + if (!input.deferNativeCleanup) { + journalInputOrder(state).finishReconstruction(state) + journalInputOrder(state).lineage.restoreFrontier( + state.lastSequence, + state.inputEntries.values() + ) + } state.oldestSequence = FIRST_JOURNAL_SEQUENCE // Empty epochs contain no recoverable history; founding a fresh epoch loses nothing. const found = diff --git a/src/main/native-chat/agent-session-journal/journal-outcome-codec.ts b/src/main/native-chat/agent-session-journal/journal-outcome-codec.ts index 4f7ee7e222d..09a6ab9dd1c 100644 --- a/src/main/native-chat/agent-session-journal/journal-outcome-codec.ts +++ b/src/main/native-chat/agent-session-journal/journal-outcome-codec.ts @@ -1,3 +1,9 @@ +import { + admitOutcomeFormat2, + JOURNAL_OUTCOME_FORMAT_2, + isNativeOutcomeField, + decodeNativeOutcome +} from './journal-outcome-format-2' import { z } from 'zod' import type { SqliteRow } from '../../sqlite/sqlite-statement' import type { JournalReducerState } from './journal-reducer' @@ -46,7 +52,7 @@ export function encodeJournalOutcomes( seq: state.lastSequence, field: entry.path, key: JSON.stringify(entry.key), - format: JOURNAL_OUTCOME_FORMAT_1, + format: JOURNAL_OUTCOME_FORMAT_2, json: entry.deleted ? 'null' : entry.path === 'items' @@ -61,7 +67,7 @@ export function encodeJournalOutcomes( seq: state.lastSequence, field: '$', key: JSON.stringify(key), - format: JOURNAL_OUTCOME_FORMAT_1, + format: JOURNAL_OUTCOME_FORMAT_2, json: JSON.stringify(value), deleted: false }) @@ -69,8 +75,8 @@ export function encodeJournalOutcomes( } for (const value of values) { // A reducer field format 1 has no home for needs a new outcome format, never a silent change. - if (admitOutcomeFormat1(value) === 'unknown') { - throw new Error(`journal outcome format 1 does not store ${value.field} ${value.key}`) + if (admitOutcomeFormat2(value) === 'unknown') { + throw new Error(`journal outcome format 2 does not store ${value.field} ${value.key}`) } } return { values, scalars: next } @@ -114,7 +120,11 @@ function admitJournalOutcome(row: SqliteRow): { value: JournalOutcome; admitted: deleted: row.deleted === 1 } const admitted = - value.format === JOURNAL_OUTCOME_FORMAT_1 ? admitOutcomeFormat1(value) : 'unknown' + value.format === JOURNAL_OUTCOME_FORMAT_1 + ? admitOutcomeFormat1(value) + : value.format === JOURNAL_OUTCOME_FORMAT_2 + ? admitOutcomeFormat2(value) + : 'unknown' if (admitted === 'unknown') { throw new AgentSessionJournalError('journal_read_only', 'the chat holds a newer outcome') } @@ -152,6 +162,16 @@ export class JournalOutcomeFold { return value } + introduction(field: string, id: string): number | undefined { + const ordinal = this.values.get(field)?.get(JSON.stringify(id))?.ordinal + return ordinal?.startsWith('s') ? Number(ordinal.slice(1, 17)) : undefined + } + + replaced(field: string, id: string): boolean { + const value = this.values.get(field)?.get(JSON.stringify(id)) + return Boolean(value && (value.deleted || value.ordinal !== JOURNAL_PREFIX_ORDINAL)) + } + /** Applies the latest values over `baseline` (the folded input prefix) or over the scalars alone: * a deletion removes, a prefix-ordinal value replaces in place, and every other value takes its * canonical place after the prefix. Each record was admitted as format 1 on read. A prefix-ordinal @@ -173,7 +193,7 @@ export class JournalOutcomeFold { ] const decode = decodingOutcome(() => decodeOutcomeFormat1Onto(scalars, baseline)) for (const [field, entries] of this.values) { - if (field === '$') { + if (field === '$' || isNativeOutcomeField(field)) { continue } // Canonical ordinals compare bytewise, as SQLite's BINARY collation orders them; the prefix @@ -191,6 +211,19 @@ export class JournalOutcomeFold { } } const state = decode.finish() + for (const [field, entries] of this.values) { + if (!isNativeOutcomeField(field)) { + continue + } + const ordered = [...entries.values()].sort((a, b) => + a.ordinal < b.ordinal ? -1 : a.ordinal > b.ordinal ? 1 : 0 + ) + for (const value of ordered) { + decodingOutcome(() => + decodeNativeOutcome(state, field, value.entryKey, value.kept, value.deleted) + ) + } + } if (state.sessionId !== sessionId || state.epoch !== epoch) { throw new JournalOutcomeDecodeError('invalid journal outcome coverage') } diff --git a/src/main/native-chat/agent-session-journal/journal-outcome-format-1.test.ts b/src/main/native-chat/agent-session-journal/journal-outcome-format-1.test.ts index 42a60bf90cf..548b8778b38 100644 --- a/src/main/native-chat/agent-session-journal/journal-outcome-format-1.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-outcome-format-1.test.ts @@ -72,7 +72,9 @@ afterEach(async () => { }) function plain(state: JournalReducerState): unknown { - const fields: Record = { ...state } + const fields: Record = Object.fromEntries( + Object.entries(state).filter(([key]) => key !== 'inputEntries' && key !== 'inputCards') + ) return JSON.parse( JSON.stringify({ ...Object.fromEntries( @@ -163,7 +165,7 @@ it('refuses a field format 1 does not define as written by a newer Orca', () => }) it('refuses a record of a newer format even when the epoch summary was not raised', () => { - insertOutcome('receipts', '"client-9"', '["client-9",{}]', 2) + insertOutcome('receipts', '"client-9"', '["client-9",{}]', 3) expect(restoreJournal(database.db, sessionId)).toMatchObject(newer) }) @@ -199,3 +201,32 @@ it('admits and decodes every declared shape as stored, and refuses one key more 'unknown' ) }) + +it('distinguishes new canonical damage from newer keys, without changing frozen format-1 admission', () => { + const own = { + context: { source: { epoch, sequence: 1 }, basis: 'observed', cut: 1 }, + witnessCardId: 'C' + } + const write = (value: unknown) => { + database.db.prepare("DELETE FROM journal_outcomes WHERE field='inputEntries'").run() + insertOutcome('inputEntries', '"client-1"', JSON.stringify(['client-1', value]), 2) + } + write(own) + expect(restoreJournal(database.db, sessionId)).toMatchObject({ newer: null, damage: null }) + for (const context of [ + { ...own.context, cut: -1 }, + { ...own.context, cut: Number.MAX_SAFE_INTEGER + 1 }, + { ...own.context, source: { epoch: '', sequence: 1 } } + ]) { + write({ ...own, context }) + expect(restoreJournal(database.db, sessionId)).toMatchObject({ + newer: null, + damage: { cause: 'unparseable-row' } + }) + } + write({ ...own, futureNativeKey: true }) + expect(restoreJournal(database.db, sessionId)).toMatchObject(newer) + // A new native family is never accepted as an additive format-1 record. + database.db.prepare("UPDATE journal_outcomes SET format=1 WHERE field='inputEntries'").run() + expect(restoreJournal(database.db, sessionId)).toMatchObject(newer) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-outcome-format-2.ts b/src/main/native-chat/agent-session-journal/journal-outcome-format-2.ts new file mode 100644 index 00000000000..df7ddafc0d4 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-outcome-format-2.ts @@ -0,0 +1,105 @@ +import { z } from 'zod' +import type { JournalReducerState } from './journal-reducer' +import { admitOutcomeFormat1, type AdmittedOutcome } from './journal-outcome-format-1' +import { + strictStorage, + storageVerdict, + type StorageDeclaresAll, + type UndeclaredStorageKeys +} from './journal-storage-strict' + +export const JOURNAL_OUTCOME_FORMAT_2 = 2 +const ordinal = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const coordinate = z.object({ epoch: z.string().min(1), sequence: ordinal }) +export const inputEntrySchema = z.object({ + context: z.object({ source: coordinate, basis: z.enum(['observed', 'legacy']), cut: ordinal }), + witnessCardId: z.string().min(1).optional() +}) +export const inputCardSchema = z.object({ + introduced: ordinal, + baseline: z.object({ coordinate: coordinate.nullable() }).optional() +}) +const entries = z.tuple([z.string().min(1), inputEntrySchema]) +const cards = z.tuple([z.string().min(1), inputCardSchema]) +const native = new Map([ + ['inputEntries', strictStorage(entries)], + ['inputCards', strictStorage(cards)] +]) + +export function isNativeOutcomeField(field: string): boolean { + return native.has(field) +} + +/** Ordinary format-1 values keep their frozen meaning; format 2 adds only the native joins. */ +export function admitOutcomeFormat2(value: { + field: string + key: string + json: string + deleted: boolean +}): 'unknown' | AdmittedOutcome { + const schema = native.get(value.field) + if (!schema) { + return admitOutcomeFormat1(value) + } + const key: unknown = JSON.parse(value.key) + if (typeof key !== 'string' || !key) { + throw new Error('invalid native journal key') + } + if (value.deleted) { + return value.json === 'null' ? { entryKey: key, kept: null } : 'unknown' + } + const kept: unknown = JSON.parse(value.json) + const verdict = storageVerdict(schema, kept) + if (verdict === 'newer') { + return 'unknown' + } + if (verdict !== 'stored' || !Array.isArray(kept) || kept[0] !== key) { + throw new Error('invalid native journal outcome') + } + return { entryKey: key, kept } +} + +const admittedEntry = + z.custom<[string, JournalReducerState['inputEntries'] extends Map ? V : never]>() +const admittedCard = + z.custom<[string, JournalReducerState['inputCards'] extends Map ? V : never]>() + +// Admission walked the declared schema once; these views keep the exact admitted bytes' value. +export function decodeNativeOutcome( + state: JournalReducerState, + field: string, + key: unknown, + kept: unknown, + deleted: boolean +): void { + if (field === 'inputEntries') { + if (deleted) { + state.inputEntries.delete(z.string().parse(key)) + } else { + const [id, value] = admittedEntry.parse(kept) + state.inputEntries.set(id, value) + } + } else if (field === 'inputCards') { + if (deleted) { + state.inputCards.delete(z.string().parse(key)) + } else { + const [id, value] = admittedCard.parse(kept) + state.inputCards.set(id, value) + } + } +} + +export type OutcomeFormat2DeclaresNativeKeys = [ + StorageDeclaresAll< + UndeclaredStorageKeys< + JournalReducerState['inputEntries'] extends Map ? V : never, + z.output + > + >, + StorageDeclaresAll< + UndeclaredStorageKeys< + JournalReducerState['inputCards'] extends Map ? V : never, + z.output + > + > +] diff --git a/src/main/native-chat/agent-session-journal/journal-outcome-format.ts b/src/main/native-chat/agent-session-journal/journal-outcome-format.ts index cfc7cb54ca6..c57ec245cb3 100644 --- a/src/main/native-chat/agent-session-journal/journal-outcome-format.ts +++ b/src/main/native-chat/agent-session-journal/journal-outcome-format.ts @@ -1,12 +1,12 @@ import type Database from '../../sqlite/sync-database' import { readJournalSessionEpoch } from './journal-row-table' import { AgentSessionJournalError } from './journal-write-guards' -import { JOURNAL_OUTCOME_FORMAT_1 } from './journal-outcome-format-1' +import { JOURNAL_OUTCOME_FORMAT_2 } from './journal-outcome-format-2' import { JournalOutcomeDecodeError } from './journal-outcome-codec' /** The newest record format this build reads, and the one it writes. Each stored record carries its * own format; older formats keep their frozen decoders and are never rewritten. */ -export const JOURNAL_OUTCOME_FORMAT = JOURNAL_OUTCOME_FORMAT_1 +export const JOURNAL_OUTCOME_FORMAT = JOURNAL_OUTCOME_FORMAT_2 /** Builds before outcome records refuse a row of this version wherever it sits in the epoch. * Under this version, how the input rows before the marker fold into keys and values is frozen: the * tail's records were written over that reading. A build that reads old rows into different keys or diff --git a/src/main/native-chat/agent-session-journal/journal-outcome-scaling.test.ts b/src/main/native-chat/agent-session-journal/journal-outcome-scaling.test.ts index be60e4ad4c4..3a083c0fdce 100644 --- a/src/main/native-chat/agent-session-journal/journal-outcome-scaling.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-outcome-scaling.test.ts @@ -28,6 +28,7 @@ import { encodeJournalState } from './journal-state-codec' import type * as RowStorage from './journal-storage-row-schema' import type * as Strict from './journal-storage-strict' import type * as OutcomeFormat1 from './journal-outcome-format-1' +import type * as OutcomeFormat2 from './journal-outcome-format-2' import type * as Transition from './journal-reducer-transition' import { projectionBase as base, @@ -44,7 +45,7 @@ import { // Admission's own work: strict walks of input rows and of outcome records, counted at the calls, // and every strict walk at all, counted at each strict schema storage builds. -const admission = vi.hoisted(() => ({ rows: 0, records: 0, walks: 0 })) +const admission = vi.hoisted(() => ({ rows: 0, records: 0, walks: 0, tombstones: 0 })) vi.mock('./journal-storage-strict', async (original) => { const actual = await original() // A schema storage cannot close further comes back as it is, so one instance can recur. @@ -84,13 +85,31 @@ vi.mock('./journal-outcome-format-1', async (original) => { ...actual, admitOutcomeFormat1: (value: Parameters[0]) => { admission.records += 1 + if (value.deleted) { + admission.tombstones++ + } return actual.admitOutcomeFormat1(value) } } }) +vi.mock('./journal-outcome-format-2', async (original) => { + const actual = await original() + return { + ...actual, + admitOutcomeFormat2: (value: Parameters[0]) => { + if (actual.isNativeOutcomeField(value.field)) { + admission.records++ + if (value.deleted) { + admission.tombstones++ + } + } + return actual.admitOutcomeFormat2(value) + } + } +}) // Every draft transition a write or a replacement folds through, to count the nodes it reaches. const transitions = vi.hoisted(() => { - const opened: { tracked: () => number }[] = [] + const opened: { tracked: () => number; trackedDerived: () => number }[] = [] return opened }) vi.mock('./journal-reducer-transition', async (original) => { @@ -146,7 +165,16 @@ const send = (seq: number, clientMessageId: string, handoverRecorded?: true): Jo clientMessageId, body: message(clientMessageId), providerHandle, - ...(handoverRecorded ? { handoverRecorded } : {}), + ...(handoverRecorded + ? { + handoverRecorded, + inputOrderContext: { + source: { epoch, sequence: seq }, + basis: 'observed' as const, + cut: seq + } + } + : {}), payloadFingerprint: agentSessionSendBodyFingerprint(sessionId, message(clientMessageId)) }) const revision = @@ -466,9 +494,23 @@ it.each(['rewind', 'replacement', 'rollover'] as const)( rows ]), ...work.writes.map(([target, rows]): [string, number] => [`${target}`, Number(rows)]), - ['draft nodes', transitions.reduce((sum, transition) => sum + transition.tracked(), 0)] + [ + 'canonical draft nodes', + transitions.reduce( + (sum, transition) => sum + transition.tracked() - transition.trackedDerived(), + 0 + ) + ] ]) - return { statements: work.statements, counts } + const derived = transitions.reduce((sum, transition) => sum + transition.trackedDerived(), 0) + // Two coordinate indexes each have an all-card and a domain AVL; a point path + // visits at most twice log2(K+2), reaching at most eight objects at each level. + expect(derived).toBeLessThanOrEqual(64 * (size + 1) * Math.ceil(Math.log2(size + 2))) + console.info( + 'replacement-draft', + JSON.stringify({ path, size, canonical: counts.get('canonical draft nodes'), derived }) + ) + return { statements: work.statements, counts, derived } } // A chat's first item and card start collections an empty chat lacks, so the line is taken // between non-empty chats: 10N more restated items cost ten times what N more do. @@ -484,7 +526,10 @@ it.each(['rewind', 'replacement', 'rollover'] as const)( grown: 10 * ((small.counts.get(counted) ?? 0) - from) }) } - expect(large.counts.get('draft nodes')).toBeGreaterThan(small.counts.get('draft nodes') ?? 0) + expect(large.counts.get('canonical draft nodes')).toBeGreaterThan( + small.counts.get('canonical draft nodes') ?? 0 + ) + expect(large.derived).toBeGreaterThan(small.derived) } ) @@ -522,8 +567,15 @@ it("opens a new chat reading each key's newest record once, the same with N and journal_outcome_epochs: 1, journal_outcomes: small.keys, journal_rows: 1, - journal_sessions: 2 + journal_sessions: 2, + other: 0 }) + // Empty legacy ownership still costs one session-scoped, position-ordered lookup per restore. + for (const { work } of [small, large]) { + expect(work.statements.filter((sql) => !/\bFROM\s+journal_\w+/.test(sql))).toStrictEqual([ + 'SELECT message_id, position, body_json, fingerprint, state, hold_reason, returned_reason, returned_rejection, consumed_as, queued_epoch, queued_sequence, options_json FROM queued_messages WHERE session_id = ? ORDER BY position ASC' + ]) + } // Each read is one seek down the key index from the last key: no scan, no sort, no revision walk. const plan = small.database.db .prepare(`EXPLAIN QUERY PLAN ${JOURNAL_NEWEST_OUTCOME_SQL}`) @@ -553,14 +605,22 @@ async function prefixChat(size: number, label: string) { it('writes to an existing chat without reading its prefix, the same at N and 10N', async () => { const small = await prefixChat(N, 'prefix-append-small') const large = await prefixChat(10 * N, 'prefix-append-large') - for (const key of ['first', 'streamed', 'accepted'] as const) { + for (const key of ['streamed', 'accepted'] as const) { expectSameWork(small[key], large[key]) } - // The first write appends the marker, then the item's record and the two scalars it changed. - expect(small.first.writes).toStrictEqual([ - ['journal_outcomes INSERT', 3], - ['journal_rows INSERT', 1] - ]) + // Conversion writes each existing identity and the one own entry once, plus the + // item's record and two scalars; every subsequent ordinary append stays point work. + for (const [size, fixture] of [ + [N, small], + [10 * N, large] + ] as const) { + expect(fixture.first.writes).toStrictEqual([ + ['journal_outcomes INSERT', size + 4], + ['journal_rows INSERT', 1] + ]) + expect(fixture.first.read.journal_rows).toBe(1) // Newly inserted marker read-back, no prefix scan. + } + expect(large.first.statements).toEqual(small.first.statements) // Prepared statements are reused for each converted key. expect(small.streamed.writes).toStrictEqual([['journal_outcomes INSERT', 1]]) expect(large.rig.state().submissions.get('send')?.dispatchState).toBe('accepted') }) @@ -587,7 +647,8 @@ it("opens an existing chat reading main's prefix rows and each tail key's newest expect(work.read.journal_rows).toBe(prefix + 2) expect(work.read.journal_outcomes).toBe(keys) } - expect(large.keys).toBe(small.keys) + // The first mandatory conversion persisted one native identity per original card. + expect(large.keys - small.keys).toBe(9 * N) }) /** The text of each keyed record an open decodes: every key's newest live record. */ @@ -613,6 +674,7 @@ it('admits each row and record an existing chat reads or writes once, at N and 1 admission.rows = 0 admission.records = 0 admission.walks = 0 + admission.tombstones = 0 const parses = vi.spyOn(JSON, 'parse') restoreJournal(database.db, sessionId) const parsed = parses.mock.calls.filter(([text]) => entries.texts.has(text)).length @@ -624,6 +686,7 @@ it('admits each row and record an existing chat reads or writes once, at N and 1 admission.rows = 0 admission.records = 0 admission.walks = 0 + admission.tombstones = 0 await rig.writer.enqueue(revision('answer', message('z'.repeat(4_096), 'assistant'), 3)) const written = { ...admission, writes: watched.take().writes } watched.stop() @@ -634,7 +697,7 @@ it('admits each row and record an existing chat reads or writes once, at N and 1 expect(open.read.journal_rows).toBe(prefix + 2) expect(open.rows).toBe(prefix) expect(open.records).toBe(keys) - expect(open.walks).toBe(open.rows + open.records) + expect(open.walks).toBe(open.rows + open.records - open.tombstones) expect(open.entries).toBeGreaterThan(0) expect(open.parsed).toBe(open.entries) expect(paged.rows).toHaveLength(100) @@ -642,7 +705,7 @@ it('admits each row and record an existing chat reads or writes once, at N and 1 // The written row's read-back, and each record the write inserts. expect(written.rows).toBe(1) expect(written.writes).toStrictEqual([['journal_outcomes INSERT', written.records]]) - expect(written.walks).toBe(written.rows + written.records) + expect(written.walks).toBe(written.rows + written.records - written.tombstones) } }) diff --git a/src/main/native-chat/agent-session-journal/journal-outcome-table.ts b/src/main/native-chat/agent-session-journal/journal-outcome-table.ts index fb5c563037b..9c667815498 100644 --- a/src/main/native-chat/agent-session-journal/journal-outcome-table.ts +++ b/src/main/native-chat/agent-session-journal/journal-outcome-table.ts @@ -1,3 +1,5 @@ +import { journalInputOrder } from './journal-input-order' +import { detachJournalNativePrefix, restoreJournalNativeState } from './journal-input-conversion' import type Database from '../../sqlite/sync-database' import type { JournalReducerState } from './journal-reducer' import { applyJournalRow } from './journal-reducer' @@ -28,12 +30,15 @@ export function readJournalOutcomes( baseline?: JournalReducerState ): JournalReducerState { const newest = db.prepare(JOURNAL_NEWEST_OUTCOME_SQL) + const prefix = detachJournalNativePrefix(baseline) const fold = new JournalOutcomeFold() let after = { field: '', key: '' } for (;;) { const row = newest.get(source.generation, after.field, after.key, source.through) if (!row) { - return fold.finish(sessionId, epoch, source.through, baseline) + const state = fold.finish(sessionId, epoch, source.through, baseline) + restoreJournalNativeState(db, state, source.format, fold, prefix) + return state } const value = fold.add(row) after = { field: value.field, key: value.key } @@ -47,10 +52,13 @@ export function persistJournalOutcomeRows( state: JournalReducerState, source: Pick, rows: readonly JournalRow[], - savedAt?: number + savedAt?: number, + upgradeNative = false, + finish?: (state: JournalReducerState) => void ): { state: JournalReducerState; adopt: () => JournalReducerState } { let scalar = state.lastSequence === 0 ? {} : journalOutcomeScalars(state) const transition = isolateJournalTransition(state, () => undefined) + journalInputOrder(transition.state).introduceOverlays(transition.state) const previous = db.prepare(`SELECT ordinal, deleted FROM journal_outcomes WHERE generation = ? AND field = ? AND entry_key = ? ORDER BY seq DESC LIMIT 1`) const insert = db.prepare(`INSERT INTO journal_outcomes @@ -61,7 +69,23 @@ export function persistJournalOutcomeRows( // unusable never reaches the model, so it is never refused by the strict record encoder. const { stored } = journalRowReadBack(state.sessionId, row) applyJournalRow(transition.state, stored, savedAt ?? row.ts) + if (row === rows.at(-1)) { + finish?.(transition.state) + } const changes = transition.changes(true) + if (upgradeNative) { + for (const path of ['inputEntries', 'inputCards'] as const) { + const changed = new Set( + changes.filter((change) => change.path === path).map((change) => change.key) + ) + for (const [key, value] of transition.state[path]) { + if (!changed.has(key)) { + changes.push({ path, key, value, deleted: false, reinsert: true }) + } + } + } + upgradeNative = false + } // Only this row's own transition is recorded, never a snapshot of a prefix value it left alone. const encoded = encodeJournalOutcomes(transition.state, changes, scalar, transition.untracked) scalar = encoded.scalars diff --git a/src/main/native-chat/agent-session-journal/journal-outcome-write.ts b/src/main/native-chat/agent-session-journal/journal-outcome-write.ts index 79959c7d1b1..ef98545d9b8 100644 --- a/src/main/native-chat/agent-session-journal/journal-outcome-write.ts +++ b/src/main/native-chat/agent-session-journal/journal-outcome-write.ts @@ -1,3 +1,6 @@ +import { journalInputAcceptance } from './journal-input-provenance' +import { copyJournalNativeState } from './journal-input-conversion' +import { journalInputOrder } from './journal-input-order' import type Database from '../../sqlite/sync-database' import { createJournalReducerState, type JournalReducerState } from './journal-reducer' import type { JournalRow } from './journal-row-schema' @@ -16,13 +19,20 @@ export function prepareJournalOutcomeWrite( rows: readonly JournalRow[], savedAt: number ): { adopt: () => JournalReducerState } { + const previous = journalOutcomeSource(db, live.sessionId, live.epoch) const source = - journalOutcomeSource(db, live.sessionId, live.epoch) ?? - startJournalOutcomeLog(db, live.sessionId, live.epoch, live.lastSequence, savedAt) + previous ?? startJournalOutcomeLog(db, live.sessionId, live.epoch, live.lastSequence, savedAt) if (source.through !== live.lastSequence) { throw new Error('journal outcome writer has stale coverage') } - const transition = persistJournalOutcomeRows(db, live, source, rows, savedAt) + const transition = persistJournalOutcomeRows( + db, + live, + source, + rows, + savedAt, + !previous || previous.format < 2 + ) advanceJournalOutcomeSource(db, live.sessionId, live.epoch, transition.state.lastSequence) return { adopt: transition.adopt } } @@ -32,15 +42,33 @@ export function publishJournalOutcomeEpoch( db: Database.Database, sessionId: string, epoch: string, - rows: readonly JournalRow[] + rows: readonly JournalRow[], + previous?: JournalReducerState ): { adopt: () => JournalReducerState } { const first = rows[0] if (!first || first.seq !== 1 || first.kind !== 'epoch') { throw new Error('missing founding epoch') } const source = startJournalOutcomeLog(db, sessionId, epoch, 0, first.ts) - const initial = createJournalReducerState(sessionId, epoch) - const transition = persistJournalOutcomeRows(db, initial, source, rows) + const initial = createJournalReducerState(sessionId, epoch, (id) => + journalInputAcceptance(db, sessionId, id) + ) + if (previous) { + copyJournalNativeState(previous, initial) + } + journalInputOrder(initial).beginReconstruction() + const transition = persistJournalOutcomeRows( + db, + initial, + source, + rows, + undefined, + true, + (state) => { + journalInputOrder(state).restore(state) + journalInputOrder(state).finishReconstruction(state) + } + ) advanceJournalOutcomeSource(db, sessionId, epoch, transition.state.lastSequence) return { adopt: transition.adopt } } diff --git a/src/main/native-chat/agent-session-journal/journal-projection-bootstrap.test.ts b/src/main/native-chat/agent-session-journal/journal-projection-bootstrap.test.ts index 397f0ba2dec..73df7e3e238 100644 --- a/src/main/native-chat/agent-session-journal/journal-projection-bootstrap.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-projection-bootstrap.test.ts @@ -1,3 +1,5 @@ +import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' +import { journalInputOrder } from './journal-input-order' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -171,3 +173,155 @@ it('refuses to publish a new epoch over one the caller did not see, writing noth expect(database.db.prepare('SELECT * FROM journal_rows ORDER BY seq').all()).toEqual(before) expect(restoreJournal(database.db, sessionId)?.state.epoch).toBe(epoch) }) + +// A frozen marker-5 prefix and an outcome-only format-1 chat carry different evidence. +it.each(['prefix', 'zero', 'recreated'] as const)( + 'lifts actual %s evidence once, then treats native tombstones as authoritative', + async (path) => { + const user = { + kind: 'message' as const, + role: 'user' as const, + blocks: [{ type: 'text' as const, text: 'M' }] + } + const rows: JournalRow[] = [ + projectionEpochRow, + ...['C', 'D'].map((id, i): JournalRow => ({ + ...base(i + 2), + kind: 'queue', + messageId: id, + change: { + op: 'queued', + body: id === 'C' ? { ...user, command: { name: 'compact' } } : user, + position: i, + queuedAt: { epoch, sequence: i + 2 } + } + })), + { + ...base(4), + kind: 'submission', + clientMessageId: 'S', + body: user, + payloadFingerprint: 'S', + providerHandle: { kind: 'codex', threadId: 'thread-1' }, + handoverRecorded: true + }, + { + ...base(5), + kind: 'dispatch', + clientMessageId: 'S', + state: 'accepted', + providerItemId: null, + reason: null + }, + { + ...base(6), + kind: 'submission', + clientMessageId: 'M', + body: user, + payloadFingerprint: 'M', + providerHandle: { kind: 'codex', threadId: 'thread-1' }, + handoverRecorded: true, + inputOrderContext: { source: { epoch, sequence: 6 }, basis: 'observed', cut: 6 } + } + ] + if (path !== 'zero') { + seedProjectionRows(database, rows) + } else { + seedOutcomeRows(database, rows) + } + let state = restoreJournal(database.db, sessionId)!.state + const first = projectionWriter(database, state) + rigs.push(first) + await first.writer.enqueue((seq, ts): JournalRow => ({ + ...base(seq), + ts, + kind: 'queue', + messageId: 'D', + change: { op: 'edited', body: { ...user, blocks: [{ type: 'text', text: 'latest D' }] } } + })) + if (path === 'recreated') { + await first.writer.enqueue((seq): JournalRow => ({ + ...base(seq), + kind: 'tombstone', + itemId: agentJournalSubmissionKey('M'), + revision: seq, + retireSubmission: true + })) + await first.writer.enqueue((seq): JournalRow => ({ + ...base(seq), + kind: 'submission', + clientMessageId: 'M', + body: user, + payloadFingerprint: 'M-recreated', + providerHandle: { kind: 'codex', threadId: 'thread-1' }, + handoverRecorded: true + })) + } + // These ordinary records are admitted by the frozen format-1 contract; old writers had no native families. + database.db.exec( + "DELETE FROM journal_outcomes WHERE field IN ('inputEntries','inputCards'); UPDATE journal_outcomes SET format=1; UPDATE journal_outcome_epochs SET format=1" + ) + const prefix = liveTestJournalRows(database.db, sessionId) + const restored = restoreJournal(database.db, sessionId)! + expect(restored).toMatchObject({ newer: null, damage: null }) + state = restored.state + expect(state.queue.cards.get('D')?.body.blocks).toEqual([{ type: 'text', text: 'latest D' }]) + expect(journalInputOrder(state).firstQueued()?.clientMessageId).toBe('M') + expect(journalInputOrder(state).forced('M')).toBe(path === 'prefix') + expect(state.inputEntries.get('M')?.context.basis).toBe( + path === 'prefix' ? 'observed' : 'legacy' + ) + if (path === 'prefix') { + expect( + ['C', 'D'].map((id) => + journalInputOrder(state).precedes(state.queue.cards.get(id)!, state.submissions.get('M')!) + ) + ).toEqual([true, true]) + } + const rig = projectionWriter(database, state) + rigs.push(rig) + database.db.exec( + "CREATE TRIGGER no_upgrade BEFORE UPDATE ON journal_outcome_epochs BEGIN SELECT RAISE(ABORT, 'upgrade failed'); END" + ) + await expect(rig.writer.enqueue((seq) => item(seq, 'upgrade'))).rejects.toThrow( + 'upgrade failed' + ) + database.db.exec('DROP TRIGGER no_upgrade') + expect( + database.db + .prepare( + "SELECT COUNT(*) AS n FROM journal_outcomes WHERE field IN ('inputEntries','inputCards')" + ) + .get()?.n + ).toBe(0) + expect(rig.state().items.has('upgrade')).toBe(false) + expect(liveTestJournalRows(database.db, sessionId)).toEqual(prefix) + await rig.writer.enqueue((seq) => item(seq, 'upgrade')) + expect(database.db.prepare('SELECT format FROM journal_outcome_epochs').get()?.format).toBe(2) + expectProjectionMatches(database, rig.state()) + const native = database.db + .prepare( + "SELECT field, entry_key FROM journal_outcomes WHERE field IN ('inputEntries','inputCards')" + ) + .all() + expect(native).toHaveLength(3) + expect(native).toContainEqual({ field: 'inputEntries', entry_key: '"M"' }) + expect(liveTestJournalRows(database.db, sessionId)).toEqual(prefix) + await rig.writer.enqueue((seq): JournalRow => ({ + ...base(seq), + kind: 'dispatch', + clientMessageId: 'M', + state: 'accepted', + providerItemId: null, + reason: null + })) + expectProjectionMatches(database, rig.state()) + expect(restoreJournal(database.db, sessionId)!.state.inputEntries.size).toBe(0) + expect( + journalInputOrder(restoreJournal(database.db, sessionId)!.state).firstQueued() + ).toBeUndefined() + // A later unrelated write cannot revive the old prefix's own component. + await rig.writer.enqueue((seq) => item(seq, 'after-retirement')) + expect(restoreJournal(database.db, sessionId)!.state.inputEntries.size).toBe(0) + } +) diff --git a/src/main/native-chat/agent-session-journal/journal-projection-failure.test.ts b/src/main/native-chat/agent-session-journal/journal-projection-failure.test.ts index f1d7d36c930..61468dc7e46 100644 --- a/src/main/native-chat/agent-session-journal/journal-projection-failure.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-projection-failure.test.ts @@ -1,3 +1,4 @@ +import { readJournalLegacyCards } from './journal-queue-import' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -8,6 +9,7 @@ import { closeTestJournalHostDatabases, openTestJournalHostDatabase } from './journal-host-database-test-support' +import { journalInputOrder } from './journal-input-order' import { encodeJournalState } from './journal-state-codec' import { JournalStopMarks } from './journal-stop-marks' import type { JournalHostDatabase } from './journal-host-database' @@ -54,6 +56,26 @@ it('adopts nothing on an outcome, coverage, receipt, outer-abort or COMMIT failu itemId: 'stop', revision: 1, stopEvent: { reason: 'user-stop', at: 1_002 } + }, + ...['C', 'D'].map((id, i): JournalRow => ({ + ...base(i + 3), + kind: 'queue', + messageId: id, + change: { + op: 'queued', + body: id === 'C' ? { ...user, command: { name: 'compact' } } : user, + position: i, + queuedAt: { epoch, sequence: i + 3 } + } + })), + { + ...base(5), + kind: 'submission', + clientMessageId: 'legacy', + body: user, + handoverRecorded: true, + providerHandle: { kind: 'codex', threadId: 'thread-1' }, + payloadFingerprint: 'legacy' } ]) rig = projectionWriter(database, state) @@ -68,6 +90,12 @@ it('adopts nothing on an outcome, coverage, receipt, outer-abort or COMMIT failu kind: 'submission', clientMessageId: id, body: user, + handoverRecorded: true, + inputOrderContext: { + source: { epoch, sequence: seq }, + basis: 'observed', + cut: journalInputOrder(rig!.state()).lineage.at(seq) + }, providerHandle: { kind: 'codex', threadId: 'thread-1' }, payloadFingerprint: agentSessionSendBodyFingerprint(sessionId, user) }) @@ -78,13 +106,57 @@ it('adopts nothing on an outcome, coverage, receipt, outer-abort or COMMIT failu }, committed: vi.fn() } - await expect(writer.enqueue(send('preserved'), receipt)).resolves.toMatchObject({ seq: 3 }) + await expect(writer.enqueue(send('preserved'), receipt)).resolves.toMatchObject({ seq: 6 }) expect(receipt.committed).toHaveBeenCalledOnce() expect(rig.state().submissions.has('preserved')).toBe(true) expect(stops.latest()?.settle).toBe(settle) expectProjectionMatches(database, rig.state()) + database.db + .prepare(`INSERT INTO queued_messages + (session_id,message_id,position,body_json,fingerprint,created_at,host_instance,state) + VALUES (?,?,?,?,?,0,'legacy','waiting')`) + .run(sessionId, 'overlay', 2, JSON.stringify(user), 'overlay') + const beforeRead = encodeJournalState(rig.state()) + journalInputOrder(rig.state()).overlay( + rig.state(), + readJournalLegacyCards(database.db, rig.state()) + ) + expect(encodeJournalState(rig.state())).toEqual(beforeRead) const saved = encodeJournalState(rig.state()) const live = rig.state() + const version = journalInputOrder(live).version() + const clock = journalInputOrder(live).lineage.clock + const pending = journalInputOrder(live) + .unsettledSubmissions() + .map((entry) => entry.clientMessageId) + const attempt = (id: string) => + writer.enqueueRows( + () => [ + (seq, ts): JournalRow => ({ + ...base(seq), + ts, + kind: 'dispatch', + clientMessageId: 'legacy', + state: 'accepted', + providerItemId: null, + reason: null + }), + send(id) + ], + receipt + ) + const unchanged = () => { + expect(live.inputCards.has('overlay')).toBe(false) + expect(journalInputOrder(live).version()).toBe(version) + expect(journalInputOrder(live).lineage.clock).toBe(clock) + expect( + journalInputOrder(live) + .unsettledSubmissions() + .map((entry) => entry.clientMessageId) + ).toEqual(pending) + expect(live.inputEntries.has('legacy')).toBe(true) + expect(live.inputCards.get('D')?.baseline).toBeDefined() + } const scenarios = [ [ 'outcome', @@ -105,8 +177,9 @@ it('adopts nothing on an outcome, coverage, receipt, outer-abort or COMMIT failu ] as const for (const [name, sql] of scenarios) { database.db.exec(sql) - await expect(writer.enqueue(send(name), receipt)).rejects.toThrow('failure') + await expect(attempt(name)).rejects.toThrow('failure') database.db.exec('DROP TRIGGER failure') + unchanged() expect(rig.state()).toBe(live) expect(encodeJournalState(live)).toEqual(saved) expect(receipt.committed).toHaveBeenCalledOnce() @@ -120,14 +193,32 @@ it('adopts nothing on an outcome, coverage, receipt, outer-abort or COMMIT failu } exec(sql) }) - await expect(writer.enqueue(send('commit'), receipt)).rejects.toThrow('injected boundary failure') + await expect(attempt('commit')).rejects.toThrow('injected boundary failure') spy.mockRestore() + unchanged() expect(rig.state()).toBe(live) expect(encodeJournalState(live)).toEqual(saved) expect(stops.latest()?.settle).toBe(settle) expect(receipt.committed).toHaveBeenCalledOnce() expect(database.db.prepare('SELECT COUNT(*) AS n FROM test_receipts').get()?.n).toBe(1) expectProjectionMatches(database, live) + await attempt('retry') + expect(live.inputEntries.has('legacy')).toBe(false) + expect(live.inputEntries.has('retry')).toBe(true) + expect(live.inputCards.has('overlay')).toBe(true) + const overlayWrites = database.db + .prepare( + "SELECT seq,value_json FROM journal_outcomes WHERE field='inputCards' AND entry_key=? AND deleted=0 ORDER BY seq" + ) + .all(JSON.stringify('overlay')) + // Both rows touch the introduced object after the legacy-baseline cleanup in the first row. + expect(overlayWrites.map((row) => row.seq)).toEqual([live.lastSequence - 1, live.lastSequence]) + expect(overlayWrites.map((row) => row.value_json)).toEqual( + Array(2).fill(JSON.stringify(['overlay', live.inputCards.get('overlay')])) + ) + + expect(receipt.committed).toHaveBeenCalledTimes(2) + expectProjectionMatches(database, live) const first = rig.state().lastSequence + 1 await writer.enqueueRows(() => [ (seq, ts) => ({ diff --git a/src/main/native-chat/agent-session-journal/journal-queue-fold.ts b/src/main/native-chat/agent-session-journal/journal-queue-fold.ts index bfe70ab58c8..7acf8850d16 100644 --- a/src/main/native-chat/agent-session-journal/journal-queue-fold.ts +++ b/src/main/native-chat/agent-session-journal/journal-queue-fold.ts @@ -1,3 +1,4 @@ +import { journalInputOrder } from './journal-input-order' // Latch handoff outcomes before rewind can retire their rows; requeue only proven-undelivered sends. import type { @@ -210,8 +211,10 @@ function spendReportedCommandCards(state: JournalReducerState): void { function setCard(state: JournalReducerState, card: QueuedMessageRow): void { if (state.queue.cards.get(card.messageId) !== card) { + const prior = state.queue.cards.get(card.messageId) state.queue.cards.set(card.messageId, card) state.queue.revision += 1 + journalInputOrder(state).queueChanged(state, prior, card) } } diff --git a/src/main/native-chat/agent-session-journal/journal-queue-import.ts b/src/main/native-chat/agent-session-journal/journal-queue-import.ts index 9cd3a8b6197..a13f537667b 100644 --- a/src/main/native-chat/agent-session-journal/journal-queue-import.ts +++ b/src/main/native-chat/agent-session-journal/journal-queue-import.ts @@ -1,3 +1,5 @@ +import type Database from '../../sqlite/sync-database' +import { journalInputOrder } from './journal-input-order' // Adopt legacy cards and delete their rows atomically, using the same fold before and after import. // Retry failed bulk import on next open; do not loop while the host is running. @@ -61,9 +63,9 @@ function unconfirmedHandOff( | { attempt: string; holdReason?: undefined } | { attempt?: undefined; holdReason: string } | null { - const attempts = [...state.submissions.values()] - .filter((submission) => submission.queuedMessageId === row.messageId) - .sort((a, b) => (a.submittedSequence ?? 0) - (b.submittedSequence ?? 0)) + const attempts = [...journalInputOrder(state).linkedTo(row.messageId)].sort( + (a, b) => (a.submittedSequence ?? 0) - (b.submittedSequence ?? 0) + ) if (!attempts.some(handOffInDoubt)) { return null } @@ -95,7 +97,10 @@ export class LegacyQueuedCards { owner: JournalReducerState key: string cards: readonly QueuedMessageRow[] + byId: ReadonlyMap display?: JournalReducerState + displayThrough?: number + handedOff: readonly AgentJournalSubmission[] } | null = null constructor(private readonly deps: LegacyDeps) {} @@ -105,8 +110,8 @@ export class LegacyQueuedCards { if (this.rows().length === 0) { return '' } - const { epoch, lastSequence } = this.deps.state() - return `${this.revision}:${epoch}:${lastSequence}` + const state = this.deps.state() + return `${this.revision}:${state.epoch}:${journalInputOrder(state).linkedVersion()}` } cards(): readonly QueuedMessageRow[] { @@ -121,23 +126,38 @@ export class LegacyQueuedCards { const change = legacyQueuedChange(state, row) return change ? [queuedCardFromChange(state, row.messageId, change)] : [] }) - this.viewed = { owner: state, key, cards } + const handedOff = cards.flatMap((card) => { + const submission = + card.consumedAs === null ? undefined : state.submissions.get(card.consumedAs) + return submission && handOffInDoubt(submission) ? [submission] : [] + }) + this.viewed = { + owner: state, + key, + cards, + handedOff, + byId: new Map(cards.map((card) => [card.messageId, card])) + } } return this.viewed.cards } + get(messageId: string): QueuedMessageRow | undefined { + this.cards() + return this.viewed?.byId.get(messageId) + } + // Failed adoption cannot hide spent sends; derive display changes only on copied state. displayState(): JournalReducerState { const state = this.deps.state() - const cards = this.cards() - if (this.viewed?.display) { + this.cards() + if (this.viewed?.display && this.viewed.displayThrough === state.lastSequence) { return this.viewed.display } - const handedOff = cards.flatMap((card) => { - const submission = - card.consumedAs === null ? undefined : state.submissions.get(card.consumedAs) - return submission && handOffInDoubt(submission) ? [{ ...submission }] : [] - }) + const handedOff = this.viewed?.handedOff ?? [] + if (this.viewed) { + this.viewed.displayThrough = state.lastSequence + } if (handedOff.length === 0) { if (this.viewed) { this.viewed.display = state @@ -150,7 +170,8 @@ export class LegacyQueuedCards { items: new Map(state.items), submissions: new Map(state.submissions) } - for (const submission of handedOff) { + for (const entry of handedOff) { + const submission = { ...entry } display.submissions.set(submission.clientMessageId, submission) foldQueuedHandOffInDoubt(display, submission) } @@ -238,12 +259,25 @@ export class LegacyQueuedCards { } private read(): QueuedMessageRow[] { - return this.deps - .database() - .db.prepare( - `SELECT ${LEGACY_QUEUED_MESSAGE_COLUMNS} FROM queued_messages WHERE session_id = ? ORDER BY position ASC` - ) - .all(this.deps.sessionId) - .flatMap((row) => readStoredQueuedMessageRow(row) ?? []) + return readLegacyRows(this.deps.database().db, this.deps.sessionId) } } + +export function readJournalLegacyCards( + db: Database.Database, + state: JournalReducerState +): QueuedMessageRow[] { + return readLegacyRows(db, state.sessionId).flatMap((row) => { + const change = legacyQueuedChange(state, row) + return change ? [queuedCardFromChange(state, row.messageId, change)] : [] + }) +} + +function readLegacyRows(db: Database.Database, sessionId: string): QueuedMessageRow[] { + return db + .prepare( + `SELECT ${LEGACY_QUEUED_MESSAGE_COLUMNS} FROM queued_messages WHERE session_id = ? ORDER BY position ASC` + ) + .all(sessionId) + .flatMap((row) => readStoredQueuedMessageRow(row) ?? []) +} diff --git a/src/main/native-chat/agent-session-journal/journal-queued-messages.ts b/src/main/native-chat/agent-session-journal/journal-queued-messages.ts index 4cb3c6b828b..38ecbb8ef51 100644 --- a/src/main/native-chat/agent-session-journal/journal-queued-messages.ts +++ b/src/main/native-chat/agent-session-journal/journal-queued-messages.ts @@ -1,3 +1,4 @@ +import { journalInputOrder } from './journal-input-order' import { journalQueueAdoptionPlan } from './journal-queue-adoption-plan' import { journalAttachmentClaimReceipt, @@ -20,7 +21,6 @@ import type { JournalQueuedMessagesDeps, JournalSubmissionConsume } from './jour import { deriveQueuePauses, journalUserStopInForce, - nextSendableQueuedCard, type DerivedQueuePause, type JournalQueuePauseMarks } from './queued-message-pause' @@ -46,7 +46,11 @@ export type JournalQueueRestatement = { } export class JournalQueuedMessages { - readonly editLeases = new QueuedMessageEditLeases(() => this.list()) + readonly editLeases = new QueuedMessageEditLeases( + () => this.list(), + undefined, + (id) => this.get(id) + ) private readonly legacy: LegacyQueuedCards private viewed: { key: string; cards: readonly QueuedMessageRow[] } | null = null @@ -58,7 +62,13 @@ export class JournalQueuedMessages { return this.deps.sessionId } + inputState(): JournalReducerState { + const state = this.deps.state() + journalInputOrder(state).overlay(state, this.legacy.cards()) + return state + } displayState(): JournalReducerState { + this.inputState() return this.legacy.displayState() } @@ -77,11 +87,20 @@ export class JournalQueuedMessages { } get(messageId: string): QueuedMessageRow | null { - return ( - this.deps.state().queue.cards.get(messageId) ?? - this.legacy.cards().find((card) => card.messageId === messageId) ?? - null - ) + return this.deps.state().queue.cards.get(messageId) ?? this.legacy.get(messageId) ?? null + } + + /** Selection reads identity summaries; current values are resolved after edits. */ + nextInputCard(automatic = true): QueuedMessageRow | null { + const state = this.inputState() + const identity = journalInputOrder(state).head(state, this.deps.reopenFloor()) + const card = identity ? this.get(identity.messageId) : null + return !card || + card.state === 'returned' || + journalInputOrder(state).paused(state, card, this.deps.reopenFloor()) || + (automatic && this.editLeases.isHeld(card.messageId)) + ? null + : card } // Card insertion, attachment claims and send receipt must commit together. @@ -210,9 +229,7 @@ export class JournalQueuedMessages { card.fingerprint === input.fingerprint && input.consumedAs !== input.messageId if (consumable && input.yieldsToPause) { - const cards = this.list() - const held = this.editLeases.heldIds(cards) - const next = nextSendableQueuedCard(this.derivePauses(cards), cards, held) + const next = this.nextInputCard() if (next?.messageId === input.messageId) { return } diff --git a/src/main/native-chat/agent-session-journal/journal-reducer-transition.ts b/src/main/native-chat/agent-session-journal/journal-reducer-transition.ts index ca9ce22b4f8..f12207624d0 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer-transition.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer-transition.ts @@ -1,3 +1,4 @@ +import { journalInputOrder, setJournalInputOrder } from './journal-input-order' // Track writes through objects, iterators and collection entries; finalize only after COMMIT. import type { JournalReducerState } from './journal-reducer' @@ -9,6 +10,7 @@ import { } from './journal-transition-collections' type DraftNode = { + original: object value: object /** What the draft holds now: the copy once written, else the object itself. */ current: () => object @@ -16,16 +18,24 @@ type DraftNode = { listen: (changed: () => void) => void } +function draftProperty(value: Value, key: PropertyKey): unknown { + const descriptor = Reflect.getOwnPropertyDescriptor(value, key) + return descriptor && ('value' in descriptor ? descriptor.value : descriptor.get?.call(value)) +} + class JournalDraft { private readonly originals = new WeakMap() private readonly proxies = new WeakMap() private readonly collections: (JournalDraftMap | JournalDraftSet)[] = [] + private readonly derived: object[] = [] private readonly assigned = new WeakSet() private readonly pending: (() => void)[] = [] private readonly resolved = new WeakMap() /** Objects and collections the fold reached; scaling tests pin how this grows with chat size. */ nodes = 0 + derivedNodes = 0 + private notifications: Set<() => void> | undefined readonly access: JournalDraftAccess = { wrap: (value, changed, path) => this.wrap(value, changed, path), @@ -43,29 +53,37 @@ class JournalDraft { return value } const node = this.proxies.get(value) ?? this.originals.get(value) ?? this.track(value, path) - node.listen(changed) + if (!path.startsWith('@inputOrder')) { + node.listen(changed) + } // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a node's draft (overlay collection or copy-on-write proxy) has the interface of the object it isolates. return node.value as T } private track(value: T, path: string): DraftNode { this.nodes += 1 + if (path.startsWith('@inputOrder')) { + this.derivedNodes++ + } const listeners = new Set<() => void>() let dirty = false - let notifying = false const mark = (): void => { dirty = true - if (notifying) { + const outer = this.notifications === undefined + const wave = (this.notifications ??= new Set()) + // Rotated/shared index nodes can reach the same ancestor by several paths. + if (wave.has(mark)) { return } - // A batch consumes changes after each row; later mutations must notify again. - notifying = true + wave.add(mark) try { for (const notify of listeners) { notify() } } finally { - notifying = false + if (outer) { + this.notifications = undefined + } } } const listen = (notify: () => void): void => { @@ -82,6 +100,7 @@ class JournalDraft { : new JournalDraftSet(value, this.access, mark, path) this.collections.push(collection) node = { + original: value, value: collection, current: () => collection, finish: () => collection.finish(), @@ -143,6 +162,7 @@ class JournalDraft { } }) node = { + original: value, value: proxy, current, finish: () => { @@ -152,7 +172,7 @@ class JournalDraft { const result = writable() const keys = Reflect.ownKeys(result) for (const key of keys) { - const child: unknown = Reflect.getOwnPropertyDescriptor(result, key)?.value + const child: unknown = draftProperty(result, key) if (assignedKeys.has(key) || this.assigned.has(value) || this.tracks(child)) { Reflect.set(result, key, this.finish(child)) } @@ -168,13 +188,16 @@ class JournalDraft { } } for (const key of keys) { - Reflect.set(value, key, Reflect.getOwnPropertyDescriptor(result, key)?.value) + Reflect.set(value, key, draftProperty(result, key)) } return value }, listen } } + if (path.startsWith('@inputOrder')) { + this.derived.push(node.value) + } this.originals.set(value, node) this.proxies.set(node.value, node) return node @@ -195,6 +218,10 @@ class JournalDraft { adopt(value: T): T { const result = this.resolve(value) + // Derived nodes adopt their own point changes; rotated ancestors do not own those writes. + for (const node of this.derived) { + this.resolve(node) + } for (const apply of this.pending) { apply() } @@ -224,6 +251,10 @@ class JournalDraft { } const node = this.proxies.get(value) ?? this.originals.get(value) if (node) { + // The derived owner points back to canonical state and shares index entries. + // Resolve that identity before children so a committed graph is finalized once. + this.resolved.set(value, node.original) + this.resolved.set(node.value, node.original) const result = node.finish() this.resolved.set(value, result) this.resolved.set(node.value, result) @@ -235,7 +266,7 @@ class JournalDraft { } // Values created during the fold may hold drafts introduced by spread or destructuring. for (const key of Reflect.ownKeys(value)) { - const child: unknown = Reflect.getOwnPropertyDescriptor(value, key)?.value + const child: unknown = draftProperty(value, key) const finished = this.finish(child) if (finished !== child) { Reflect.set(value, key, finished) @@ -257,16 +288,28 @@ export function isolateJournalTransition( untracked: (key: string, value: unknown) => unknown adopt: () => JournalReducerState tracked: () => number + trackedDerived: () => number } { const draft = new JournalDraft() const candidate = draft.wrap(state, () => undefined, '') + // Register canonical maps before the facade can reach them through an isolated query. + void candidate.inputEntries + void candidate.inputCards + const order = draft.wrap(journalInputOrder(state), () => undefined, '@inputOrder') + setJournalInputOrder(candidate, order) const result = fold(candidate) return { state: candidate, result, - changes: (reset) => draft.changes(reset), + changes: (reset) => + draft.changes(reset).filter((entry) => !entry.path.startsWith('@inputOrder')), untracked: (_key, value) => draft.untracked(value), - adopt: () => draft.adopt(candidate), + adopt: () => { + const adopted = draft.adopt({ state: candidate, order }) + setJournalInputOrder(adopted.state, adopted.order) + return adopted.state + }, + trackedDerived: () => draft.derivedNodes, tracked: () => draft.nodes } } diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.test.ts b/src/main/native-chat/agent-session-journal/journal-reducer.test.ts index 25a0e192027..4422fd1ae3d 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.test.ts @@ -1,3 +1,4 @@ +import { journalInputOrder } from './journal-input-order' import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' import { describe, expect, it } from 'vitest' import { @@ -48,6 +49,23 @@ function fold(rows: JournalRow[]): JournalReducerState { const state = createJournalReducerState('session-1', EPOCH) for (const row of rows) { applyJournalRow(state, row) + expect( + journalInputOrder(state) + .unsettledSubmissions() + .map((entry) => entry.clientMessageId) + .sort() + ).toEqual( + [...state.submissions.values()] + .filter((entry) => entry.dispatchState === 'pending' || entry.dispatchState === 'unknown') + .map((entry) => entry.clientMessageId) + .sort() + ) + for (const entry of state.submissions.values()) { + if (entry.dispatchState !== 'pending') { + expect(journalInputOrder(state).forced(entry.clientMessageId)).toBe(false) + expect(journalInputOrder(state).place(entry.clientMessageId)).toBeUndefined() + } + } } return state } @@ -735,3 +753,34 @@ describe('re-adding a tombstoned row', () => { expect(renderJournalState(state).items).toEqual([]) }) }) + +it('retires unknown execution debt with the submission tombstone', () => { + const state = fold([ + { + kind: 'submission', + clientMessageId: 'retired', + providerHandle: { kind: 'codex', threadId: 'thread-1' }, + payloadFingerprint: 'retired', + body: userText('retired'), + handoverRecorded: true, + ...base(1) + }, + { + kind: 'dispatch', + clientMessageId: 'retired', + state: 'unknown', + providerItemId: null, + reason: null, + ...base(2) + }, + { + kind: 'tombstone', + itemId: agentJournalSubmissionKey('retired'), + revision: 1, + retireSubmission: true, + ...base(3) + } + ]) + expect(state.submissions.has('retired')).toBe(false) + expect(journalInputOrder(state).unsettledSubmissions()).toEqual([]) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index f11b47d99d4..01e107fcb37 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -1,4 +1,10 @@ import { journalItemDeclaresEcho } from './journal-item-echo' +import { + journalInputOrder, + initializeJournalInputOrder, + type JournalInputOrder +} from './journal-input-order' +import type { JournalInputEntry, JournalInputCard } from './journal-input-lineage' // Live append and replay share this fold. Revisions change content, preserving creation position // and unspecified producer linkage; queued/rejected submissions use their handoff/settlement positions. @@ -79,11 +85,17 @@ export type JournalReducerState = { normalizedHandOffs: Set /** The queued messages, folded from their own rows and their hand-offs (`journal-queue-fold.ts`). */ queue: JournalQueueState + inputEntries: Map + inputCards: Map historyOrigin: JournalHistoryOrigin | null } -export function createJournalReducerState(sessionId: string, epoch: string): JournalReducerState { - return { +export function createJournalReducerState( + sessionId: string, + epoch: string, + originalAcceptance?: ConstructorParameters[1] +): JournalReducerState { + const state: JournalReducerState = { sessionId, epoch, lastSequence: 0, @@ -108,8 +120,12 @@ export function createJournalReducerState(sessionId: string, epoch: string): Jou queuedHandOffPlaces: new Map(), normalizedHandOffs: new Set(), queue: createJournalQueueState(), + inputEntries: new Map(), + inputCards: new Map(), historyOrigin: null } + initializeJournalInputOrder(state, originalAcceptance) + return state } export function applyJournalRow( @@ -117,8 +133,10 @@ export function applyJournalRow( row: JournalRow, savedAt = row.ts ): void { + journalInputOrder(state).beforeRow(state, row) applyJournalRowFacts(state, row, savedAt) foldJournalQueueRow(state, row) + journalInputOrder(state).afterRow(state, row) } function applyJournalRowFacts(state: JournalReducerState, row: JournalRow, savedAt: number): void { diff --git a/src/main/native-chat/agent-session-journal/journal-row-builders.ts b/src/main/native-chat/agent-session-journal/journal-row-builders.ts index 16267bfab19..e69c34cd58a 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-builders.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-builders.ts @@ -1,16 +1,16 @@ +import { journalInputOrder } from './journal-input-order' +import type { JournalInputContext } from './journal-input-lineage' import { journalItemEchoFields, type JournalItemEcho } from './journal-item-echo' import type { AgentJournalSubmission, AgentJournalItemBody, AgentJournalItemIdentity, - AgentJournalMessageItem, AgentJournalProducerLinkage, AgentJournalRowAttribution, AgentJournalTurnScope, AgentSessionJournalIdentity, AgentSessionJournalProviderHandle } from '../../../shared/agent-session-journal-types' -import type { AgentSessionExecutionOptions } from '../../../shared/agent-session-execution-options' import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' import { agentJournalLinkageFields } from '../../../shared/agent-session-journal-producer' @@ -37,7 +37,7 @@ import { import { assertSubmissionIdUnused } from './journal-write-guards' import { turnEndAfterStop } from './journal-stop-turn-end' import { nextJournalItemRevision } from './journal-item-revision' -import type { ResolveDispatchInput } from './journal-store-contracts' +import type { ResolveDispatchInput, JournalSubmissionInput } from './journal-store-contracts' import { journalLifecycleMutationItemId, journalLifecycleMutationRow, @@ -99,16 +99,7 @@ export function journalTombstoneRowBuilder( export function journalSubmissionRowBuilder( state: () => JournalReducerState, identity: Pick, - input: { - clientMessageId: string - payloadFingerprint: string - body: AgentJournalMessageItem - fence: number - handoverRecorded?: true - queuedMessageId?: string - origin?: 'client' | 'host' - options?: AgentSessionExecutionOptions - }, + input: JournalSubmissionInput, /** Stamp queue links from consumption here so every handoff preserves them. */ consume?: { messageId: string } ): RowBuilder { @@ -120,6 +111,11 @@ export function journalSubmissionRowBuilder( const queuedMessageId = consume?.messageId ?? input.queuedMessageId return buildJournalSubmissionRow({ state: state(), + inputOrderContext: { + source: { epoch: state().epoch, sequence: seq }, + basis: 'observed', + cut: journalInputOrder(state()).lineage.at(seq) + }, providerHandle: agentSessionJournalProviderHandle(identity), ...input, ...(queuedMessageId !== undefined ? { queuedMessageId } : {}), @@ -286,23 +282,19 @@ export function buildJournalTombstoneRow(input: { } } -export function buildJournalSubmissionRow(input: { - state: JournalReducerState - clientMessageId: string - payloadFingerprint: string - providerHandle: AgentSessionJournalProviderHandle - body: AgentJournalMessageItem - seq: number - fence: number - ts: number - handoverRecorded?: true - queuedMessageId?: string - origin?: 'client' | 'host' - source?: AgentJournalSubmission['source'] - options?: AgentSessionExecutionOptions -}): JournalSubmissionRow { +export function buildJournalSubmissionRow( + input: Omit & + Pick & { + state: JournalReducerState + providerHandle: AgentSessionJournalProviderHandle + seq: number + ts: number + inputOrderContext?: JournalInputContext + } +): JournalSubmissionRow { return { kind: 'submission', + ...(input.inputOrderContext ? { inputOrderContext: input.inputOrderContext } : {}), clientMessageId: input.clientMessageId, payloadFingerprint: input.payloadFingerprint, providerHandle: input.providerHandle, diff --git a/src/main/native-chat/agent-session-journal/journal-row-encoding.ts b/src/main/native-chat/agent-session-journal/journal-row-encoding.ts new file mode 100644 index 00000000000..5437eeb69b5 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-row-encoding.ts @@ -0,0 +1,9 @@ +import type { JournalRow } from './journal-row-schema' + +export function serializeJournalRow(row: JournalRow): string { + return JSON.stringify(row) +} + +export function journalRowByteLength(row: JournalRow): number { + return Buffer.byteLength(serializeJournalRow(row), 'utf8') + 1 +} diff --git a/src/main/native-chat/agent-session-journal/journal-row-schema.test.ts b/src/main/native-chat/agent-session-journal/journal-row-schema.test.ts index 83c2dbd5f37..b69645093e2 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-schema.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-schema.test.ts @@ -559,3 +559,66 @@ it.each([{ dispatchId: '' }, { dispatchId: null }, { task: null }, { task: 17 }] ).toBe(false) } ) + +it('keeps ordering context additive and treats malformed optional facts as legacy annotations', () => { + const row = { + ...BASE, + kind: 'submission', + clientMessageId: 'context', + payloadFingerprint: 'context', + providerHandle: { kind: 'codex', threadId: 't' }, + body: { kind: 'message', role: 'user', blocks: [] }, + inputOrderContext: { source: { epoch: 'source', sequence: 2 }, basis: 'observed', cut: 2 } + } + expect(parseJournalRow(JSON.stringify(row))).toMatchObject({ + ok: true, + row: { inputOrderContext: row.inputOrderContext, body: row.body } + }) + expect( + parseJournalRow( + JSON.stringify({ + ...row, + inputOrderContext: { ...row.inputOrderContext, basis: 'unreadable' } + }) + ) + ).toEqual({ + ok: true, + row: expect.not.objectContaining({ inputOrderContext: expect.anything() }) + }) + const epoch = { + ...BASE, + kind: 'epoch', + reason: 'unreconcilable_prefix', + providerHandle: row.providerHandle, + inputOrderLineage: { + clock: 10, + cards: [{ messageId: 'C', introduced: 1, baseline: { coordinate: null } }] + } + } + expect(parseJournalRow(JSON.stringify(epoch))).toMatchObject({ + ok: true, + row: { inputOrderLineage: epoch.inputOrderLineage } + }) + const overlapping = { + ...epoch, + inputOrderLineage: { + clock: 10, + cards: [ + { + messageId: 'C', + introduced: 1, + baseline: { coordinate: { epoch: 'old', sequence: -1 } } + } + ] + } + } + expect(parseJournalRow(JSON.stringify(overlapping))).toEqual({ + ok: true, + row: expect.not.objectContaining({ inputOrderLineage: expect.anything() }) + }) + const invalid = parseJournalRow(JSON.stringify({ ...epoch, inputOrderLineage: { clock: 'bad' } })) + expect(invalid).toEqual({ + ok: true, + row: expect.not.objectContaining({ inputOrderLineage: expect.anything() }) + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-row-schema.ts b/src/main/native-chat/agent-session-journal/journal-row-schema.ts index 24dd927fe9a..890ccd059cc 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-schema.ts @@ -1,3 +1,5 @@ +export { serializeJournalRow, journalRowByteLength } from './journal-row-encoding' +import type { JournalInputContext, JournalInputLineage } from './journal-input-lineage' import type { JournalItemEcho } from './journal-item-echo' // Upcast on read; preserve unknown versions/kinds and fail load instead of skipping history. // New kinds need reader-first rollout or a version bump because older builds may delete them. @@ -42,6 +44,7 @@ export type JournalEpochRow = JournalRowBase & { kind: 'epoch' reason: AgentJournalEpochReason providerHandle: AgentSessionJournalProviderHandle + inputOrderLineage?: JournalInputLineage /** Preserve original adoption through same-conversation epoch replacement. */ historyOrigin?: unknown } @@ -119,6 +122,7 @@ export function isJournalStopOrResumeRow(row: JournalRow): row is JournalStopOrR /** Persist before dispatch; the optimistic bubble gives provider echoes a slot without duplication. */ export type JournalSubmissionRow = JournalRowBase & { kind: 'submission' + inputOrderContext?: JournalInputContext clientMessageId: string payloadFingerprint: string providerHandle: AgentSessionJournalProviderHandle @@ -190,10 +194,6 @@ export type JournalRowParse = /** Future content forbids writing/compacting this journal. */ | { ok: false; unreadable: true } -export function serializeJournalRow(row: JournalRow): string { - return JSON.stringify(row) -} - export function parseJournalRow(line: string): JournalRowParse { let parsed: unknown try { @@ -357,7 +357,3 @@ function lifecycleMutationContent(value: unknown): AgentJournalContentVerdict { } return isJournalTag(kind) ? 'unreadable' : 'malformed' } - -export function journalRowByteLength(row: JournalRow): number { - return Buffer.byteLength(serializeJournalRow(row), 'utf8') + 1 -} diff --git a/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts b/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts index ff35d9f7128..6e2842c6f3f 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts @@ -1,6 +1,13 @@ +import { isInputContext, isInputLineage } from './journal-input-lineage' import { isAdmissibleAgentSessionContextUsage } from '../../../shared/agent-session-context-usage-schema' export function dropUnusableRowAnnotations(record: Record): void { + if (record.inputOrderContext !== undefined && !isInputContext(record.inputOrderContext)) { + delete record.inputOrderContext + } + if (record.inputOrderLineage !== undefined && !isInputLineage(record.inputOrderLineage)) { + delete record.inputOrderLineage + } dropUnusableProducerLinkage(record) dropUnusableTurnScope(record) dropUnusableOwnerFence(record) diff --git a/src/main/native-chat/agent-session-journal/journal-state-codec.ts b/src/main/native-chat/agent-session-journal/journal-state-codec.ts index 2263f74089c..c2b3cb6cdac 100644 --- a/src/main/native-chat/agent-session-journal/journal-state-codec.ts +++ b/src/main/native-chat/agent-session-journal/journal-state-codec.ts @@ -19,6 +19,8 @@ export function encodeJournalState(state: JournalReducerState): { tombstones: [...state.tombstones], removedTurnIds: [...state.removedTurnIds], submissions: [...state.submissions], + inputEntries: [...state.inputEntries], + inputCards: [...state.inputCards], receipts: [...state.receipts], aliases: [...state.aliases], itemEchoes: [...state.itemEchoes].map(([key, value]) => diff --git a/src/main/native-chat/agent-session-journal/journal-state-layout.ts b/src/main/native-chat/agent-session-journal/journal-state-layout.ts index 2f589dfe4a1..81cab4e10f5 100644 --- a/src/main/native-chat/agent-session-journal/journal-state-layout.ts +++ b/src/main/native-chat/agent-session-journal/journal-state-layout.ts @@ -15,6 +15,8 @@ export const journalStateLayout = { tombstones: 'map', removedTurnIds: 'set', submissions: 'map', + inputEntries: 'map', + inputCards: 'map', receipts: 'map', aliases: 'map', itemEchoes: 'map', diff --git a/src/main/native-chat/agent-session-journal/journal-state-schema.ts b/src/main/native-chat/agent-session-journal/journal-state-schema.ts index 426acf5973f..05a12f404d5 100644 --- a/src/main/native-chat/agent-session-journal/journal-state-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-state-schema.ts @@ -1,6 +1,11 @@ -import type { z } from 'zod' +import { z } from 'zod' import type { JournalReducerState } from './journal-reducer' import { outcomeFormat1Fields } from './journal-outcome-format-1' +import { inputEntrySchema, inputCardSchema } from './journal-outcome-format-2' // Every reducer key needs an explicit stored representation in the format this build writes. -export const fields = outcomeFormat1Fields satisfies Record +export const fields = { + ...outcomeFormat1Fields, + inputEntries: z.array(z.tuple([z.string().min(1), inputEntrySchema])), + inputCards: z.array(z.tuple([z.string().min(1), inputCardSchema])) +} satisfies Record diff --git a/src/main/native-chat/agent-session-journal/journal-storage-admission.test.ts b/src/main/native-chat/agent-session-journal/journal-storage-admission.test.ts index b4b9512a66c..f75b548ba37 100644 --- a/src/main/native-chat/agent-session-journal/journal-storage-admission.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-storage-admission.test.ts @@ -1,3 +1,4 @@ +import { admitOutcomeFormat2 } from './journal-outcome-format-2' import { readFileSync } from 'node:fs' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -193,7 +194,10 @@ const prefix = z ) function plain(state: JournalReducerState): unknown { - const fields: Record = { ...state } + const { inputEntries, inputCards, ...ordinary } = state + expect([...inputEntries]).toEqual([]) + expect([...inputCards]).toEqual([['card-2', { introduced: 11 }]]) + const fields: Record = ordinary return JSON.parse( JSON.stringify({ ...Object.fromEntries( @@ -223,7 +227,9 @@ it('folds the checked-in input rows into the same chat, every value of which sto } expect(plain(loaded.state)).toEqual(prefix.expected) const collections = journalStateCollections(loaded.state) - expect(collections.filter(({ entries }) => [...entries].length === 0)).toEqual([]) + expect( + collections.filter(({ entries }) => [...entries].length === 0).map(({ path }) => path) + ).toEqual(['inputEntries']) expect(collections.length).toBeGreaterThanOrEqual(15) // What restating this chat on a new epoch would store, admitted as this build stores it. const stored = [ @@ -247,7 +253,13 @@ it('folds the checked-in input rows into the same chat, every value of which sto ] expect(stored.length).toBeGreaterThan(20) for (const value of stored) { - expect(admitOutcomeFormat1({ ...value, deleted: false })).toEqual({ + const native = value.field === 'inputCards' || value.field === 'inputEntries' + if (native) { + expect(admitOutcomeFormat1({ ...value, deleted: false })).toBe('unknown') + } + expect( + (native ? admitOutcomeFormat2 : admitOutcomeFormat1)({ ...value, deleted: false }) + ).toEqual({ entryKey: JSON.parse(value.key), kept: JSON.parse(value.json) }) @@ -275,3 +287,15 @@ it('reads an open string such as an agent type as a value, and still refuses a k // @ts-expect-error `host` is a key the stored provider handle does not declare expect(declared>()).toBeUndefined() }) + +it('keeps explicitly opaque annotation values whole without hiding sibling or nested structured keys', () => { + type Actual = { annotation: { future: { keys: number[] } }; nested: { kept: string } } + type Stored = { annotation?: unknown; nested: { kept: string } } + expect(declared>()).toBeUndefined() + // @ts-expect-error an opaque annotation does not declare a sibling key + expect(declared>()).toBeUndefined() + expect( + // @ts-expect-error structured siblings still recursively declare their nested keys + declared>() + ).toBeUndefined() +}) diff --git a/src/main/native-chat/agent-session-journal/journal-storage-row-schema.ts b/src/main/native-chat/agent-session-journal/journal-storage-row-schema.ts index 49c4ab4c9a5..060b94e6ad6 100644 --- a/src/main/native-chat/agent-session-journal/journal-storage-row-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-storage-row-schema.ts @@ -69,6 +69,7 @@ const storedRow = z.discriminatedUnion('kind', [ z.object({ ...envelope, kind: z.literal('epoch'), + inputOrderLineage: value, reason: value, providerHandle: storedProviderHandle, historyOrigin: value @@ -98,6 +99,7 @@ const storedRow = z.discriminatedUnion('kind', [ z.object({ ...envelope, kind: z.literal('submission'), + inputOrderContext: value, clientMessageId: value, payloadFingerprint: value, providerHandle: storedProviderHandle, diff --git a/src/main/native-chat/agent-session-journal/journal-storage-strict.ts b/src/main/native-chat/agent-session-journal/journal-storage-strict.ts index d01a2d594d9..c1096ff1be1 100644 --- a/src/main/native-chat/agent-session-journal/journal-storage-strict.ts +++ b/src/main/native-chat/agent-session-journal/journal-storage-strict.ts @@ -115,17 +115,19 @@ type ArmKeys = [A] extends [never] * output `S` leaves undeclared; `never` when it declares them all. An unknown-kind value never * reaches storage: admission refuses it. A primitive holds no keys; it is settled before `object` * because an open or branded string (`AgentType`'s `string & {}`) is assignable to `object`. */ -export type UndeclaredStorageKeys = T extends { readonly kind: AgentJournalUnknownKind } +export type UndeclaredStorageKeys = unknown extends S ? never - : T extends string | number | boolean | bigint | symbol | null | undefined + : T extends { readonly kind: AgentJournalUnknownKind } ? never - : T extends readonly (infer E)[] - ? [Extract] extends [never] - ? 'an array storage does not declare' - : UndeclaredStorageKeys[number]> - : T extends object - ? ArmKeys> - : never + : T extends string | number | boolean | bigint | symbol | null | undefined + ? never + : T extends readonly (infer E)[] + ? [Extract] extends [never] + ? 'an array storage does not declare' + : UndeclaredStorageKeys[number]> + : T extends object + ? ArmKeys> + : never /** Compiles only when every persisted key is declared. */ export type StorageDeclaresAll = Undeclared diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index bbce643d73a..2955d0b42ca 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -139,6 +139,7 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal rowWriter, submissionWriter: new JournalSubmissionWriter({ state: host.state, + reopenFloor: () => host.journal().reopenFloor(), identity: host.identity, rowWriter, queuedMessages, diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index b46f2987f3c..8042d8165b4 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -1,8 +1,10 @@ +import { journalInputOrder } from './journal-input-order' // Each session owns its fold and queue; all statements use the host's shared database connection. import { JournalWriteProjection } from './journal-write-projection' import type { AgentJournalDispatchRejection } from '../../../shared/agent-session-failure-words' import { randomUUID } from 'node:crypto' +import { retireCoveredJournalFloor } from './journal-commit' import { createJournalPublicReader } from './journal-public-read' import type { AgentJournalAcceptanceReceipt, @@ -12,7 +14,6 @@ import type { AgentJournalRenderItem, AgentJournalSnapshot, AgentJournalSubmission, - AgentJournalThreadGoal, AgentJournalTurnLifecycle, AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' @@ -42,7 +43,6 @@ import type { JournalItemLinkageVisitor, JournalLifecycleBatchInput, JournalPlannedLifecycleBatchInput, - JournalReadSince, JournalTombstoneInput } from './journal-store-contracts' import { @@ -96,6 +96,7 @@ export class AgentSessionJournal { state: () => this.state, adopt: (state) => { this.state = state + this.reopenUnmarked = retireCoveredJournalFloor(state, this.reopenUnmarked) } }) const collaborators = createJournalStoreCollaborators({ @@ -108,10 +109,14 @@ export class AgentSessionJournal { readOnly: () => this.database.readOnly, cursor: this.cursor, adopt: (loaded) => { + journalInputOrder(loaded.state).carryRecovery(journalInputOrder(this.state)) this.state = loaded.state this.onCommitted?.() }, - prepare: (db, rows, savedAt) => this.projection.prepare(db, rows, savedAt), + prepare: (db, rows, savedAt) => { + this.queuedMessages.inputState() + return this.projection.prepare(db, rows, savedAt) + }, commit: (rows) => { this.onCommitted?.(rows) this.projection.verify() @@ -142,6 +147,10 @@ export class AgentSessionJournal { return this.identity.agent } + get inputOrder() { + return journalInputOrder(this.state) + } + /** Only rows from this handle's opening epoch can predate the open. */ wroteBeforeOpen(sequence: number | undefined): boolean { return journalWroteBeforeOpen(this.state, this.openedThrough, sequence) @@ -155,6 +164,7 @@ export class AgentSessionJournal { async open(): Promise { await this.restore() this.openedThrough = this.cursor() + journalInputOrder(this.state).markOpeningRecovery() } /** Refuse future writes and drain admitted ones; the host retains database ownership. */ @@ -224,7 +234,7 @@ export class AgentSessionJournal { journalThreadGoalSource(this.state.items.values()) /** Read across the journal, beyond any client's loaded page. */ - threadGoal = (): AgentJournalThreadGoal | null => this.threadGoalSource().goal + threadGoal = () => this.threadGoalSource().goal /** Read across the journal, beyond any client's loaded page. */ contextUsage = (): AgentSessionContextUsage => @@ -261,7 +271,7 @@ export class AgentSessionJournal { return this.queue.readInOrder(read) } - readSince(cursor: AgentJournalCursor, limit?: number): JournalReadSince { + readSince(cursor: AgentJournalCursor, limit?: number) { const { sessionId } = this.identity const rowsAfter = journalRowsAfterReader(this.database.db, sessionId, this.state.epoch, limit) return readJournalSince({ state: this.state, rowsAfter }, cursor, () => this.cursor()) @@ -325,7 +335,6 @@ export class AgentSessionJournal { const sequence = since ?? this.state.lastSequence + 1 this.reopenUnmarked = { epoch: this.state.epoch, sequence } await this.appendQueueReopen(fence, since) - this.reopenUnmarked = null } } @@ -354,10 +363,7 @@ export class AgentSessionJournal { } /** Failed initialization proves handed-over but unanswered sends never ran. */ - async rejectPendingSubmissions( - fence: number, - rejection: AgentJournalDispatchRejection - ): Promise { + async rejectPendingSubmissions(fence: number, rejection: AgentJournalDispatchRejection) { return rejectJournalPendingSubmissions(this, fence, rejection) } @@ -372,9 +378,7 @@ export class AgentSessionJournal { return this.epochController.roll(reason, fence) } - replaceEpochItems( - ...args: Parameters - ): Promise { + replaceEpochItems(...args: Parameters) { return this.epochController.replace(...args) } } diff --git a/src/main/native-chat/agent-session-journal/journal-submission-links.ts b/src/main/native-chat/agent-session-journal/journal-submission-links.ts new file mode 100644 index 00000000000..e1d421f17f8 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-submission-links.ts @@ -0,0 +1,37 @@ +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' + +/** Reverse links derive from submission rows and share the reducer lifetime. */ +export class JournalSubmissionLinks { + private readonly linked = new Map>() + private readonly ids = new Map() + private revision = 0 + + version(): number { + return this.revision + } + to(id: string): Iterable { + return this.linked.get(id)?.values() ?? [] + } + refresh(id: string, entry: AgentJournalSubmission | undefined): void { + const priorLink = this.ids.get(id) + const link = entry?.queuedMessageId + if (priorLink !== link && priorLink) { + const prior = this.linked.get(priorLink)! + prior.delete(id) + if (prior.size === 0) { + this.linked.delete(priorLink) + } + this.ids.delete(id) + this.revision++ + } + if (link && entry) { + const entries = this.linked.get(link) ?? new Map() + if (entries.get(id) !== entry) { + entries.set(id, entry) + this.linked.set(link, entries) + this.ids.set(id, link) + } + this.revision++ + } + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-submission-writer.ts b/src/main/native-chat/agent-session-journal/journal-submission-writer.ts index fd96fabb3ba..fdcab7e4fc1 100644 --- a/src/main/native-chat/agent-session-journal/journal-submission-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-submission-writer.ts @@ -3,6 +3,7 @@ import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import { journalDispatchRowApplies } from './journal-dispatch-settlement' +import { journalQueueReopenRowBuilder } from './journal-stop-and-resume-rows' import type { JournalQueuedMessages, JournalQueuePlan } from './journal-queued-messages' import type { JournalAttachmentClaim } from './journal-submission-hook' import type { JournalReducerState } from './journal-reducer' @@ -17,6 +18,7 @@ import type { export type JournalSubmissionWriterDeps = { state: () => JournalReducerState + reopenFloor: () => AgentJournalCursor | null identity: AgentSessionJournalIdentity rowWriter: JournalRowWriter queuedMessages: JournalQueuedMessages @@ -41,8 +43,13 @@ export class JournalSubmissionWriter { return rowWriter .enqueueRows( () => { + const floor = this.deps.reopenFloor() + const reopen = + floor?.epoch === state().epoch + ? [journalQueueReopenRowBuilder(state, input.fence, floor.sequence)] + : [] if (!consume) { - return [submission] + return [...reopen, submission] } queuedMessages.assertConsumable({ ...consume, @@ -51,7 +58,7 @@ export class JournalSubmissionWriter { }) // Adopt legacy cards in the handoff transaction. card = queuedMessages.plan([{ messageId: consume.messageId }]) - return [...card.rows, submission] + return [...reopen, ...card.rows, submission] }, { write: (db, row) => { diff --git a/src/main/native-chat/agent-session-journal/journal-unsent-send-hold-queue.spec.ts b/src/main/native-chat/agent-session-journal/journal-unsent-send-hold-queue.spec.ts index 27ca1cdc49b..274207b4413 100644 --- a/src/main/native-chat/agent-session-journal/journal-unsent-send-hold-queue.spec.ts +++ b/src/main/native-chat/agent-session-journal/journal-unsent-send-hold-queue.spec.ts @@ -1,5 +1,5 @@ import type { createFixture } from './journal-unsent-send-hold-contract-test-fixture' -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import { USER_MESSAGE_SOURCE } from '../../../shared/agent-session/agent-session-message-source' @@ -51,32 +51,28 @@ export function registerJournalUnsentSendHoldQueueTests( await fixture.accept(earlier, 'A', { origin: 'client', source: USER_MESSAGE_SOURCE }) await fixture.accept(earlier, 'B', { origin: 'client', source: USER_MESSAGE_SOURCE }) }) - // Simulate a crash after keeping A at a stale position but before settling B. - await interrupted.resolveDispatch( - { - clientMessageId: 'A', - state: 'rejected', - ...fixture.HOST_RESTARTED, - fence: 0, - recovered: true, + const resolve = interrupted.resolveDispatch.bind(interrupted) + const fault = vi + .spyOn(interrupted, 'resolveDispatch') + .mockImplementation((...args) => + args[0].clientMessageId === 'B' + ? Promise.reject(new Error('keep and fallback unavailable')) + : resolve(...args) + ) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + await expect(fixture.hold(interrupted)).rejects.toThrow('settling unsent sends failed') + expect(interrupted.submission('A')).toMatchObject({ + dispatchState: 'rejected', keptAsQueuedMessageId: 'A' - }, - () => - interrupted.queuedMessages.plan([ - { - messageId: 'A', - change: { - op: 'queued', - body: fixture.message('text of A'), - queuedAt: { - epoch: 'epoch-1', - sequence: interrupted.submission('A')!.acceptedSequence! - }, - position: 5 - } - } - ]) - ) + }) + expect(interrupted.submission('B')?.dispatchState).toBe('pending') + expect(interrupted.queuedMessages.get('A')?.position).toBe(0) + expect(interrupted.queuedMessages.get('C')?.position).toBe(2) + } finally { + fault.mockRestore() + warn.mockRestore() + } await interrupted.close() const journal = await fixture.open() await fixture.hold(journal) diff --git a/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts b/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts index aea71686dd0..4817d8abfc5 100644 --- a/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts +++ b/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts @@ -76,7 +76,7 @@ export async function holdUnsentSends( ? isUnansweredHandedOverSubmission(entry) && (input.unrun === true || input.unrun(entry)) : isQueuedAgentJournalSubmission(entry) && (hold.cause === 'hostRestarted' - ? journal.wroteBeforeOpen(entry.acceptedSequence) + ? journal.inputOrder.recoveryPending(entry.clientMessageId) : (hold.which?.(entry) ?? true)) ) .sort((a, b) => (a.acceptedSequence ?? 0) - (b.acceptedSequence ?? 0)) @@ -91,18 +91,16 @@ export async function holdUnsentSends( journal.itemBody(agentJournalSubmissionKey(submission.clientMessageId)) ) })) - const positions = headOfQueuePositions(journal, kept) + const positions = headOfQueuePositions(journal, kept, Boolean(input.unrun)) const rejection = agentSessionFailureWords(agentSessionFailureFact(hold.cause), { surface: 'rejection' }) const failures: unknown[] = [] - for (const [index, { submission, body }] of kept.entries()) { + let ranksCommitted = false + for (const { submission, body } of kept) { const { clientMessageId } = submission - const moves = [ - ...(index === 0 ? positions.earlier : []), - ...positions.placed.filter((entry) => entry.messageId === submission.queuedMessageId) - ] - const position = positions.placed.find((entry) => entry.messageId === clientMessageId)?.position + const moves = ranksCommitted ? [] : positions.moves + const position = positions.placed.get(clientMessageId) const card = body && position !== undefined ? { body, position } : null const keep: (() => JournalQueuePlan) | undefined = card || moves.length > 0 @@ -119,7 +117,7 @@ export async function holdUnsentSends( ] ) if (card && !queuedMessages.get(clientMessageId)) { - const queuedAt = { epoch, sequence: submission.acceptedSequence ?? 0 } + const queuedAt = { epoch, sequence: submission.submittedSequence! } const change = { op: 'queued' as const, body: card.body, @@ -145,6 +143,9 @@ export async function holdUnsentSends( card ? { ...reject, keptAsQueuedMessageId: clientMessageId } : reject, keep ) + if (keep) { + ranksCommitted = true + } } catch (error) { if (!keep) { failures.push(error) @@ -166,45 +167,105 @@ export async function holdUnsentSends( return unsent.at(-1)?.acceptedSequence ?? 0 } -/** Order earlier kept cards first, then admissions; queuedAt cannot compare across epochs or handoff origins. */ +/** Merge logical places once, then materialize finite ranks with the first successful keep. */ function headOfQueuePositions( journal: AgentSessionJournal, - batch: readonly { submission: AgentJournalSubmission; body: AgentJournalMessageItem | null }[] -): { placed: QueuedMessagePositionMove[]; earlier: QueuedMessagePositionMove[] } { + batch: readonly { submission: AgentJournalSubmission; body: AgentJournalMessageItem | null }[], + unrun: boolean +): { placed: Map; moves: QueuedMessagePositionMove[] } { const cards = journal.queuedMessages.list() const keptIds = new Set( journal.submissions().flatMap((entry) => entry.keptAsQueuedMessageId ?? []) ) const { epoch } = journal.cursor() - // Earlier settlements survive either through explicit links or cards from earlier epochs. - const earlier = cards - .filter( - (card) => - card.state === 'waiting' && - (keptIds.has(card.messageId) || (card.queuedAt !== null && card.queuedAt.epoch !== epoch)) - ) - .map((card) => card.messageId) - const placed: QueuedMessagePositionMove[] = [] - for (const { submission, body } of batch) { - const { clientMessageId } = submission - if (body && !earlier.includes(clientMessageId)) { - placed.push({ messageId: clientMessageId, position: 0 }) - } else if ( - // Explicit Send handoffs return to the head; automatic queue handoffs stay put. - submission.origin === 'client' && - submission.queuedMessageId !== undefined && - cards.some((card) => card.consumedAs === clientMessageId) + const state = journal.queuedMessages.displayState() + const forcedBatch = batch.some(({ submission }) => + journal.inputOrder.forced(submission.clientMessageId) + ) + const earlier: string[] = [] + const rest: (typeof cards)[number][] = [] + const cardsById = new Map(cards.map((card) => [card.messageId, card])) + const explicitHeads = new Map( + batch.flatMap(({ submission }) => { + const id = submission.origin === 'client' ? submission.queuedMessageId : undefined + return id && cardsById.get(id)?.consumedAs === submission.clientMessageId + ? [[submission.clientMessageId, id] as const] + : [] + }) + ) + const explicit = new Set(explicitHeads.values()) + for (const card of cards) { + if (explicit.has(card.messageId)) { + continue + } + if ( + card.state === 'waiting' && + ((unrun && keptIds.has(card.messageId)) || + (!forcedBatch && card.queuedAt && card.queuedAt.epoch !== epoch)) ) { - placed.push({ messageId: submission.queuedMessageId, position: 0 }) + earlier.push(card.messageId) + } else { + rest.push(card) } } - const inHead = (card: (typeof cards)[number]): boolean => - earlier.includes(card.messageId) || placed.some((move) => move.messageId === card.messageId) - const others = cards.filter((card) => !inHead(card)).map((card) => card.position) - const anchor = others.length > 0 ? Math.min(...others) : 1 - const first = anchor - earlier.length - placed.length + const at = new Map() + const head: string[] = [] + const firstCut = batch + .flatMap( + ({ submission }) => journal.inputOrder.place(submission.clientMessageId)?.context.cut ?? [] + ) + .reduce((a, b) => Math.min(a, b), Infinity) + let previousBoundary = 0 + for (let i = 0; i < rest.length; i++) { + if ( + rest[i].state === 'waiting' && + keptIds.has(rest[i].messageId) && + (journal.inputOrder.lineage.cards.get(rest[i].messageId)?.introduced ?? -Infinity) < firstCut + ) { + previousBoundary = i + 1 + } + } + for (const { submission, body } of batch) { + const id = body ? submission.clientMessageId : explicitHeads.get(submission.clientMessageId) + if (!id || keptIds.has(id)) { + continue + } + if (!body && previousBoundary > 0) { + head.push(id) + continue + } + const after = journal.inputOrder.forced(submission.clientMessageId) + ? journal.inputOrder.predecessors(state, submission.clientMessageId).position + : -Infinity + let low = 0, + high = rest.length + while (low < high) { + const mid = (low + high) >>> 1 + if (rest[mid].position <= after) { + low = mid + 1 + } else { + high = mid + } + } + const boundary = Math.max(previousBoundary, low) + previousBoundary = boundary + const bucket = at.get(boundary) ?? [] + bucket.push(id) + at.set(boundary, bucket) + } + const order = [...earlier, ...head] + for (let i = 0; i <= rest.length; i++) { + order.push(...(at.get(i) ?? [])) + if (i < rest.length) { + order.push(rest[i].messageId) + } + } + const placed = new Map(order.map((id, index) => [id, index])) return { - earlier: earlier.map((messageId, index) => ({ messageId, position: first + index })), - placed: placed.map((move, index) => ({ ...move, position: first + earlier.length + index })) + placed, + moves: cards.flatMap((card) => { + const position = placed.get(card.messageId)! + return position === card.position ? [] : [{ messageId: card.messageId, position }] + }) } } diff --git a/src/main/native-chat/agent-session-journal/queued-message-edit-leases.test.ts b/src/main/native-chat/agent-session-journal/queued-message-edit-leases.test.ts index 13a02fca786..929217de906 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-edit-leases.test.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-edit-leases.test.ts @@ -61,6 +61,17 @@ describe('queued edit leases', () => { expect(leases.heldIds().size).toBe(0) }) + it('reuses an edit key only after its previous target becomes invalid', () => { + rows.push(card('second')) + expect(leases.acquire(desktop, base)).toMatchObject({ status: 'held' }) + expect(leases.acquire(desktop, { ...base, messageId: 'second' })).toEqual({ status: 'changed' }) + rows[0]!.fingerprint = 'newer' + expect(leases.acquire(desktop, { ...base, messageId: 'second' })).toMatchObject({ + status: 'held' + }) + expect([...leases.heldIds()]).toEqual(['second']) + }) + it('expires on its own and wakes the queue with no other activity', () => { leases.acquire(desktop, base) advance(QUEUED_MESSAGE_EDIT_LEASE_MS - 1) diff --git a/src/main/native-chat/agent-session-journal/queued-message-edit-leases.ts b/src/main/native-chat/agent-session-journal/queued-message-edit-leases.ts index c9865054b01..0a93f9f1a44 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-edit-leases.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-edit-leases.ts @@ -14,14 +14,18 @@ type EditLease = Pick() + private readonly byId = new Map>() private timer: ReturnType | undefined private timerAt: number | null = null private wake: (() => void) | undefined private disposed = false + private revision = 0 constructor( private readonly rows: () => readonly QueuedMessageRow[], - private readonly now: () => number = () => performance.now() + private readonly now: () => number = () => performance.now(), + private readonly get: (id: string) => QueuedMessageRow | null | undefined = (id) => + this.rows().find((row) => row.messageId === id) ) {} /** Wake the queue even when no journal activity follows expiry. */ @@ -29,27 +33,70 @@ export class QueuedMessageEditLeases { this.wake = wake } - /** Transaction callers supply their own card snapshot. */ - heldIds(rows: readonly QueuedMessageRow[] = this.rows()): ReadonlySet { - if (this.leases.size === 0) { - return new Set() + version(): number { + this.expire() + return this.revision + } + + isHeld(messageId: string): boolean { + this.expire() + if (!this.byId.has(messageId)) { + return false } - const byId = new Map(rows.map((row) => [row.messageId, row])) - const now = this.now() - let lapsed = false + const revision = this.revision + const row = this.get(messageId) + for (const key of this.byId.get(messageId) ?? []) { + const lease = this.leases.get(key)! + if (!row || !leaseMatchesRow(lease, row)) { + this.remove(key) + } + } + if (this.revision !== revision) { + this.schedule() + } + return this.byId.has(messageId) + } + + /** Output enumeration revalidates only leased identities; selection uses exact-ID reads. */ + heldIds(rows?: readonly QueuedMessageRow[]): ReadonlySet { + this.expire() + const snapshot = rows && new Map(rows.map((row) => [row.messageId, row])) for (const [key, lease] of this.leases) { - const row = byId.get(lease.messageId) - lapsed ||= lease.until <= now - if (lease.until <= now || !row || !leaseMatchesRow(lease, row)) { - this.leases.delete(key) + const row = snapshot ? snapshot.get(lease.messageId) : this.get(lease.messageId) + if (!row || !leaseMatchesRow(lease, row)) { + this.remove(key) } } this.schedule() - // A pruned deadline cancels its timer; wake in that timer's place. - if (lapsed) { - this.wake?.() + return new Set(this.byId.keys()) + } + + private remove(key: string): void { + const lease = this.leases.get(key) + if (!lease) { + return } - return new Set([...this.leases.values()].map((lease) => lease.messageId)) + this.leases.delete(key) + const keys = this.byId.get(lease.messageId)! + keys.delete(key) + if (keys.size === 0) { + this.byId.delete(lease.messageId) + } + this.revision++ + } + + private expire(): void { + if (this.timerAt === null || this.timerAt > this.now()) { + return + } + const now = this.now() + for (const [key, lease] of this.leases) { + if (lease.until <= now) { + this.remove(key) + } + } + this.schedule() + this.wake?.() } /** Repeated acquisition returns the same lease without extending it. */ @@ -57,14 +104,15 @@ export class QueuedMessageEditLeases { key: EditLeaseKey, input: { messageId: string; fingerprint: string } ): AgentSessionQueuedMessageEditHoldResult { - this.heldIds() + this.expire() + this.isHeld(this.leases.get(leaseKey(key))?.messageId ?? input.messageId) const existing = this.leases.get(leaseKey(key)) if (existing) { return existing.messageId === input.messageId && existing.fingerprint === input.fingerprint ? this.answer(existing) : { status: 'changed' } } - const row = this.rows().find((entry) => entry.messageId === input.messageId) + const row = this.get(input.messageId) if (this.disposed || !row || !isUnsettledQueuedMessage(row)) { return { status: 'gone' } } @@ -79,13 +127,17 @@ export class QueuedMessageEditLeases { until: this.now() + QUEUED_MESSAGE_EDIT_LEASE_MS } this.leases.set(leaseKey(key), lease) + const keys = this.byId.get(lease.messageId) ?? new Set() + keys.add(leaseKey(key)) + this.byId.set(lease.messageId, keys) + this.revision++ this.schedule() return this.answer(lease) } /** Expired/invalidated editors must acquire a new lease. */ renew(key: EditLeaseKey, messageId: string): AgentSessionQueuedMessageEditHoldResult { - this.heldIds() + this.isHeld(messageId) const lease = this.leases.get(leaseKey(key)) if (!lease || lease.messageId !== messageId) { return { status: 'expired' } @@ -98,16 +150,14 @@ export class QueuedMessageEditLeases { /** Release only the exact caller/edit pair. */ release(key: EditLeaseKey, messageId: string): void { if (this.leases.get(leaseKey(key))?.messageId === messageId) { - this.leases.delete(leaseKey(key)) + this.remove(leaseKey(key)) this.schedule() } } retire(messageId: string): void { - for (const [key, lease] of this.leases) { - if (lease.messageId === messageId) { - this.leases.delete(key) - } + for (const key of this.byId.get(messageId) ?? []) { + this.remove(key) } this.schedule() } @@ -115,6 +165,8 @@ export class QueuedMessageEditLeases { dispose(): void { this.disposed = true this.leases.clear() + this.byId.clear() + this.revision++ this.schedule() } @@ -146,7 +198,7 @@ export class QueuedMessageEditLeases { const now = this.now() for (const [key, lease] of this.leases) { if (lease.until <= now) { - this.leases.delete(key) + this.remove(key) } } this.schedule() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts index 382275028b7..fd6e96383d0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts @@ -319,7 +319,8 @@ export class StructuredAgentSessionClientDelivery { submissions: () => journal.submissions().map(this.currentWork(sessionId, journal).publishedSubmission), cursor: () => journal.cursor(), - activeTurnId: () => this.currentWork(sessionId, journal).activeTurnId() + activeTurnId: () => this.currentWork(sessionId, journal).activeTurnId(), + behindCommand: (id: string) => journal.inputOrder.forced(id) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-admission.spec.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-admission.spec.ts index 8bdc355bc79..4dd23b0e6a0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-admission.spec.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-admission.spec.ts @@ -1,3 +1,5 @@ +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' import * as autoSandbox from '../../codex/codex-structured-auto-sandbox' import { expect, it, vi } from 'vitest' import { codexInputFinalizationFixture } from './structured-agent-session-codex-input-host.test' @@ -8,7 +10,8 @@ import { import { CODEX_INPUT_FINALIZATION_ADMISSION_CASES, HOST_TEST_SESSION as SESSION, - hostTestMessage + hostTestMessage, + hostTestOperationId } from './structured-agent-session-host-test-data' import { agentSessionFailureFact } from '../../../shared/agent-session-failure' import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' @@ -235,12 +238,12 @@ it.each(CODEX_INPUT_FINALIZATION_ADMISSION_CASES)( } ) -it.each(['stop', 'exit', 'exit-after-close-record'] as const)( +it.each(['stop', 'stop-save-refusal', 'exit', 'exit-after-close-record'] as const)( 'unwinds actual Auto preparation on %s without a late write', async (ending) => { const { host, adapter, codex, config, pick, permissionSend, stop, starts } = codexInputFinalizationFixture() - const exiting = ending !== 'stop' + const exiting = ending.startsWith('exit') Object.defineProperty(process, 'platform', { configurable: true, value: 'linux' }) await pick('auto') const paused = Promise.withResolvers() @@ -320,7 +323,33 @@ it.each(['stop', 'exit', 'exit-after-close-record'] as const)( expect(child.close).toMatchObject({ cause: 'evict' }) expect(signal?.aborted).toBe(false) } - if (ending === 'stop') { + if (ending === 'stop-save-refusal') { + const db = codexInputFinalizationFixture().database() + const receipt = journal.receiptFor(id) + db.exec(`CREATE TRIGGER fail_stop_receipt BEFORE INSERT ON agent_session_command_receipts + BEGIN SELECT RAISE(ABORT, 'receipt unavailable'); END`) + try { + expect(await stop()).toMatchObject({ + ok: false, + refusal: { details: { reason: 'stopFailed', agent: 'codex' } } + }) + } finally { + db.exec('DROP TRIGGER fail_stop_receipt') + } + expect(journal.receiptFor(id)).toEqual(receipt) + expect(journal.queuedMessages.pauses()).toEqual([]) + expect( + journal + .snapshot() + .items.some( + (item) => + item.body.kind === 'message' && + item.body.blocks.some( + (block) => block.type === 'text' && block.text === 'Stopped manually' + ) + ) + ).toBe(false) + } else if (ending === 'stop') { await stop() } else { codex.connections[0]?.handlers.onExit?.( @@ -348,6 +377,23 @@ it.each(['stop', 'exit', 'exit-after-close-record'] as const)( expect(starts()).toHaveLength(0) const writes = await Promise.allSettled(written.mock.results.map((result) => result.value)) expect(writes.filter((result) => result.status === 'rejected')).toEqual([]) + if (ending === 'stop-save-refusal') { + const next = await permissionSend('next after refused Stop') + await vi.waitFor(() => expect(starts()).toHaveLength(1)) + expect(starts()[0]?.params).toMatchObject({ + clientUserMessageId: next, + input: [{ type: 'text', text: 'next after refused Stop' }] + }) + const fixture = codexInputFinalizationFixture() + fixture.turns.start() + fixture.turns.echo(next) + expect( + fixture.submission((await host.waitForSendSettlement(SESSION, next))?.value) + ).toMatchObject({ clientMessageId: next, dispatchState: 'accepted' }) + await fixture.waitForInputFinalization(journal, next) + expect(journal.queuedMessages.list()).toEqual([]) + expect(starts()).toHaveLength(1) + } } finally { paused.resolve(config) await Promise.allSettled(lateReads) @@ -355,3 +401,147 @@ it.each(['stop', 'exit', 'exit-after-close-record'] as const)( } } ) + +it('keeps rewind outside a committed preparation until the input settles', async () => { + const { host, adapter, permissionSend, turns, starts, rewindTo } = codexInputFinalizationFixture() + const { sessions, serialize, conversationDelivery } = host.collaboratorsForTests() + const journal = sessions.get(SESSION)!.journal + const prepared = Promise.withResolvers() + const release = Promise.withResolvers() + const dispatch = adapter.dispatch.bind(adapter) + vi.spyOn(adapter, 'dispatch').mockImplementation(async (input) => { + prepared.resolve() + await release.promise + return dispatch(input) + }) + const providerRewind = vi.spyOn(adapter, 'rewind') + let rewinding: ReturnType | undefined + try { + const id = await permissionSend('committed preparation') + await prepared.promise + expect(journal.submission(id)?.handedOverAt).toBeDefined() + const epoch = journal.epoch + const reserved = Promise.withResolvers() + const order = conversationDelivery.loop['deps'].deliveryOrder + const reserve = order.reserve.bind(order) + vi.spyOn(order, 'reserve').mockImplementation((sessionId) => { + reserved.resolve() + return reserve(sessionId) + }) + rewinding = rewindTo(agentJournalSubmissionKey(id), epoch) + await reserved.promise + await serialize(SESSION, async () => {}) + expect(providerRewind).not.toHaveBeenCalled() + expect(journal.epoch).toBe(epoch) + expect(starts()).toHaveLength(0) + release.resolve() + await vi.waitFor(() => expect(starts()).toHaveLength(1)) + turns.start() + turns.echo(id) + await codexInputFinalizationFixture().waitForInputFinalization(journal, id) + expect(await rewinding).toMatchObject({ + ok: false, + refusal: { details: { rewindReason: 'busy' } } + }) + expect(providerRewind).not.toHaveBeenCalled() + expect(journal.epoch).toBe(epoch) + } finally { + release.resolve() + await rewinding + } +}) + +it('keeps earlier B before historically forced proven-unrun A through close and Resume delivery', async () => { + const { + host, + adapter, + codex, + config, + permissionSend, + began, + turns, + starts, + pick, + waitForInputFinalization + } = codexInputFinalizationFixture() + const { sessions, serialize } = host.collaboratorsForTests() + const journal = sessions.get(SESSION)!.journal + const w = await permissionSend('working W') + await began(w) + await serialize(SESSION, () => + journal.queuedMessages.insert({ + messageId: 'force-C', + body: { ...hostTestMessage('/compact'), command: { name: 'compact' } } + }) + ) + const a = await permissionSend('forced A', { personality: 'friendly' }) + expect(journal.inputOrder.forced(a)).toBe(true) + await serialize(SESSION, () => journal.queuedMessages.withdraw(['force-C'])) + await pick('auto') + const preparation = Promise.withResolvers() + const read = vi.fn(() => preparation.promise) + codex.routes['config/read'] = read + try { + turns.end('completed') + await vi.waitFor(() => expect(read).toHaveBeenCalledOnce()) + expect(journal.submission(a)).toMatchObject({ + dispatchState: 'pending', + handedOverAt: expect.any(Number) + }) + const b = await permissionSend('ordinary B', { personality: 'serious' }) + expect(journal.inputOrder.forced(b)).toBe(false) + await host.close(SESSION, 'user-close') + await waitForInputFinalization(journal, a) + expect(starts()).toHaveLength(1) + const kept = journal.queuedMessages.list().filter((c) => c.state === 'waiting') + expect.soft(kept.map((c) => c.messageId)).toEqual([b, a]) + for (const [id, personality] of [ + [a, 'friendly'], + [b, 'serious'] + ] as const) { + expect(kept.find((c) => c.messageId === id)?.options).toEqual(journal.submission(id)?.options) + expect(kept.find((c) => c.messageId === id)?.options).toMatchObject({ personality }) + expect(journal.submission(id)).toMatchObject({ + dispatchState: 'rejected', + keptAsQueuedMessageId: id + }) + } + preparation.resolve(config) + codex.routes['config/read'] = () => config + const caller = { callerKey: 'client-1' } + await host.revealSession(SESSION) + const method = 'agentSession.queuedMessagesResume' + expect( + await host.queuedMessagesResume(caller, { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method, + sessionId: SESSION, + fields: {} + }) + } + }) + ).toMatchObject({ ok: true }) + const reopened = sessions.get(SESSION)!.journal + const delivered: string[] = [] + for (const { messageId: id } of kept) { + await vi.waitFor(() => expect(reopened.queuedMessages.get(id)?.consumedAs).toBeTruthy()) + const h = reopened.queuedMessages.get(id)!.consumedAs! + await began(h) + expect(reopened.submission(h)?.dispatchState).toBe('accepted') + delivered.push(reopened.submission(h)!.queuedMessageId!) + turns.end('completed') + await host.flushStreamedEvents(SESSION) + } + expect.soft(delivered).toEqual([b, a]) + expect(starts()).toHaveLength(3) + expect(reopened.queuedMessages.list().filter((c) => c.state === 'waiting')).toEqual([]) + expect(reopened.queuedMessages.displayState().inputEntries.size).toBe(0) + } finally { + preparation.resolve(config) + await adapter.closeAll() + } +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-finalization.spec.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-finalization.spec.ts index 4949f688cc6..a6a74d41951 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-finalization.spec.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-finalization.spec.ts @@ -619,24 +619,46 @@ export function registerCodexInputFinalization(current: typeof codexInputFinaliz results.map((result) => (result.type === 'return' ? result.value : undefined)) ) } + const stoppedX = 'stopped-before-carrier' + if (boundary === 'retire') { + await host.collaboratorsForTests().serialize(SESSION, async () => { + await journal.appendSubmission({ + clientMessageId: stoppedX, + payloadFingerprint: stoppedX, + body: hostTestMessage('earlier stopped X'), + source: { kind: 'user' }, + origin: 'client', + handoverRecorded: true, + fence: original.fence + }) + await journal.stops.accept({ + event: { reason: 'user-stop' }, + fence: original.fence, + liveTurnId: () => null + }) + }) + expect(journal.submission(stoppedX)?.rejection).toMatchObject({ neverReachedAgent: true }) + } const id = await permissionSend('same original input', { personality: 'friendly' }) const waiting = host.waitForSendSettlement(SESSION, id) const originalBody = hostTestMessage('same original input') // Ready replacement-child exit proof settles before lease release; terminal rows keep that fence. const settlesAtChildFence = ['readiness', 'rebind'].includes(boundary) - let boundFence = journal.submission(id)!.fence let stopping: Promise = Promise.resolve() try { await entered.promise expect(continuingStructuredAgentSessionInput(journal, id)).toBe(true) expect(signal?.aborted).toBe(false) + if (boundary === 'retire') { + expect(journal.stoppedHistoryCarried().get(id)).toEqual([stoppedX]) + } const child = sessions.get(SESSION)?.child expect(journal.submission(id)).toMatchObject({ clientMessageId: id, options: { personality: 'friendly' } }) expect(journal.itemBody(agentJournalSubmissionKey(id))).toEqual(originalBody) - boundFence = child?.fence ?? journal.submission(id)!.fence + const boundFence = child?.fence ?? journal.submission(id)!.fence if (boundary === 'rebind') { expect(reboundFence).toBe(child!.fence) } @@ -755,6 +777,19 @@ export function registerCodexInputFinalization(current: typeof codexInputFinaliz expect(journal.submission(id)?.rejection).not.toHaveProperty('neverReachedAgent') expect(starts()).toHaveLength(0) expect(codex.connections).toHaveLength(boundary === 'retire' ? 1 : 2) + const next = await permissionSend('new question after Stop') + await vi.waitFor(() => expect(starts()).toHaveLength(1)) + expect(starts()[0]?.params).toMatchObject({ + clientUserMessageId: next, + input: [{ type: 'text', text: 'new question after Stop' }] + }) + turns.start() + turns.echo(next) + expect( + current().submission((await host.waitForSendSettlement(SESSION, next))?.value) + ).toMatchObject({ clientMessageId: next, dispatchState: 'accepted' }) + expect(journal.stoppedHistoryCarried().get(next) ?? []).toEqual([]) + await current().waitForInputFinalization(journal, next) } finally { held.resolve() retryReady.resolve() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-host.test.ts index 49e1c8cfea2..2a7510fd36c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-host.test.ts @@ -783,6 +783,13 @@ export function codexInputFinalizationFixture() { adapter, codex, config, + database: () => openTestJournalHostDatabase(root).db, + rewindTo: (itemId: string, expectedEpoch: string) => + host.rewind(CALLER, { + envelope: envelope('agentSession.rewind', { itemId, expectedEpoch }), + itemId, + expectedEpoch + }), turns, permissionSend, began, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-perf-test-support.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-perf-test-support.ts index d93d5252c40..3916ff1b905 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-perf-test-support.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-input-perf-test-support.ts @@ -124,10 +124,10 @@ export function measureCodexInputWork( }) expect(work.reportSubmissions).toBeGreaterThanOrEqual(population + 2) } else { - expect(plans.mock.calls.reduce((sum, [changes]) => sum + changes.length, 0)).toBe( - 2 * population + 4 - ) - expect(comparisons).toBe(population * population + 6 * population + 3) + expect( + plans.mock.calls.reduce((sum, [changes]) => sum + changes.length, 0) + ).toBeLessThanOrEqual(2 * population + 4) + expect(comparisons).toBe(0) expect(work.cards).toBeGreaterThanOrEqual(population) } if (population === 100) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts index cda94b5d20f..f092640eb99 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts @@ -21,6 +21,7 @@ import { structuredAgentSessionEndedChildFailure } from './structured-agent-sess import { waitsForCodexTurnOptions } from './structured-agent-session-message-options' import { sameProviderChild } from './structured-agent-session-provider-child' import { DeliveryProgress } from './structured-agent-session-delivery-progress' +import { selectableStructuredSubmission } from './structured-agent-session-input-selection' export type { StructuredAgentSessionDeliveryLoopDeps } from './structured-agent-session-delivery-loop-deps' @@ -61,10 +62,7 @@ export class StructuredAgentSessionDeliveryLoop { return Promise.resolve() } this.running.get(sessionId)?.woke() - this.deps.deliveryOrder.reconcile( - sessionId, - this.deps.sessions.get(sessionId)?.journal.submissions() ?? [] - ) + this.deps.deliveryOrder.reconcile(sessionId, this.deps.sessions.get(sessionId)?.journal) if (this.running.has(sessionId)) { return this.runs.get(sessionId) ?? Promise.resolve() } @@ -187,9 +185,9 @@ export class StructuredAgentSessionDeliveryLoop { if (!(await this.deps.closeWhatTheUserClosed(sessionId))) { return this.stop(sessionId) } - const oldest = oldestQueuedSubmission(session) // Commands block while their child lives; ended command commits wake delivery again. const work = this.deps.currentWork(sessionId) + const oldest = selectableStructuredSubmission(session.journal, work) if (!oldest || (session.child && work && structuredAgentSessionCommandRunning(work))) { return this.stop(sessionId) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-order.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-order.ts index 183e3e62912..b420b3ff464 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-order.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-order.ts @@ -1,5 +1,5 @@ -import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' -import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { selectedStructuredSubmission } from './structured-agent-session-input-selection' import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' export type StructuredAgentSessionDeliveryPlace = { @@ -26,8 +26,9 @@ type Lane = { export class StructuredAgentSessionDeliveryOrder { private readonly lanes = new Map() - reconcile(sessionId: string, submissions: readonly AgentJournalSubmission[]): void { - const queued = submissions.filter(isQueuedAgentJournalSubmission) + reconcile(sessionId: string, journal: AgentSessionJournal | undefined): void { + const selected = journal && selectedStructuredSubmission(journal) + const queued = selected ? [selected] : [] const lane = this.lanes.get(sessionId) if (lane?.options && (queued.length > 0 || lane.active.size > 0)) { lane.options.demand = 'sealed' @@ -39,9 +40,7 @@ export class StructuredAgentSessionDeliveryOrder { place.release() } } - for (const submission of queued.sort( - (a, b) => (a.acceptedSequence ?? 0) - (b.acceptedSequence ?? 0) - )) { + for (const submission of queued) { const current = this.lanes.get(sessionId) if (!current?.sends.has(submission.clientMessageId)) { const place = this.reserve(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts index 33d7d14a8fc..92e9c8230d1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts @@ -1,3 +1,4 @@ +import { selectableStructuredSubmission } from './structured-agent-session-input-selection' import { structuredAgentSessionReceivingAccount } from './structured-agent-session-receiving-account' // Open conversations with queued sends always need delivery; opening also wakes leftover recovery. @@ -234,9 +235,8 @@ export function createStructuredAgentSessionConversationDelivery(input: { }) const adoptOpened = async ( sessionId: string, - opened: OpenedStructuredAgentSessionConversation + { session }: OpenedStructuredAgentSessionConversation ): Promise => { - const { session } = opened sessions.set(sessionId, session) input.clientDelivery.publishRestored(sessionId) await settleInterruptedStructuredAgentSessionCommands(deps, sessionId, session) @@ -247,13 +247,13 @@ export function createStructuredAgentSessionConversationDelivery(input: { } const wakesQueued = new Set() const afterCommit = (sessionId: string, journal: AgentSessionJournal): void => { - input.deliveryOrder.reconcile(sessionId, journal.submissions()) + input.deliveryOrder.reconcile(sessionId, journal) const work = hostStructuredAgentSessionCurrentWork({ store: deps.store, sessions }, sessionId) if ( wakesQueued.has(sessionId) || !work || structuredAgentSessionCommandRunning(work) || - !journal.submissions().some(isQueuedAgentJournalSubmission) + !selectableStructuredSubmission(journal, work) ) { return } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-input-order-projection.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-input-order-projection.test.ts new file mode 100644 index 00000000000..d49a0386e4a --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-input-order-projection.test.ts @@ -0,0 +1,633 @@ +import * as transitions from '../agent-session-journal/journal-reducer-transition' +import * as outcomes from '../agent-session-journal/journal-outcome-codec' +import { journalInputOrder, JournalInputOrder } from '../agent-session-journal/journal-input-order' +import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' +import { QueuedMessageEditLeases } from '../agent-session-journal/queued-message-edit-leases' +import * as pause from '../agent-session-journal/queued-message-pause' +import { expect, it, vi } from 'vitest' +import { JournalInputCardIndex } from '../agent-session-journal/journal-input-card-index' +import { JournalInputPlaceIndex } from '../agent-session-journal/journal-input-place-index' +import { JournalInputDomainIndex } from '../agent-session-journal/journal-input-domain-index' +import { JournalInputLineageOwner } from '../agent-session-journal/journal-input-lineage' +import * as receiptTable from '../agent-session-journal/command-receipt-table' +import { loadTestJournal } from '../agent-session-journal/journal-host-database-test-support' +import { selectStructuredAgentSessionInput } from './structured-agent-session-input-selection' +import { + createQueuedMessageTestRig, + eventually +} from './structured-agent-session-queued-message-rig.test-fixture' +import { openRigTurnFor } from './structured-agent-session-queued-rig-turn.test-fixture' +import { + HOST_TEST_SESSION as SESSION, + hostTestMessage +} from './structured-agent-session-host-test-data' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { QUEUED_MESSAGE_PAUSED_SEND_FAILED } from '../../../shared/agent-session-queued-message-wire' + +it.each([10, 100])( + 'bounds actual edit/growth, replay, replacement and complete recovery N=%i', + async (n) => { + type Counts = { + rows: number + queueVisits: number + inputVisits: number + sets: number + queries: number + domains: number + lineage: number + receipts: number + nodes: number + nodeReads: number + draftNodes: number + derivedDraftNodes: number + nativeEntries: number + nativeCards: number + nativeBytes: number + } + const phases: Record = {} + let phase = 'setup' + const metric = () => + (phases[phase] ??= { + rows: 0, + queueVisits: 0, + inputVisits: 0, + sets: 0, + queries: 0, + domains: 0, + lineage: 0, + receipts: 0, + nodes: 0, + nodeReads: 0, + draftNodes: 0, + derivedDraftNodes: 0, + nativeEntries: 0, + nativeCards: 0, + nativeBytes: 0 + }) + const consumers: Record< + string, + { + heads: number + lists: number + pauses: number + leases: number + gets: number + waitingRows: number + leaseRows: number + headVisits: number + predicates: number + indexedCards: number + } + > = {} + const upstream: Record< + string, + Record + > = {} + const source = (): string => { + const stack = new Error('Attribute production queue reads').stack ?? '' + if (selecting || stack.includes('selectStructuredAgentSessionInput')) { + return 'selection' + } + if (stack.includes('insert')) { + return 'insertion' + } + if (stack.includes('snapshot') || stack.includes('Snapshot')) { + return 'publication' + } + return 'other' + } + let selecting = 0 + const seam = () => + (consumers[phase] ??= { + heads: 0, + lists: 0, + pauses: 0, + leases: 0, + gets: 0, + waitingRows: 0, + leaseRows: 0, + headVisits: 0, + predicates: 0, + indexedCards: 0 + }) + const stackLimit = Error.stackTraceLimit + Error.stackTraceLimit = 40 + const indexCard = JournalInputCardIndex.prototype.indexCard + const indexed = vi + .spyOn(JournalInputCardIndex.prototype, 'indexCard') + .mockImplementation(function (this: JournalInputCardIndex, ...args) { + if (selecting) { + seam().indexedCards++ + } + return indexCard.apply(this, args) + }) + const next = JournalQueuedMessages.prototype.nextInputCard + const head = vi + .spyOn(JournalQueuedMessages.prototype, 'nextInputCard') + .mockImplementation(function (this: JournalQueuedMessages, ...args) { + selecting++ + seam().heads++ + try { + return next.apply(this, args) + } finally { + selecting-- + } + }) + const list = JournalQueuedMessages.prototype.list + const lists = vi.spyOn(JournalQueuedMessages.prototype, 'list') + lists.mockImplementation(function (this: JournalQueuedMessages) { + const caller = source(), + before = this['viewed'], + result = list.call(this) + const callers = (upstream[phase] ??= {}) + const row = (callers[caller] ??= { calls: 0, returned: 0, materialized: 0 }) + row.calls++ + row.returned += result.length + if (before !== this['viewed']) { + row.materialized += result.length + } + if (caller === 'selection') { + seam().lists++ + } + return result + }) + const derive = pause.deriveQueuePauses + const pauses = vi.spyOn(pause, 'deriveQueuePauses') + pauses.mockImplementation((...args) => { + if (source() === 'selection') { + seam().pauses++ + seam().waitingRows += args[0].cards.length + } + return derive(...args) + }) + const held = QueuedMessageEditLeases.prototype.heldIds + const leases = vi.spyOn(QueuedMessageEditLeases.prototype, 'heldIds') + leases.mockImplementation(function (this: QueuedMessageEditLeases, ...args) { + if (source() === 'selection') { + seam().leases++ + seam().leaseRows += args[0]?.length ?? 0 + } + return held.apply(this, args) + }) + const pick = pause.nextSendableQueuedCard + const picks = vi + .spyOn(pause, 'nextSendableQueuedCard') + .mockImplementation((pauses, rows, held) => { + const counting = source() === 'selection' + const counted = rows.slice() + counted[Symbol.iterator] = function* () { + for (const row of rows) { + if (counting) { + seam().headVisits++ + } + yield row + } + return undefined + } + return pick(pauses, counted, held) + }) + const predicate = pause.queuePauseHolding + const predicates = vi.spyOn(pause, 'queuePauseHolding').mockImplementation((...args) => { + if (source() === 'selection') { + seam().predicates++ + } + return predicate(...args) + }) + const get = JournalQueuedMessages.prototype.get + const gets = vi.spyOn(JournalQueuedMessages.prototype, 'get') + gets.mockImplementation(function (this: JournalQueuedMessages, ...args) { + if (selecting) { + seam().gets++ + } + return get.apply(this, args) + }) + const isolate = transitions.isolateJournalTransition + const prepared = vi + .spyOn(transitions, 'isolateJournalTransition') + .mockImplementation((...args) => { + const draft = isolate(...args) + return { + ...draft, + adopt: () => { + const state = draft.adopt() + metric().draftNodes += draft.tracked() - draft.trackedDerived() + metric().derivedDraftNodes += draft.trackedDerived() + return state + } + } + }) + const encode = outcomes.encodeJournalOutcomes + const encoded = vi.spyOn(outcomes, 'encodeJournalOutcomes').mockImplementation((...args) => { + const result = encode(...args) + for (const value of result.values) { + if (value.field === 'inputEntries') { + metric().nativeEntries++ + } + if (value.field === 'inputCards') { + metric().nativeCards++ + } + if (value.field === 'inputEntries' || value.field === 'inputCards') { + metric().nativeBytes += Buffer.byteLength(value.json) + } + } + return result + }) + const reflected = Reflect.get + const nodes = new WeakSet() + const nodeReads = vi.spyOn(Reflect, 'get').mockImplementation((...args) => { + const [target] = args + // The prepared draft reads current objects through this seam. Observe rather than + // installing accessors on transaction nodes, which would mutate the measured graph. + if (Object.hasOwn(target, 'height') && Object.hasOwn(target, 'entry')) { + metric().nodeReads++ + if (!nodes.has(target)) { + nodes.add(target) + metric().nodes++ + } + } + return reflected(...args) + }) + + const boundaryCalls: Record = {} + const predecessor = JournalInputOrder.prototype.predecessors + const predecessors = vi + .spyOn(JournalInputOrder.prototype, 'predecessors') + .mockImplementation(function (this: JournalInputOrder, ...args) { + boundaryCalls[phase] = (boundaryCalls[phase] ?? 0) + 1 + return predecessor.apply(this, args) + }) + const set = JournalInputPlaceIndex.prototype.set + const indexSet = vi.spyOn(JournalInputPlaceIndex.prototype, 'set').mockImplementation(function ( + this: JournalInputPlaceIndex, + ...args + ) { + metric().sets++ + set.apply(this, args) + }) + const query = JournalInputPlaceIndex.prototype.query + const indexQuery = vi + .spyOn(JournalInputPlaceIndex.prototype, 'query') + .mockImplementation(function (this: JournalInputPlaceIndex, ...args) { + metric().queries++ + return query.apply(this, args) + }) + const domainSet = JournalInputDomainIndex.prototype.set + const domain = vi.spyOn(JournalInputDomainIndex.prototype, 'set').mockImplementation(function ( + this: JournalInputDomainIndex, + ...args + ) { + metric().domains++ + return domainSet.apply(this, args) + }) + const change = JournalInputLineageOwner.prototype.change + const line = vi + .spyOn(JournalInputLineageOwner.prototype, 'change') + .mockImplementation(function (this: JournalInputLineageOwner, ...args) { + metric().lineage++ + return change.apply(this, args) + }) + const receipt = receiptTable.readCommandReceipt + const receipts = vi.spyOn(receiptTable, 'readCommandReceipt').mockImplementation((...args) => { + metric().receipts++ + return receipt(...args) + }) + const before = JournalInputOrder.prototype.beforeRow + const rows = vi.spyOn(JournalInputOrder.prototype, 'beforeRow').mockImplementation(function ( + this: JournalInputOrder, + state, + row + ) { + metric().rows++ + const values = state.queue.cards.values.bind(state.queue.cards) + const visits = vi.spyOn(state.queue.cards, 'values').mockImplementation(function* () { + for (const card of values()) { + metric().queueVisits++ + yield card + } + return undefined + }) + try { + return before.call(this, state, row) + } finally { + visits.mockRestore() + } + }) + const after = JournalInputOrder.prototype.afterRow + const folded = vi.spyOn(JournalInputOrder.prototype, 'afterRow').mockImplementation(function ( + this: JournalInputOrder, + state, + row + ) { + const values = state.submissions.values.bind(state.submissions) + const visits = vi.spyOn(state.submissions, 'values').mockImplementation(function* () { + for (const input of values()) { + metric().inputVisits++ + yield input + } + return undefined + }) + try { + return after.call(this, state, row) + } finally { + visits.mockRestore() + } + }) + const rig = await createQueuedMessageTestRig({ + restartable: true, + probeOwner: async () => ({ outcome: 'pid-absent' }) + }) + try { + const w = await rig.workingSend() + await openRigTurnFor(rig, w) + const { serialize, sessions } = rig.host.collaboratorsForTests() + const journal = sessions.get(SESSION)!.journal + const q = Array.from({ length: n }, (_, i) => `Q${i}`) + await serialize(SESSION, async () => { + for (const [i, id] of q.entries()) { + await journal.queuedMessages.insert({ + messageId: id, + body: + i === 0 + ? { ...hostTestMessage('/compact'), command: { name: 'compact' } } + : hostTestMessage(id) + }) + } + for (let i = 1; i < n; i++) { + if (i % 3 === 0) { + continue + } + const card = journal.queuedMessages.get(q[i])! + await journal.queuedMessages['write'](() => ({ + changes: [ + { + messageId: card.messageId, + change: { + op: 'queued', + body: card.body, + position: card.position, + queuedAt: i % 3 === 1 ? null : { epoch: 'imported-domain', sequence: 1 } + } + } + ], + result: undefined + })) + } + await journal.queuedMessages.insert({ + messageId: 'E', + body: hostTestMessage('held'), + holdReason: QUEUED_MESSAGE_PAUSED_SEND_FAILED + }) + }) + phase = 'edits-growth' + const ids: string[] = [] + for (let i = 0; i < n; i++) { + await serialize(SESSION, async () => { + for (const id of ['E', 'Q1']) { + expect( + await journal.queuedMessages.update({ + messageId: id, + expectedBodyFingerprint: journal.queuedMessages.get(id)!.fingerprint, + text: `${id}-edit-${i}`, + images: [] + }) + ).toMatchObject({ status: 'updated' }) + } + await journal.queuedMessages.insert({ + messageId: `D${i}`, + body: hostTestMessage(`D${i}`) + }) + }) + const m = rig.send(`M${i}`) + ids.push(m.id) + expect(await m.result).toMatchObject({ ok: true }) + expect(journal.inputOrder.forced(m.id)).toBe(true) + expect( + journal.inputOrder.precedes( + journal.queuedMessages.get(`D${i}`)!, + journal.submission(m.id)! + ) + ).toBe(true) + if (i) { + expect( + journal.inputOrder.precedes( + journal.queuedMessages.get(`D${i}`)!, + journal.submission(ids[i - 1])! + ) + ).toBe(false) + } + } + await serialize(SESSION, () => + journal.queuedMessages + .insert({ messageId: 'L', body: hostTestMessage('later sentinel') }) + .then(() => {}) + ) + expect(metric().queueVisits).toBe(0) + expect(metric().inputVisits).toBe(0) + expect(metric().receipts).toBe(n) // Existing direct acceptance receipt read, no ordering probe. + // Two coordinate indexes, one point per new card; body-only edits never update an order index. + expect(metric().sets).toBe(2 * (n + 1)) + expect(metric().nativeEntries).toBe(n) + expect(metric().nativeCards).toBe(n + 1) + // Four AVL point paths (all/domain in two coordinate indexes), at most + // twice log height, reaching at most eight draft objects per level. + expect(metric().derivedDraftNodes).toBeLessThanOrEqual( + 64 * (n + 1) * Math.ceil(Math.log2(2 * n + 3)) + ) + expect(metric().nodeReads).toBeLessThanOrEqual( + 200 * (n + 1) * Math.ceil(Math.log2(2 * n + 3)) + ) + expect(journal.queuedMessages.displayState().inputEntries.size).toBe(n + 1) + expect(journal.inputOrder.lineage.cards.size).toBe(2 * n + 2) + expect(domain).not.toHaveBeenCalled() + for (let i = 0; i < n; i++) { + const key = { callerKey: 'projection', editId: `lease-${i}` } + expect( + journal.queuedMessages.editLeases.acquire(key, { + messageId: 'Q0', + fingerprint: journal.queuedMessages.get('Q0')!.fingerprint + }).status + ).toBe('held') + expect(selectStructuredAgentSessionInput(journal).kind).toBe('none') + journal.queuedMessages.editLeases.release(key, 'Q0') + expect(selectStructuredAgentSessionInput(journal)).toMatchObject({ + kind: 'card', + card: { messageId: 'Q0' } + }) + } + rig.host.stopDelivery() + phase = 'original-replay' + expect(loadTestJournal(rig.root, SESSION)?.damage).toBeNull() + expect(metric().receipts).toBe(0) + for (let pass = 0; pass < 2; pass++) { + phase = `replacement-${pass}` + await serialize(SESSION, () => + journal.replaceEpochItems('unreconcilable_prefix', 1, []).then(() => {}) + ) + expect(metric().rows).toBe(2 * (3 * n + 3)) // Builder and isolated publication each fold the same actual cohort. + expect(metric().receipts).toBe(n) + expect( + [...journal.queuedMessages.displayState().inputCards.values()].filter( + (card) => card.baseline !== undefined + ).length + ).toBeLessThanOrEqual(2 * n + 4) + phase = `output-replay-${pass}` + const replay = loadTestJournal(rig.root, SESSION)! + expect(replay.damage).toBeNull() + expect(replay.newer).toBeNull() + expect(metric().rows).toBe(0) // Keyed restore assigns canonical facts and rebuilds derived indexes once. + expect(ids.every((id) => journalInputOrder(replay.state).forced(id))).toBe(true) + } + phase = 'recovery' + await rig.restartHostProcess() + for (const id of ids) { + expect(await rig.submission(id)).toMatchObject({ + dispatchState: 'rejected', + keptAsQueuedMessageId: id + }) + } + const reopened = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + const expected = [...q, 'E', ...ids.flatMap((id, i) => [`D${i}`, id]), 'L'] + const cards = reopened.queuedMessages.list().filter((card) => card.state === 'waiting') + expect(cards.map((card) => card.messageId)).toEqual(expected) + expect( + cards.every( + (card, i) => + Number.isFinite(card.position) && (i === 0 || card.position > cards[i - 1].position) + ) + ).toBe(true) + expect(reopened.queuedMessages.get('E')?.holdReason).toBe(QUEUED_MESSAGE_PAUSED_SEND_FAILED) + expect(reopened.queuedMessages.get('Q1')?.body).toEqual(hostTestMessage(`Q1-edit-${n - 1}`)) + expect(reopened.queuedMessages.displayState().inputEntries.size).toBe(0) + // A real held-card retry releases E at its existing selected Queue place. + await rig.host.collaboratorsForTests().serialize(SESSION, () => + reopened.queuedMessages['write'](() => ({ + changes: [ + { + messageId: 'E', + change: { + op: 'queued', + body: reopened.queuedMessages.get('E')!.body, + position: reopened.queuedMessages.get('E')!.position, + queuedAt: reopened.queuedMessages.get('E')!.queuedAt + } + } + ], + result: undefined + })) + ) + phase = 'delivery' + const base = rig.dispatch.mock.calls.length + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + await eventually(() => expect(rig.compact).toHaveBeenCalledOnce()) + const command = rig.compact.mock.calls[0]![0].command + rig.finishCompact() + await eventually(() => + expect(reopened.item(agentJournalItemKey(command.identity))?.body).toMatchObject({ + kind: 'turn', + state: 'completed' + }) + ) + const actual = ['Q0'] + for (let i = 0; i < expected.length - 1; i++) { + await eventually(() => expect(rig.dispatch.mock.calls).toHaveLength(base + i + 1)) + const input = rig.dispatch.mock.calls[base + i]![0] + const submission = await rig.submission(input.clientMessageId) + actual.push(submission!.queuedMessageId!) + if (submission!.queuedMessageId === 'Q1') { + expect(input.body).toEqual(hostTestMessage(`Q1-edit-${n - 1}`)) + } + await rig.settleAccepted(input.clientMessageId, `actual-${i}`) + expect(await rig.submission(input.clientMessageId)).toMatchObject({ + dispatchState: 'accepted' + }) + } + expect(actual).toEqual(expected) + expect(new Set(actual).size).toBe(expected.length) + expect(reopened.queuedMessages.list().filter((card) => card.state === 'waiting')).toEqual([]) + expect(reopened.inputOrder.unsettledSubmissions()).toEqual([]) + expect( + rig.dispatch.mock.calls.slice(base).some(([input]) => input.clientMessageId === w) + ).toBe(false) + expect(reopened.queuedMessages.displayState().inputEntries.size).toBe(0) + expect(reopened.queuedMessages.editLeases.heldIds().size).toBe(0) + // Each folded card transition updates two current trees; recovery/pause adds a bounded bulk pass. + // A set removes/inserts in two AVLs; rotations and summary refreshes cost a fixed number of node reads per level. + for (const [name, count] of Object.entries(phases)) { + expect(count.nodeReads, name).toBeLessThanOrEqual( + 200 * (count.sets + count.queries + 1) * Math.ceil(Math.log2(4 * n + 8)) + ) + expect(count.nodes, name).toBeLessThanOrEqual(count.nodeReads) + } + expect(phases['original-replay'].sets).toBeLessThanOrEqual(6 * n + 16) + for (let pass = 0; pass < 2; pass++) { + for (const name of [`replacement-${pass}`, `output-replay-${pass}`]) { + // The builder and isolated publisher each initialize two trees from K cards; + // their first card also takes the ordinary affected-ID update. Keyed replay builds once. + expect(phases[name].sets, name).toBe( + name.startsWith('replacement') ? 4 * (2 * n + 3) : 2 * (2 * n + 2) + ) + // One fold query per admitted input; each actual consumer boundary call adds one point query. + expect(phases[name].queries, name).toBeLessThanOrEqual(n + (boundaryCalls[name] ?? 0)) + expect(phases[name].lineage, name).toBe( + name.startsWith('replacement') ? 2 * (2 * n + 2) : 0 + ) + } + } + // Per row, at most two affected-card refreshes of two indexes; two effective-pause passes may rebuild K. + expect(phases.recovery.rows).toBeLessThanOrEqual(8 * n + 12) + expect(phases.recovery.sets).toBeLessThanOrEqual(4 * phases.recovery.rows + 4 * (2 * n + 2)) + expect(phases.recovery.queries).toBe(n) // One predecessor query per pending input; keyed restore does not readmit it. + expect(phases.delivery.sets).toBeLessThanOrEqual(32 * n + 8) + expect(phases.delivery.queries).toBe(0) + console.info('current-consumer-evidence', JSON.stringify({ n, phases, consumers, upstream })) + for (const [name, count] of Object.entries(consumers)) { + expect(count.lists, name).toBe(0) + expect(count.pauses, name).toBe(0) + expect(count.leases, name).toBe(0) + expect(count.waitingRows, name).toBe(0) + expect(count.leaseRows, name).toBe(0) + expect(count.headVisits, name).toBe(0) + // Each head or effective-pause reindex evaluates at most one predicate per affected card. + expect(count.predicates, name).toBeLessThanOrEqual(count.heads + count.indexedCards) + // One latest-value get and one lease fingerprint get at most per head. + expect(count.gets, name).toBeLessThanOrEqual(2 * count.heads) + } + expect(consumers['edits-growth'].indexedCards).toBe(0) + expect(consumers['edits-growth'].heads).toBeGreaterThan(n) + console.info('current-consumer-counts', JSON.stringify({ n, consumers })) + console.info( + 'ordered-place-counts', + JSON.stringify({ + n, + phases, + domainCalls: domain.mock.calls.length, + lineageCalls: line.mock.calls.length, + complete: true + }) + ) + } finally { + await rig.dispose() + Error.stackTraceLimit = stackLimit + indexed.mockRestore() + picks.mockRestore() + predicates.mockRestore() + predecessors.mockRestore() + head.mockRestore() + lists.mockRestore() + pauses.mockRestore() + leases.mockRestore() + gets.mockRestore() + encoded.mockRestore() + prepared.mockRestore() + nodeReads.mockRestore() + indexSet.mockRestore() + indexQuery.mockRestore() + domain.mockRestore() + line.mockRestore() + receipts.mockRestore() + rows.mockRestore() + folded.mockRestore() + } + }, + 60_000 +) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-input-order-reopen.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-input-order-reopen.test.ts new file mode 100644 index 00000000000..5d4d93fd8de --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-input-order-reopen.test.ts @@ -0,0 +1,344 @@ +import { journalInputOrder } from '../agent-session-journal/journal-input-order' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { expect, it, vi } from 'vitest' +import { + createQueuedMessageTestRig, + eventually +} from './structured-agent-session-queued-message-rig.test-fixture' +import { openRigTurnFor } from './structured-agent-session-queued-rig-turn.test-fixture' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { + HOST_TEST_SESSION as SESSION, + hostTestMessage +} from './structured-agent-session-host-test-data' +import { loadTestJournal } from '../agent-session-journal/journal-host-database-test-support' +import { queuePauseHolding } from '../agent-session-journal/queued-message-pause' + +it.each([false, true])( + 'failed reopen=%s: unentered M retains ordinary head recovery through graceful restart', + async (fail) => { + const rig = await createQueuedMessageTestRig({ + restartable: true, + probeOwner: async () => ({ outcome: 'pid-absent' }) + }) + let release = () => {} + try { + const warm = await rig.workingSend() + await rig.settleAccepted(warm, 'warm') + const { sessions, serialize, conversationDelivery } = rig.host.collaboratorsForTests() + const journal = sessions.get(SESSION)!.journal + await serialize(SESSION, async () => { + await journal.queuedMessages.insert({ + messageId: 'C', + body: { ...hostTestMessage('/compact'), command: { name: 'compact' } } + }) + const mark = fail + ? vi + .spyOn(journal, 'appendQueueReopen') + .mockRejectedValueOnce(new Error('injected reopen write failure')) + : null + await journal.markQueueReopen(1).catch(() => {}) + mark?.mockRestore() + }) + const n = rig.send('N opens work without an accepted echo') + expect(await n.result).toMatchObject({ ok: true }) + await eventually(async () => expect((await rig.submission(n.id))?.handedOverAt).toBeDefined()) + await openRigTurnFor(rig, n.id) + expect(journal.queuedMessages.pauses().map((p) => p.reason)).toEqual(['restarted']) + + const order = conversationDelivery.loop['deps'].deliveryOrder + const reserved = order.reserve(SESSION) + release = reserved.release + const before = order.beforeSend.bind(order) + const waiting = vi.spyOn(order, 'beforeSend').mockImplementation((...args) => before(...args)) + const effectiveEligible = journal.queuedMessages + .list() + .filter( + (card) => + card.state === 'waiting' && + card.holdReason === null && + !queuePauseHolding(journal.queuedMessages.pauses(), card) + ) + .map((card) => card.messageId) + expect(effectiveEligible).toEqual([]) + const m = rig.send('M ordinary steer while C is held, never entered', undefined, { + carried: { personality: 'friendly', effort: 'high' } + }) + expect(await m.result).toMatchObject({ ok: true }) + await serialize(SESSION, async () => {}) + await eventually(() => expect(waiting.mock.calls.some(([, id]) => id === m.id)).toBe(true)) + expect(await rig.submission(m.id)).toMatchObject({ dispatchState: 'pending' }) + expect((await rig.submission(m.id))?.handedOverAt).toBeUndefined() + expect(rig.dispatch.mock.calls.some(([input]) => input.clientMessageId === m.id)).toBe(false) + const original = journal.submission(m.id)! + const receipt = rig.store.readCommandReceipt({ kind: 'global' }, m.id) + expect(receipt).toMatchObject({ + verdict: 'readable', + receipt: { + status: 'accepted', + method: 'agentSession.send', + result: { kind: 'journal-row', epoch: journal.epoch, sequence: original.acceptedSequence } + } + }) + expect(original.options).toEqual({ personality: 'friendly', effort: 'high' }) + const replayBefore = loadTestJournal(rig.root, SESSION)! + expect(replayBefore.damage).toBeNull() + expect(replayBefore.newer).toBeNull() + expect(journal.inputOrder.forced(m.id)).toBe(false) + expect(journalInputOrder(replayBefore.state).forced(m.id)).toBe(false) + waiting.mockRestore() + + await rig.restartHostProcess() + const replayAfter = loadTestJournal(rig.root, SESSION)! + expect(replayAfter.damage).toBeNull() + expect(replayAfter.newer).toBeNull() + const shutdownM = replayAfter.state.submissions.get(m.id) + expect(shutdownM?.handedOverAt).toBeUndefined() + const recoveredM = await rig.submission(m.id) + expect(recoveredM).toMatchObject({ dispatchState: 'rejected', keptAsQueuedMessageId: m.id }) + expect(recoveredM?.handedOverAt).toBeUndefined() + expect(recoveredM?.options).toEqual(original.options) + expect(recoveredM?.payloadFingerprint).toBe(original.payloadFingerprint) + expect(recoveredM?.acceptedSequence).toBe(original.acceptedSequence) + const reopened = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + expect(rig.store.readCommandReceipt({ kind: 'global' }, m.id)).toEqual(receipt) + expect(reopened.queuedMessages.get(m.id)).toMatchObject({ + body: hostTestMessage('M ordinary steer while C is held, never entered'), + options: original.options + }) + const cards = reopened.queuedMessages.list().filter((c) => c.state === 'waiting') + const observedOrder = cards.map((c) => c.messageId) + expect(observedOrder).toEqual([m.id, 'C']) + expect(rig.compact).not.toHaveBeenCalled() + const deliveries: string[] = [] + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + for (const id of observedOrder) { + if (id === 'C') { + await eventually(() => expect(rig.compact).toHaveBeenCalledOnce()) + deliveries.push('C') + const command = rig.compact.mock.calls[0]![0].command + rig.finishCompact() + await eventually(() => + expect(reopened.item(agentJournalItemKey(command.identity))?.body).toMatchObject({ + kind: 'turn', + state: 'completed' + }) + ) + } else { + await eventually(async () => + expect((await rig.handoff(m.id))?.handedOverAt).toBeDefined() + ) + deliveries.push('M') + const handoff = await rig.handoffId(m.id) + expect(rig.compact).not.toHaveBeenCalled() + await rig.settleAccepted(handoff, 'M-after-restart') + expect(await rig.submission(handoff)).toMatchObject({ dispatchState: 'accepted' }) + } + } + expect(deliveries).toEqual(['M', 'C']) + } finally { + release() + await rig.dispose() + } + } +) + +it.each([false, true])( + 'reopen mark failure=%s keeps a paused command from forcing a later steer', + async (fail) => { + const rig = await createQueuedMessageTestRig() + try { + const warm = await rig.workingSend() + await rig.settleAccepted(warm, 'warm') + const { sessions, serialize } = rig.host.collaboratorsForTests() + const journal = sessions.get(SESSION)!.journal + await serialize(SESSION, async () => { + await journal.queuedMessages.insert({ + messageId: 'C', + body: { ...hostTestMessage('/compact'), command: { name: 'compact' } } + }) + const mark = fail + ? vi.spyOn(journal, 'appendQueueReopen').mockRejectedValueOnce(new Error('disk full')) + : null + await journal.markQueueReopen(1).catch(() => {}) + mark?.mockRestore() + }) + expect(journal.queuedMessages.pauses().map((p) => p.reason)).toEqual(['restarted']) + const n = rig.send('N opens next work') + expect(await n.result).toMatchObject({ ok: true }) + await eventually(async () => expect((await rig.submission(n.id))?.handedOverAt).toBeDefined()) + await openRigTurnFor(rig, n.id) + expect(journal.queuedMessages.pauses().map((p) => p.reason)).toEqual(['restarted']) + const m = rig.send('M steer while C is held') + expect(await m.result).toMatchObject({ + ok: true, + value: { submission: { clientMessageId: m.id } } + }) + await serialize(SESSION, async () => {}) + expect(journal.inputOrder.forced(m.id)).toBe(false) + expect(rig.compact).not.toHaveBeenCalled() + await eventually(async () => expect((await rig.submission(m.id))?.handedOverAt).toBeDefined()) + await rig.settleAccepted(n.id, 'N') + expect(journal.queuedMessages.pauses()).toEqual([]) + const later = rig.send('later admission after actual accepted N') + expect(await later.result).toMatchObject({ ok: true }) + expect(journal.inputOrder.forced(later.id)).toBe(true) + expect( + journal.inputOrder.precedes(journal.queuedMessages.get('C')!, journal.submission(later.id)!) + ).toBe(true) + await rig.settleAccepted(m.id, 'M') + await openRigTurnFor(rig, n.id, 'interrupted') + await eventually(() => expect(rig.compact).toHaveBeenCalledOnce()) + const command = rig.compact.mock.calls[0]![0].command + expect(journal.submission(later.id)?.handedOverAt).toBeUndefined() + rig.finishCompact() + await eventually(() => + expect(journal.item(agentJournalItemKey(command.identity))?.body).toMatchObject({ + kind: 'turn', + state: 'completed' + }) + ) + await eventually(async () => + expect((await rig.submission(later.id))?.handedOverAt).toBeDefined() + ) + await rig.settleAccepted(later.id, 'later') + for (const id of [n.id, m.id, later.id]) { + expect(journal.submission(id)?.dispatchState).toBe('accepted') + expect( + rig.dispatch.mock.calls.filter(([input]) => input.clientMessageId === id) + ).toHaveLength(1) + } + } finally { + await rig.dispose() + } + } +) + +// Two host restarts/replacements and four ordered handoffs need the extended budget. +it('keeps an earlier-process recovery identity through two adoptions after both writes fail, then delivers the observed C,N,M place', async () => { + const rig = await createQueuedMessageTestRig({ + restartable: true, + probeOwner: async () => ({ outcome: 'pid-absent' }) + }) + const resolve = AgentSessionJournal.prototype.resolveDispatch + let blocked = true + let older = '' + const warned = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const failure = vi + .spyOn(AgentSessionJournal.prototype, 'resolveDispatch') + .mockImplementation(function (this: AgentSessionJournal, ...args) { + return blocked && args[0].clientMessageId === older + ? Promise.reject(new Error('both recovery writes unavailable')) + : resolve.apply(this, args) + }) + let release = () => {} + try { + const w = await rig.workingSend() + await openRigTurnFor(rig, w) + const initial = rig.host.collaboratorsForTests() + const reservation = initial.conversationDelivery.loop['deps'].deliveryOrder.reserve(SESSION) + release = reservation.release + const o = rig.send('older O') + older = o.id + expect(await o.result).toMatchObject({ ok: true }) + expect((await rig.submission(older))?.handedOverAt).toBeUndefined() + await initial.serialize(SESSION, async () => { + const journal = initial.sessions.get(SESSION)!.journal + await journal.queuedMessages.insert({ + messageId: 'C', + body: { ...hostTestMessage('/compact'), command: { name: 'compact' } } + }) + await journal.queuedMessages['write'](() => ({ + changes: [ + { + messageId: 'N', + change: { op: 'queued', body: hostTestMessage('null N'), position: 2, queuedAt: null } + } + ], + result: undefined + })) + }) + await rig.restartHostProcess() + expect(await rig.submission(older)).toMatchObject({ dispatchState: 'pending' }) + rig.host.stopDelivery() + const { serialize, sessions } = rig.host.collaboratorsForTests() + const journal = sessions.get(SESSION)!.journal + expect(journal.inputOrder.recoveryPending(older)).toBe(true) + const fence = journal.queuedMessages.displayState().highestFence + await serialize(SESSION, () => journal.appendQueueResume(fence).then(() => {})) + const m = rig.send('fresh M') + expect(await m.result).toMatchObject({ ok: true }) + expect(journal.inputOrder.forced(m.id)).toBe(true) + expect( + journal.inputOrder.precedes(journal.queuedMessages.get('N')!, journal.submission(m.id)!) + ).toBe(true) + for (let pass = 0; pass < 2; pass++) { + await serialize(SESSION, () => + journal.replaceEpochItems('unreconcilable_prefix', fence, []).then(() => {}) + ) + expect(journal.inputOrder.recoveryPending(older)).toBe(true) + expect(journal.inputOrder.recoveryPending(m.id)).toBe(false) + expect( + journal.inputOrder.precedes(journal.queuedMessages.get('N')!, journal.submission(m.id)!) + ).toBe(true) + const replay = loadTestJournal(rig.root, SESSION)! + expect(replay.damage).toBeNull() + expect( + journalInputOrder(replay.state).precedes( + replay.state.queue.cards.get('N')!, + replay.state.submissions.get(m.id)! + ) + ).toBe(true) + } + blocked = false + release() + await rig.restartHostProcess() + for (const id of [older, m.id]) { + expect(await rig.submission(id)).toMatchObject({ + dispatchState: 'rejected', + keptAsQueuedMessageId: id + }) + } + const reopened = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + expect( + reopened.queuedMessages + .list() + .filter((card) => card.state === 'waiting') + .map((card) => card.messageId) + ).toEqual([older, 'C', 'N', m.id]) + expect(reopened.inputOrder.recoveryPending(older)).toBe(false) + const base = rig.dispatch.mock.calls.length + expect(await rig.resume()).toMatchObject({ ok: true }) + const actual: string[] = [] + await eventually(() => expect(rig.dispatch.mock.calls).toHaveLength(base + 1)) + actual.push( + (await rig.submission(rig.dispatch.mock.calls[base]![0].clientMessageId))!.queuedMessageId! + ) + await rig.settleAccepted(rig.dispatch.mock.calls[base]![0].clientMessageId, 'O') + await eventually(() => expect(rig.compact).toHaveBeenCalledOnce()) + actual.push('C') + const command = rig.compact.mock.calls[0]![0].command + rig.finishCompact() + await eventually(() => + expect(reopened.item(agentJournalItemKey(command.identity))?.body).toMatchObject({ + kind: 'turn', + state: 'completed' + }) + ) + for (let i = 0; i < 2; i++) { + await eventually(() => expect(rig.dispatch.mock.calls).toHaveLength(base + i + 2)) + const input = rig.dispatch.mock.calls[base + i + 1]![0] + actual.push((await rig.submission(input.clientMessageId))!.queuedMessageId!) + await rig.settleAccepted(input.clientMessageId, `after-C-${i}`) + } + expect(actual).toEqual([older, 'C', 'N', m.id]) + expect(new Set(actual).size).toBe(4) + expect(reopened.queuedMessages.list().filter((card) => card.state === 'waiting')).toEqual([]) + } finally { + blocked = false + release() + failure.mockRestore() + warned.mockRestore() + await rig.dispose() + } +}, 60_000) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-input-order-rewind.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-input-order-rewind.test.ts new file mode 100644 index 00000000000..4aa3345ce6f --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-input-order-rewind.test.ts @@ -0,0 +1,119 @@ +import { it, expect, vi } from 'vitest' +import { + createQueuedMessageTestRig, + eventually, + QUEUED_RIG_CALLER as CALLER +} from './structured-agent-session-queued-message-rig.test-fixture' +import { + HOST_TEST_SESSION as SESSION, + hostTestOperationId +} from './structured-agent-session-host-test-data' +import { + agentJournalItemKey, + agentJournalSubmissionKey +} from '../../../shared/agent-session-journal-item-key' + +it('preserves existing queuedAt across two live replacements despite an older original receipt', async () => { + const rewind = vi.fn(async () => ({ ok: true as const })) + const rig = await createQueuedMessageTestRig({ restartable: true, rewind }) + try { + const first = await rig.workingSend() + await rig.settleAccepted(first, 'first') + const second = rig.send('second rewind target') + await second.result + await eventually(async () => + expect((await rig.submission(second.id))?.handedOverAt).toBeDefined() + ) + await rig.settleAccepted(second.id, 'second') + const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + const epoch1 = journal.epoch + const original = journal.replaceEpochItems.bind(journal) + const snapshots: unknown[] = [] + let command: Promise | undefined, rewindAgain: Promise | undefined + const c = hostTestOperationId() + const fields = { command: 'compact' as const, delivery: 'queue-if-active' as const } + const capture = () => ({ + epoch: journal.epoch, + submissions: journal.submissions().map((s) => ({ + id: s.clientMessageId, + seq: s.acceptedSequence, + handed: s.handedOverAt, + state: s.dispatchState + })), + cards: journal.queuedMessages.list(), + pauses: journal.queuedMessages.pauses() + }) + vi.spyOn(journal, 'replaceEpochItems').mockImplementation(async (...args) => { + snapshots.push(capture()) + const result = await original(...args) + snapshots.push(capture()) + if (snapshots.length === 2) { + // Real admissions queue before delivery resumes after the first replacement. + command = rig.host.conversationCommand(CALLER, { + ...fields, + envelope: rig.envelope(fields, 'agentSession.conversationCommand', c), + userSend: true + }) + const target = journal + .snapshot() + .items.find( + (item) => + item.body.kind === 'message' && + item.body.blocks.some( + (block) => block.type === 'text' && block.text === 'work on this' + ) + )! + const next = { itemId: target.itemId, expectedEpoch: journal.epoch } + rewindAgain = rig.host.rewind(CALLER, { + ...next, + envelope: rig.envelope(next, 'agentSession.rewind', hostTestOperationId()) + }) + } + return result + }) + const m = rig.send('M from E1') + const cut = { itemId: agentJournalSubmissionKey(second.id), expectedEpoch: epoch1 } + const rewound = rig.host.rewind(CALLER, { + ...cut, + envelope: rig.envelope(cut, 'agentSession.rewind', hostTestOperationId()) + }) + expect(await m.result).toMatchObject({ ok: true }) + expect(await rewound).toMatchObject({ ok: true }) + expect(await command).toMatchObject({ ok: true, value: { queued: { state: 'waiting' } } }) + const secondVerdict = await rewindAgain + expect(secondVerdict).toMatchObject({ ok: true }) + expect(snapshots).toHaveLength(4) + const receipt = rig.store.readCommandReceipt({ kind: 'global' }, m.id) + const card = journal.queuedMessages.get(c)! + expect(receipt).toMatchObject({ + verdict: 'readable', + receipt: { + status: 'accepted', + method: 'agentSession.send', + sessionId: SESSION, + result: { kind: 'journal-row', epoch: epoch1 } + } + }) + expect(card.queuedAt!.epoch).not.toBe(epoch1) + expect(card.queuedAt!.epoch).toBe(journal.epoch) + expect(card.queuedAt!.sequence).toBeGreaterThanOrEqual( + journal.submission(m.id)!.acceptedSequence! + ) + expect(journal.queuedMessages.pauses()).toEqual([]) + await eventually(async () => expect((await rig.submission(m.id))?.handedOverAt).toBeDefined()) + await rig.settleAccepted(m.id, 'M') + await eventually(() => expect(rig.compact).toHaveBeenCalledTimes(1)) + const compact = rig.compact.mock.calls[0]![0].command + rig.finishCompact() + await eventually(() => + expect(journal.item(agentJournalItemKey(compact.identity))?.body).toMatchObject({ + kind: 'turn', + state: 'completed' + }) + ) + expect(await rig.submission(m.id)).toMatchObject({ dispatchState: 'accepted' }) + } finally { + vi.restoreAllMocks() + await rig.dispose() + } +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-input-order.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-input-order.test.ts new file mode 100644 index 00000000000..2d1d92a97f6 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-input-order.test.ts @@ -0,0 +1,384 @@ +import { expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { structuredAgentSessionQueueRequest } from '../../../renderer/src/components/native-chat/structured-agent-session-queue-request' +import { + createQueuedMessageTestRig, + eventually, + QUEUED_RIG_CALLER +} from './structured-agent-session-queued-message-rig.test-fixture' +import { openRigTurnFor } from './structured-agent-session-queued-rig-turn.test-fixture' +import { + HOST_TEST_SESSION as SESSION, + hostTestOperationId +} from './structured-agent-session-host-test-data' + +async function queueCompact(rig: Awaited>) { + const id = hostTestOperationId() + const fields = { command: 'compact' as const, delivery: 'queue-if-active' as const } + expect( + await rig.host.conversationCommand(QUEUED_RIG_CALLER, { + envelope: rig.envelope(fields, 'agentSession.conversationCommand', id), + ...fields, + userSend: true + }) + ).toMatchObject({ ok: true, value: { queued: { messageId: id } } }) + return id +} + +it.each([ + { followUpsQueue: false, alternate: undefined }, + { followUpsQueue: true, alternate: true } +])( + 'fresh and stale publication both run C,D,M before later cards (Queue=$followUpsQueue)', + async ({ followUpsQueue, alternate }) => { + vi.useFakeTimers() + try { + for (const fresh of [true, false]) { + const rig = await createQueuedMessageTestRig() + try { + const working = await rig.workingSend() + const compactId = await queueCompact(rig) + const earlier = rig.send('earlier card', 'queue-if-active') + await earlier.result + const delivery = structuredAgentSessionQueueRequest( + { + capability: 'supported', + followUpsQueue, + isWorking: true, + stopping: false, + session: { + queuedMessages: fresh + ? [ + { + messageId: compactId, + state: 'waiting', + body: { + kind: 'message', + role: 'user', + blocks: [], + command: { name: 'compact' } + }, + position: 1 + } + ] + : [] + }, + prompts: [] + }, + alternate + ) + const later = rig.send('after compaction and the earlier card', delivery) + expect(await later.result).toMatchObject({ + ok: true, + value: fresh ? { queued: {} } : { submission: { clientMessageId: later.id } } + }) + const tail = rig.send('newer card', 'queue-if-active') + await tail.result + await vi.advanceTimersByTimeAsync(100) + expect(rig.dispatch).toHaveBeenCalledOnce() + expect(rig.compact).not.toHaveBeenCalled() + await rig.settleAccepted(working, 'first') + await eventually(() => expect(rig.compact).toHaveBeenCalledOnce()) + expect(rig.dispatch).toHaveBeenCalledOnce() + const command = rig.compact.mock.calls[0]![0].command + const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + rig.finishCompact() + await eventually(() => + expect(journal.item(agentJournalItemKey(command.identity))?.body).toMatchObject({ + kind: 'turn', + state: 'completed' + }) + ) + await eventually(async () => + expect((await rig.handoff(earlier.id))?.handedOverAt).toBeDefined() + ) + const earlierId = await rig.handoffId(earlier.id) + expect(rig.dispatch.mock.calls.at(-1)?.[0].clientMessageId).toBe(earlierId) + await rig.settleAccepted(earlierId, 'earlier') + await eventually(async () => + expect( + fresh + ? (await rig.handoff(later.id))?.handedOverAt + : (await rig.submission(later.id))?.handedOverAt + ).toBeDefined() + ) + const laterId = fresh ? await rig.handoffId(later.id) : later.id + expect(rig.dispatch.mock.calls.at(-1)?.[0].clientMessageId).toBe(laterId) + expect(await rig.handoff(tail.id)).toBeUndefined() + await rig.settleAccepted(laterId, 'later') + expect(await rig.submission(laterId)).toMatchObject({ dispatchState: 'accepted' }) + await eventually(async () => + expect((await rig.handoff(tail.id))?.handedOverAt).toBeDefined() + ) + expect(rig.dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([ + working, + earlierId, + laterId, + await rig.handoffId(tail.id) + ]) + } finally { + await rig.dispose() + } + } + } finally { + vi.useRealTimers() + } + } +) + +it('explicit card Send after handover entry preserves the committed input then card, once each', async () => { + vi.useFakeTimers() + const rig = await createQueuedMessageTestRig() + try { + const working = await rig.workingSend() + await openRigTurnFor(rig, working) + const card = rig.send('explicit card', 'queue-if-active') + await card.result + const prepared = Promise.withResolvers() + const release = Promise.withResolvers() + const writes: string[] = [] + let held = false + rig.dispatch.mockImplementation(async (input) => { + if ( + !held && + input.body.blocks.some((block) => block.type === 'text' && block.text === 'preparing input') + ) { + held = true + prepared.resolve() + await release.promise + } + writes.push(input.clientMessageId) + return { state: 'admitted' } + }) + const message = rig.send('preparing input') + await message.result + await prepared.promise + expect((await rig.submission(message.id))?.handedOverAt).toBeDefined() + const sent = await rig.sendNow(card.id) + expect(sent).toMatchObject({ ok: true, value: { submission: { queuedMessageId: card.id } } }) + release.resolve() + await eventually(async () => expect((await rig.handoff(card.id))?.handedOverAt).toBeDefined()) + const cardId = await rig.handoffId(card.id) + await eventually(() => expect(writes).toEqual([message.id, cardId])) + expect(rig.dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([ + working, + message.id, + cardId + ]) + await rig.settleAccepted(cardId, 'card') + await rig.settleAccepted(message.id, 'message') + await rig.settleAccepted(working, 'working') + expect(await rig.submission(message.id)).toMatchObject({ dispatchState: 'accepted' }) + } finally { + await rig.dispose() + vi.useRealTimers() + } +}) + +it('removing C and explicitly sending D keeps forced M queued until W actually ends', async () => { + vi.useFakeTimers() + const rig = await createQueuedMessageTestRig() + try { + const working = await rig.workingSend() + await openRigTurnFor(rig, working) + const compactId = await queueCompact(rig) + const earlier = rig.send('D', 'queue-if-active') + await earlier.result + const message = rig.send('forced M without delivery') + await message.result + await rig.deleteQueued(compactId) + await rig.sendNow(earlier.id) + await eventually(async () => + expect((await rig.handoff(earlier.id))?.handedOverAt).toBeDefined() + ) + await rig.settleAccepted(await rig.handoffId(earlier.id), 'explicit-D') + await vi.advanceTimersByTimeAsync(100) + expect((await rig.submission(message.id))?.handedOverAt).toBeUndefined() + expect(await rig.submission(message.id)).toMatchObject({ dispatchState: 'pending' }) + expect(rig.compact).not.toHaveBeenCalled() + await rig.settleAccepted(working, 'W') + await openRigTurnFor(rig, working, 'interrupted') + await eventually(async () => + expect((await rig.submission(message.id))?.handedOverAt).toBeDefined() + ) + await rig.settleAccepted(message.id, 'M') + expect(await rig.submission(message.id)).toMatchObject({ dispatchState: 'accepted' }) + } finally { + await rig.dispose() + vi.useRealTimers() + } +}) + +it('reselects after the external reservation wait without rejecting the original pending input', async () => { + const rig = await createQueuedMessageTestRig() + let release = () => {} + try { + const working = await rig.workingSend() + await openRigTurnFor(rig, working) + const card = rig.send('explicit before entry', 'queue-if-active') + await card.result + const { conversationDelivery, serialize } = rig.host.collaboratorsForTests() + const order = conversationDelivery.loop['deps'].deliveryOrder + const held = order.reserve(SESSION) + release = held.release + const entered = Promise.withResolvers() + const before = order.beforeSend.bind(order) + const waiting = vi.spyOn(order, 'beforeSend').mockImplementation((...args) => { + entered.resolve() + return before(...args) + }) + const message = rig.send('original M still pending') + await message.result + await entered.promise + expect(await rig.submission(message.id)).toMatchObject({ dispatchState: 'pending' }) + expect((await rig.submission(message.id))?.handedOverAt).toBeUndefined() + await rig.sendNow(card.id) + await serialize(SESSION, async () => {}) + release() + await eventually(async () => expect((await rig.handoff(card.id))?.handedOverAt).toBeDefined()) + const cardId = await rig.handoffId(card.id) + await eventually(async () => + expect((await rig.submission(message.id))?.handedOverAt).toBeDefined() + ) + expect(rig.dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([ + working, + cardId, + message.id + ]) + await rig.settleAccepted(cardId, 'card') + await rig.settleAccepted(message.id, 'M') + expect(await rig.submission(message.id)).toMatchObject({ dispatchState: 'accepted' }) + waiting.mockRestore() + } finally { + release() + await rig.dispose() + } +}) + +it('streams N and 10N frames without rescanning queue/history or waking a forced input', async () => { + const rig = await createQueuedMessageTestRig() + try { + const working = await rig.workingSend() + await openRigTurnFor(rig, working) + const compactId = await queueCompact(rig) + const message = rig.send('forced M') + await message.result + const { sessions, conversationDelivery, serialize } = rig.host.collaboratorsForTests() + const journal = sessions.get(SESSION)!.journal + const scope = { kind: 'thread' as const } + for (let index = 0; index < 100; index++) { + await journal.appendItem( + { provider: 'orca', clientMessageId: `old-${index}` }, + { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'history' }] }, + { fence: 1, turnScope: scope } + ) + } + await serialize(SESSION, async () => {}) + const pauses = vi.spyOn(journal.queuedMessages, 'pauses') + const visits = vi.spyOn(journal, 'visitItems') + const wake = vi.spyOn(conversationDelivery.loop, 'wake') + let revision = 0 + for (const count of [10, 100]) { + pauses.mockClear() + visits.mockClear() + wake.mockClear() + for (let index = 0; index < count; index++) { + rig.providerEvents().appendItem( + { provider: 'orca', clientMessageId: 'stream' }, + { + kind: 'message', + role: 'assistant', + blocks: [{ type: 'text', text: `token-${revision++}` }] + }, + { turnScope: scope } + ) + await rig.host.flushStreamedEvents(SESSION) + await serialize(SESSION, async () => {}) + } + expect(pauses).not.toHaveBeenCalled() + expect(visits).not.toHaveBeenCalled() + expect(wake).not.toHaveBeenCalled() + expect(journal.submission(message.id)?.handedOverAt).toBeUndefined() + } + pauses.mockRestore() + visits.mockRestore() + wake.mockRestore() + await rig.settleAccepted(working, 'W') + await openRigTurnFor(rig, working, 'interrupted') + await eventually(() => expect(rig.compact).toHaveBeenCalledOnce()) + rig.finishCompact() + await eventually(async () => + expect((await rig.submission(message.id))?.handedOverAt).toBeDefined() + ) + await rig.settleAccepted(message.id, 'M') + expect(await rig.handoff(compactId)).toMatchObject({ dispatchState: 'accepted' }) + expect(await rig.submission(message.id)).toMatchObject({ dispatchState: 'accepted' }) + } finally { + await rig.dispose() + } +}) + +it.each(['resume', 'new-send'] as const)( + 'restart holds C,D,M in their place until %s releases the queue', + async (release) => { + const rig = await createQueuedMessageTestRig({ + restartable: true, + probeOwner: async () => ({ outcome: 'pid-absent' }) + }) + try { + await rig.workingSend() + const compactId = await queueCompact(rig) + const earlier = rig.send('D before held M', 'queue-if-active') + await earlier.result + const message = rig.send('M kept after restart') + await message.result + const tail = rig.send('later card', 'queue-if-active') + await tail.result + await rig.restartHostProcess() + expect(await rig.submission(message.id)).toMatchObject({ + dispatchState: 'rejected', + keptAsQueuedMessageId: message.id + }) + expect((await rig.drafts()).map((card) => card.messageId)).toEqual([ + compactId, + earlier.id, + message.id, + tail.id + ]) + expect(rig.compact).not.toHaveBeenCalled() + expect(await rig.handoff(message.id)).toBeUndefined() + if (release === 'resume') { + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + } else { + const next = rig.send('N releases reopen only when accepted') + expect(await next.result).toMatchObject({ ok: true }) + await eventually(async () => + expect((await rig.submission(next.id))?.handedOverAt).toBeDefined() + ) + expect(rig.compact).not.toHaveBeenCalled() + await rig.settleAccepted(next.id, 'N') + } + await eventually(() => expect(rig.compact).toHaveBeenCalledOnce()) + const command = rig.compact.mock.calls[0]![0].command + rig.finishCompact() + const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + await eventually(() => + expect(journal.item(agentJournalItemKey(command.identity))?.body).toMatchObject({ + kind: 'turn', + state: 'completed' + }) + ) + for (const id of [earlier.id, message.id, tail.id]) { + await eventually(async () => expect((await rig.handoff(id))?.handedOverAt).toBeDefined()) + const handed = await rig.handoffId(id) + await rig.settleAccepted(handed, id) + expect(await rig.submission(handed)).toMatchObject({ dispatchState: 'accepted' }) + } + expect( + rig.dispatch.mock.calls.filter(([input]) => input.clientMessageId === message.id) + ).toHaveLength(0) + } finally { + await rig.dispose() + } + } +) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-input-recovery-frontier.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-input-recovery-frontier.test.ts new file mode 100644 index 00000000000..716326a1cf1 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-input-recovery-frontier.test.ts @@ -0,0 +1,297 @@ +import { journalInputOrder } from '../agent-session-journal/journal-input-order' +import { writeLegacyGapFixture } from './structured-agent-session-legacy-gap.test-support' +import type { JournalReducerState } from '../agent-session-journal/journal-reducer' +import { expect, it, vi } from 'vitest' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + loadTestJournal, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { selectStructuredAgentSessionInput } from './structured-agent-session-input-selection' +import { + createQueuedMessageTestRig, + eventually +} from './structured-agent-session-queued-message-rig.test-fixture' +import { openRigTurnFor } from './structured-agent-session-queued-rig-turn.test-fixture' +import { + HOST_TEST_SESSION as SESSION, + hostTestMessage +} from './structured-agent-session-host-test-data' + +it.each([ + { mode: 'mixed', partial: false }, + { mode: 'mixed', partial: true }, + { mode: 'ordinary', partial: true }, + { mode: 'moved', partial: true }, + { mode: 'ordinary-first', partial: false } +])( + 'preserves personal recovery and committed prefix $mode partial=$partial', + async ({ mode, partial }) => { + const rig = await createQueuedMessageTestRig({ + restartable: true, + probeOwner: async () => ({ outcome: 'pid-absent' }) + }) + let release = () => {} + let blocked = '' + const resolve = AgentSessionJournal.prototype.resolveDispatch + const fault = vi + .spyOn(AgentSessionJournal.prototype, 'resolveDispatch') + .mockImplementation(function (this: AgentSessionJournal, ...args) { + return args[0].clientMessageId === blocked + ? Promise.reject(new Error('keep and fallback fail')) + : resolve.apply(this, args) + }) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + const w = await rig.workingSend() + await openRigTurnFor(rig, w) + const { sessions, serialize, conversationDelivery } = rig.host.collaboratorsForTests() + release = conversationDelivery.loop['deps'].deliveryOrder.reserve(SESSION).release + const journal = sessions.get(SESSION)!.journal + const mixed = mode === 'mixed' + const first = mode === 'ordinary-first' ? rig.send('M1') : undefined + await first?.result + await serialize(SESSION, async () => { + await journal.queuedMessages.insert({ + messageId: 'C', + body: + mixed || first + ? { ...hostTestMessage('/compact'), command: { name: 'compact' } } + : hostTestMessage('C') + }) + if (mixed || first) { + await journal.queuedMessages.insert({ messageId: 'D', body: hostTestMessage('D') }) + } + }) + const m1 = first ?? rig.send('M1') + await m1.result + if (mixed) { + await rig.deleteQueued('C') + } + const m2 = rig.send('M2') + await m2.result + expect(journal.inputOrder.forced(m1.id)).toBe(mixed) + expect(journal.inputOrder.forced(m2.id)).toBe(Boolean(first)) + expect(journal.submission(m1.id)?.handedOverAt).toBeUndefined() + expect(journal.submission(m2.id)?.handedOverAt).toBeUndefined() + if (partial) { + blocked = m2.id + } + await rig.restartHostProcess() + release() + expect(await rig.submission(m1.id)).toMatchObject({ keptAsQueuedMessageId: m1.id }) + if (partial) { + expect(await rig.submission(m2.id)).toMatchObject({ dispatchState: 'pending' }) + const reopened = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + expect( + reopened.queuedMessages + .list() + .filter((c) => c.state === 'waiting') + .map((c) => c.messageId) + ).toEqual(mixed ? ['D', m1.id] : [m1.id, 'C']) + if (mode === 'moved') { + expect( + await rig.host + .collaboratorsForTests() + .serialize(SESSION, () => + reopened.queuedMessages.move({ messageId: 'C', beforeMessageId: m1.id }) + ) + ).toMatchObject({ status: 'moved' }) + } + blocked = '' + await rig.restartHostProcess() + } + expect(await rig.submission(m2.id)).toMatchObject({ keptAsQueuedMessageId: m2.id }) + const reopened = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + const expected = mixed + ? ['D', m1.id, m2.id] + : first + ? [m1.id, 'C', 'D', m2.id] + : mode === 'moved' + ? ['C', m1.id, m2.id] + : [m1.id, m2.id, 'C'] + const cards = reopened.queuedMessages.list().filter((c) => c.state === 'waiting') + expect(cards.map((c) => c.messageId)).toEqual(expected) + expect( + cards.every( + (c, i) => Number.isFinite(c.position) && (i === 0 || c.position > cards[i - 1].position) + ) + ).toBe(true) + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + const actual: string[] = [] + for (const id of expected) { + if (first && id === 'C') { + await eventually(() => expect(rig.compact).toHaveBeenCalledOnce()) + rig.finishCompact() + actual.push(id) + continue + } + await eventually(async () => expect((await rig.handoff(id))?.handedOverAt).toBeDefined()) + const handoff = await rig.handoffId(id) + actual.push((await rig.submission(handoff))!.queuedMessageId!) + await rig.settleAccepted(handoff, `frontier-${actual.length}`) + expect(await rig.submission(handoff)).toMatchObject({ dispatchState: 'accepted' }) + } + expect(actual).toEqual(expected) + expect(reopened.queuedMessages.list().filter((c) => c.state === 'waiting')).toEqual([]) + expect(reopened.queuedMessages.displayState().inputEntries.size).toBe(0) + expect(rig.dispatch.mock.calls.filter(([i]) => i.clientMessageId === w)).toHaveLength(1) + } finally { + blocked = '' + release() + fault.mockRestore() + warn.mockRestore() + await rig.dispose() + } + } +) + +it('retains an automatic unentered link through two production replacements and its real reservation release', async () => { + const rig = await createQueuedMessageTestRig() + let release = () => {} + try { + const w = await rig.workingSend() + await openRigTurnFor(rig, w) + const { sessions, serialize, conversationDelivery } = rig.host.collaboratorsForTests() + const journal = sessions.get(SESSION)!.journal + await serialize(SESSION, async () => { + await journal.queuedMessages.insert({ + messageId: 'C', + body: { ...hostTestMessage('/compact'), command: { name: 'compact' } } + }) + await journal.queuedMessages.insert({ + messageId: 'D', + body: hostTestMessage('latest D'), + options: { effort: 'high' } + }) + }) + const m = rig.send('M') + await m.result + const receipt = rig.store.readCommandReceipt({ kind: 'global' }, m.id) + const intro = journal.inputOrder.lineage.cards.get('D')!.introduced + await rig.deleteQueued('C') + const reservation = conversationDelivery.loop['deps'].deliveryOrder.reserve(SESSION) + release = reservation.release + await rig.settleAccepted(w, 'W') + await openRigTurnFor(rig, w, 'interrupted') + await eventually(async () => expect((await rig.handoff('D'))?.dispatchState).toBe('pending')) + const h = await rig.handoffId('D') + expect((await rig.submission(h))?.handedOverAt).toBeUndefined() + for (let i = 0; i < 2; i++) { + await serialize(SESSION, () => + journal.replaceEpochItems('unreconcilable_prefix', 1, []).then(() => {}) + ) + expect(journal.inputOrder.lineage.cards.get('D')!.introduced).toBe(intro) + expect(journal.queuedMessages.get('D')).toMatchObject({ + state: 'dispatched', + consumedAs: h, + body: hostTestMessage('latest D'), + options: { effort: 'high' } + }) + expect(selectStructuredAgentSessionInput(journal)).toMatchObject({ + kind: 'submission', + submission: { clientMessageId: h } + }) + const replay = loadTestJournal(rig.root, SESSION)! + expect(replay.damage).toBeNull() + expect(journalInputOrder(replay.state).lineage.cards.get('D')!.introduced).toBe(intro) + expect( + journalInputOrder(replay.state).precedes( + replay.state.queue.cards.get('D')!, + replay.state.submissions.get(m.id)! + ) + ).toBe(true) + expect(rig.store.readCommandReceipt({ kind: 'global' }, m.id)).toEqual(receipt) + } + release() + await eventually(async () => expect((await rig.submission(h))?.handedOverAt).toBeDefined()) + await rig.settleAccepted(h, 'H') + await eventually(async () => expect((await rig.submission(m.id))?.handedOverAt).toBeDefined()) + await rig.settleAccepted(m.id, 'M') + expect(rig.dispatch.mock.calls.map(([i]) => i.clientMessageId)).toEqual([w, h, m.id]) + expect(await rig.submission(h)).toMatchObject({ dispatchState: 'accepted' }) + expect(await rig.submission(m.id)).toMatchObject({ dispatchState: 'accepted' }) + expect(journal.queuedMessages.displayState().inputEntries.size).toBe(0) + expect(loadTestJournal(rig.root, SESSION)!.state.inputEntries.size).toBe(0) + } finally { + release() + await rig.dispose() + } +}) + +it.each([10, 100])( + 'preserves initial legacy terminal gaps through replacement and actual recovery N=%i', + async (n) => { + const rig = await createQueuedMessageTestRig({ + restartable: true, + legacyInputs: true, + probeOwner: async () => ({ outcome: 'pid-absent' }) + }) + try { + const original = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + await rig.host.flushAllStreamedEvents() + writeLegacyGapFixture(rig.root, original, n) + const check = (state: JournalReducerState) => { + const m = state.submissions.get('legacy-M')! + expect.soft(journalInputOrder(state).forced('legacy-M')).toBe(true) + expect + .soft( + ['C', 'D'].map((id) => journalInputOrder(state).precedes(state.queue.cards.get(id)!, m)) + ) + .toEqual([true, true]) + } + check(loadTestJournal(rig.root, SESSION)!.state) + const journal = new AgentSessionJournal({ + identity: original['identity'], + database: openTestJournalHostDatabase(rig.root) + }) + await journal.open() + for (let pass = 0; pass < 2; pass++) { + await journal.replaceEpochItems('unreconcilable_prefix', 1, []) + check(journal.queuedMessages.displayState()) + check(loadTestJournal(rig.root, SESSION)!.state) + } + await journal.close() + rig.crashRestartHostProcess() + await rig.submission('legacy-M') + const reopened = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + const saved = reopened.queuedMessages + .list() + .filter((c) => c.state === 'waiting') + .map((c) => c.messageId) + expect.soft(saved).toEqual(['C', 'D', 'legacy-M']) + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + const delivered: string[] = [] + for (const id of saved) { + if (id === 'C') { + await eventually(() => expect(rig.compact).toHaveBeenCalledOnce()) + rig.finishCompact() + delivered.push(id) + continue + } + await eventually(async () => expect((await rig.handoff(id))?.handedOverAt).toBeDefined()) + const h = await rig.handoffId(id) + await rig.settleAccepted(h, id) + delivered.push(id) + expect(await rig.submission(h)).toMatchObject({ + dispatchState: 'accepted', + queuedMessageId: id + }) + } + expect.soft(delivered).toEqual(['C', 'D', 'legacy-M']) + expect + .soft( + rig.dispatch.mock.calls.map( + ([input]) => reopened.submission(input.clientMessageId)?.queuedMessageId + ) + ) + .toEqual(['D', 'legacy-M']) + expect(reopened.queuedMessages.list().filter((c) => c.state === 'waiting')).toEqual([]) + expect(reopened.queuedMessages.displayState().inputEntries.size).toBe(0) + expect(loadTestJournal(rig.root, SESSION)!.state.inputEntries.size).toBe(0) + expect([...reopened.inputOrder.lineage.cards.values()].every((c) => !c.baseline)).toBe(true) + } finally { + await rig.dispose() + } + } +) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-input-selection.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-input-selection.ts new file mode 100644 index 00000000000..88c9d422000 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-input-selection.ts @@ -0,0 +1,100 @@ +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { QueuedMessageRow } from '../agent-session-journal/queued-message-table' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { pendingPromptsAllUnanswerableHere } from '../../../shared/agent-session-approval-subject' + +export type StructuredInputSelection = + | { kind: 'card'; card: QueuedMessageRow } + | { kind: 'submission'; submission: AgentJournalSubmission } + | { kind: 'none' } + +/** One live checkpoint owns automatic input order; debt alone is not execution. */ +export function selectStructuredAgentSessionInput( + journal: AgentSessionJournal, + currentInput?: string, + unanswerable = false +): StructuredInputSelection { + const queued = journal.inputOrder.firstQueued() + const current = currentInput ? journal.submission(currentInput) : undefined + const selected = + current?.dispatchState === 'pending' && + (!queued || current.submittedSequence! < queued.submittedSequence!) + ? current + : queued + const nextCard = journal.queuedMessages.nextInputCard() + if (!selected) { + return nextCard ? { kind: 'card', card: nextCard } : { kind: 'none' } + } + // Explicit card Send intentionally overrides the automatic queue place. + const override = journal.inputOrder.explicitSend() + if (override) { + return { kind: 'submission', submission: override } + } + if (unanswerable) { + return { kind: 'submission', submission: selected } + } + const predecessors = journal.inputOrder.predecessors( + journal.queuedMessages.inputState(), + selected.clientMessageId + ) + const handoff = predecessors.handoff?.consumedAs + if (handoff) { + const submission = journal.submission(handoff) + if (submission) { + return { kind: 'submission', submission } + } + } + if (predecessors.head) { + return nextCard ? { kind: 'card', card: nextCard } : { kind: 'none' } + } + return { kind: 'submission', submission: selected } +} + +export function selectedStructuredSubmission( + journal: AgentSessionJournal +): AgentJournalSubmission | undefined { + const selected = selectStructuredAgentSessionInput(journal) + return selected.kind === 'submission' ? selected.submission : undefined +} + +export function selectableStructuredSubmission( + journal: AgentSessionJournal, + work: StructuredAgentSessionCurrentWork | null, + currentInput?: string +): AgentJournalSubmission | undefined { + if (!journal.inputOrder.firstQueued() && currentInput === undefined) { + return undefined + } + const unanswerable = work !== null && pendingPromptsAllUnanswerableHere(work.actionablePrompts()) + const selected = selectStructuredAgentSessionInput(journal, currentInput, unanswerable) + if (selected.kind !== 'submission') { + return undefined + } + const { submission } = selected + if (unanswerable) { + return submission + } + return journal.inputOrder.forced(submission.clientMessageId) && + work && + structuredInputExecutionWorking(journal, work, currentInput) + ? undefined + : submission +} + +/** Provider-held/uncertain work still blocks cards; deferred, proven-unhanded inputs do not. */ +export function structuredInputExecutionWorking( + journal: AgentSessionJournal, + work: StructuredAgentSessionCurrentWork, + currentInput?: string +): boolean { + if (work.activeTurnId() !== null) { + return true + } + for (const entry of journal.inputOrder.enteredSubmissions()) { + if (entry.clientMessageId !== currentInput && work.owesSend(entry)) { + return true + } + } + return false +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-gap.test-support.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-gap.test-support.ts new file mode 100644 index 00000000000..faa2586c95b --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-gap.test-support.ts @@ -0,0 +1,81 @@ +import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { insertCommandReceiptIfAbsent } from '../agent-session-journal/command-receipt-table' +import { writeCommandReceiptTestRecord } from '../agent-session-journal/command-receipt-test-support' +import { + insertTestJournalRow, + writeTestLegacyTerminalGaps, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { JournalSubmissionRow } from '../agent-session-journal/journal-row-schema' +import { hostTestMessage } from './structured-agent-session-host-test-data' + +/** Actual unannotated initial-source writer shape, including terminal direct inputs between consumers. */ +export function writeLegacyGapFixture(root: string, journal: AgentSessionJournal, n: number): void { + const database = openTestJournalHostDatabase(root) + const { sessionId } = journal['identity'] + const { epoch, sequence } = journal.cursor() + let seq = sequence + const base = () => ({ v: 3, epoch, seq: ++seq, fence: 1, ts: 1 }) + for (const id of ['C', 'D']) { + const coordinate = { epoch, sequence: seq + 1 } + insertTestJournalRow(database.db, sessionId, { + ...base(), + kind: 'queue', + messageId: id, + change: { + op: 'queued', + body: + id === 'C' + ? { ...hostTestMessage('/compact'), command: { name: 'compact' } } + : hostTestMessage(id), + position: id === 'C' ? 0 : 1, + queuedAt: coordinate + } + }) + } + const submission = (id: string) => { + const row: JournalSubmissionRow = { + ...base(), + kind: 'submission' as const, + clientMessageId: id, + payloadFingerprint: id, + body: hostTestMessage(id), + providerHandle: agentSessionJournalProviderHandle(journal['identity']), + handoverRecorded: true, + origin: 'client' as const, + options: { effort: 'high' } + } + insertTestJournalRow(database.db, sessionId, row) + return row.seq + } + seq = writeTestLegacyTerminalGaps( + database.db, + sessionId, + { + epoch, + sequence: seq, + fence: 1, + providerHandle: agentSessionJournalProviderHandle(journal['identity']) + }, + n + ) + const acceptedSequence = submission('legacy-M') + database.transaction((db) => { + writeCommandReceiptTestRecord(db, sessionId) + insertCommandReceiptIfAbsent( + db, + { kind: 'global' }, + { + operationId: 'legacy-M', + sessionId, + callerKey: 'fixture', + method: 'agentSession.send', + fingerprint: 'legacy-M', + status: 'accepted', + acceptedAt: 1, + result: { kind: 'journal-row', epoch, sequence: acceptedSequence } + } + ) + }) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-opening-send.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-opening-send.ts index bdd44669988..737d01638d1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-opening-send.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-opening-send.ts @@ -2,7 +2,8 @@ import { structuredAgentSessionSendOpeningTurn } from '../../../shared/structure import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' -import { oldestQueuedSubmission } from './structured-agent-session-start-failure-row' +import type { oldestQueuedSubmission } from './structured-agent-session-start-failure-row' +import { selectableStructuredSubmission } from './structured-agent-session-input-selection' /** Read opening-turn evidence from the journal fold without a snapshot. */ export function structuredAgentSessionSendOpeningTurnInJournal( @@ -25,9 +26,9 @@ export function structuredAgentSessionSendOpeningTurnInJournal( export function structuredAgentSessionNextHandover( session: Pick, fence: number, - work?: Pick | null + work?: StructuredAgentSessionCurrentWork | null ): ReturnType { return structuredAgentSessionSendOpeningTurnInJournal(session.journal, fence, work) ? undefined - : oldestQueuedSubmission(session) + : selectableStructuredSubmission(session.journal, work ?? null) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queue-import-reuse.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queue-import-reuse.test.ts index 62463e7ce2b..2d76613a9cc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queue-import-reuse.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queue-import-reuse.test.ts @@ -1,3 +1,9 @@ +import { JournalWriteProjection } from '../agent-session-journal/journal-write-projection' +import { encodeJournalState } from '../agent-session-journal/journal-state-codec' +import { journalOutcomeSource } from '../agent-session-journal/journal-outcome-format' +import { LegacyQueuedCards } from '../agent-session-journal/journal-queue-import' +import { openRigTurnFor } from './structured-agent-session-queued-rig-turn.test-fixture' +import { loadTestJournal } from '../agent-session-journal/journal-host-database-test-support' import { expect, it, vi } from 'vitest' import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' @@ -60,7 +66,8 @@ it.each([100, 1_000])('raw selection reuse: %i waiting cards', async (size) => { const turn = { provider: 'codex' as const, threadId: 'progress', turnId: 't', ordinal: 0 } await journal.appendItem(turn, message('progress'), threadWrite) feed.publish(SESSION, journal) - expect(visits).toBe(2 * size) + // Provider progress cannot change a legacy card’s exact handoff classification. + expect(visits).toBe(0) expect(journal.publicRead().normalizationKey).toBe('') expect(journal.queuedMessages.list().filter((c) => c.state === 'waiting')).toHaveLength(size) for (const events of received) { @@ -205,3 +212,232 @@ it('a rewind while the import has failed moves the legacy cards first: a refused await nothingSentTo('edited') expect(legacyIds()).toEqual([]) }) + +it.each([ + { importNext: false, format1: false }, + { importNext: true, format1: false }, + { importNext: false, format1: true } +])( + 'persists fallback identity through recovery (next import=$importNext, format1=$format1)', + async ({ importNext, format1 }) => { + const rig = fixture.rig + await rig.host.close(SESSION, 'evict') + const seed = journalFor(SESSION) + await seed.open() + await seed.queuedMessages.insert({ messageId: 'C', body: message('/compact') }) + await seed.close() + if (format1) { + db().exec( + "DELETE FROM journal_outcomes WHERE field IN ('inputEntries','inputCards'); UPDATE journal_outcomes SET format=1; UPDATE journal_outcome_epochs SET format=1" + ) + } + seedLegacy(SESSION, [['D', 'waiting', 'original D']]) + db().prepare('UPDATE queued_messages SET position = 2 WHERE message_id = ?').run('D') + db().exec(`CREATE TEMP TRIGGER overlay_import_failure BEFORE DELETE ON queued_messages + BEGIN SELECT RAISE(ABORT, 'import failure'); END`) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const importAtOpen = LegacyQueuedCards.prototype.importAtOpen + const fault = vi + .spyOn(LegacyQueuedCards.prototype, 'importAtOpen') + .mockImplementation(function (this: LegacyQueuedCards) { + return importAtOpen + .call(this) + .finally(() => db().exec('DROP TRIGGER IF EXISTS overlay_import_failure')) + }) + try { + const w = await rig.workingSend() + await openRigTurnFor(rig, w) + await rig.settleAccepted(w, 'W') + const live = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + expect(journalOutcomeSource(db(), SESSION, live.epoch)?.format).toBe(2) + expect(live.queuedMessages.get('C')?.state).toBe('waiting') + expect(legacyIds()).toEqual(['D']) + live.publicRead() + const m = rig.send('original M', undefined, { carried: { effort: 'high' } }) + const result = await m.result + expect(result).toMatchObject({ ok: true, value: { submission: { clientMessageId: m.id } } }) + const receipt = rig.store.readCommandReceipt({ kind: 'global' }, m.id) + expect(live.inputOrder.forced(m.id)).toBe(true) + expect(live.submission(m.id)?.handedOverAt).toBeUndefined() + expect(live.inputOrder.precedes(live.queuedMessages.get('D')!, live.submission(m.id)!)).toBe( + true + ) + if (!importNext) { + db().exec(`CREATE TEMP TRIGGER overlay_import_failure BEFORE DELETE ON queued_messages + BEGIN SELECT RAISE(ABORT, 'import failure'); END`) + } + await rig.host.flushAllStreamedEvents() + rig.crashRestartHostProcess() + await rig.submission(m.id) + const reopened = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal + const order = reopened.queuedMessages + .list() + .filter((c) => c.state === 'waiting') + .map((c) => c.messageId) + // Let the complete production iteration expose even the causal wrong C,M,D ordering. + expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) + const delivered: string[] = [] + for (const id of order) { + if (id === 'C') { + await eventually(() => expect(rig.compact).toHaveBeenCalledOnce()) + rig.finishCompact() + } else { + await eventually(async () => expect((await rig.handoff(id))?.handedOverAt).toBeDefined()) + const h = await rig.handoffId(id) + const calls = rig.dispatch.mock.calls.filter(([input]) => input.clientMessageId === h) + expect(calls).toHaveLength(1) + expect(calls[0]![0].body).toEqual(message(id === 'D' ? 'original D' : 'original M')) + if (id === m.id) { + expect(calls[0]![0].executionOptions).toMatchObject({ effort: 'high' }) + } + await rig.settleAccepted(h, id) + expect(await rig.submission(h)).toMatchObject({ + dispatchState: 'accepted', + queuedMessageId: id + }) + } + delivered.push(id) + } + expect(rig.store.readCommandReceipt({ kind: 'global' }, m.id)).toEqual(receipt) + expect(reopened.item(agentJournalSubmissionKey(m.id))?.body).toEqual(message('original M')) + expect(reopened.submission(m.id)?.options).toEqual({ effort: 'high' }) + expect(reopened.queuedMessages.list().filter((c) => c.state === 'waiting')).toEqual([]) + const restored = loadTestJournal(rig.root, SESSION)! + expect(restored.damage).toBeNull() + expect(restored.state.inputEntries.size).toBe(0) + expect(restored.state.inputCards.size).toBe(0) + expect(reopened['state'].inputCards.size).toBe(0) + expect(rig.dispatch.mock.calls.filter(([input]) => input.clientMessageId === w)).toHaveLength( + 1 + ) + expect(delivered).toEqual(['C', 'D', m.id]) + } finally { + db().exec('DROP TRIGGER IF EXISTS overlay_import_failure') + fault.mockRestore() + warn.mockRestore() + } + } +) + +it.each([ + { readFirst: false, legacyCommand: false }, + { readFirst: true, legacyCommand: false }, + { readFirst: false, legacyCommand: true } +])( + 'keeps reads derivative and persists fallback (public=$readFirst, legacyCommand=$legacyCommand)', + async ({ readFirst, legacyCommand }) => { + const seed = journalFor('overlay-direct') + await seed.open() + if (!legacyCommand) { + await seed.queuedMessages.insert({ messageId: 'C', body: message('/compact') }) + } + await seed.close() + seedLegacy( + 'overlay-direct', + legacyCommand + ? [ + ['C', 'waiting', '/compact'], + ['D', 'waiting', 'D'] + ] + : [['D', 'waiting', 'D']] + ) + db().exec(`CREATE TEMP TRIGGER overlay_direct_import_failure BEFORE DELETE ON queued_messages + BEGIN SELECT RAISE(ABORT, 'import failure'); END`) + const journal = journalFor('overlay-direct') + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + await journal.open() + const state = journal['state'] + const before = encodeJournalState(state) + if (readFirst) { + journal.publicRead() + journal.queuedMessages.nextInputCard() + expect(encodeJournalState(state)).toEqual(before) + JournalWriteProjection.verifyEachCommit = true + await journal.appendItem( + { provider: 'orca', clientMessageId: 'unrelated' }, + message('unrelated'), + threadWrite + ) + } + const target = await journal.appendSubmission({ + clientMessageId: 'original-M', + payloadFingerprint: 'original', + body: message('original M'), + handoverRecorded: true, + origin: 'client', + fence: 0, + options: { effort: 'high' } + }) + expect(journal.inputOrder.forced('original-M')).toBe(true) + expect( + journal.inputOrder.precedes( + journal.queuedMessages.get('D')!, + journal.submission('original-M')! + ) + ).toBe(true) + const saved = loadTestJournal(fixture.rig.root, 'overlay-direct')! + expect(saved.damage).toBeNull() + expect(saved.state.inputCards.get('D')).toEqual(journal['state'].inputCards.get('D')) + expect(saved.state.inputCards.has('D')).toBe(true) + expect( + db() + .prepare( + "SELECT COUNT(*) AS n FROM journal_outcomes WHERE field='inputCards' AND entry_key=? AND deleted=0" + ) + .get(JSON.stringify('D'))?.n + ).toBe(1) + expect(target).toEqual({ + epoch: journal.epoch, + sequence: journal.submission('original-M')!.acceptedSequence + }) + const introduction = journal['state'].inputCards.get('D')!.introduced + db().exec('DROP TRIGGER overlay_direct_import_failure') + for (let pass = 0; pass < 2; pass++) { + await journal.replaceEpochItems('unreconcilable_prefix', 0, []) + const restored = loadTestJournal(fixture.rig.root, 'overlay-direct')! + expect(restored.damage).toBeNull() + expect(restored.state.inputCards.get('D')?.introduced).toBe(introduction) + expect( + journal.inputOrder.precedes( + journal.queuedMessages.get('D')!, + journal.submission('original-M')! + ) + ).toBe(true) + expect(restored.state.inputEntries.get('original-M')).toEqual( + journal['state'].inputEntries.get('original-M') + ) + } + } finally { + JournalWriteProjection.verifyEachCommit = false + db().exec('DROP TRIGGER IF EXISTS overlay_direct_import_failure') + warn.mockRestore() + await journal.close() + } + } +) + +it('discovers real overlay ownership before pruning a genuinely saved native identity', async () => { + const session = 'overlay-saved' + const seed = journalFor(session) + await seed.open() + await seed.queuedMessages.insert({ messageId: 'D', body: message('saved D') }) + const saved = seed['state'].inputCards.get('D') + const native = db() + .prepare( + "SELECT value_json FROM journal_outcomes WHERE field='inputCards' AND entry_key=? ORDER BY seq DESC LIMIT 1" + ) + .get(JSON.stringify('D')) + expect(native?.value_json).toBe(JSON.stringify(['D', saved])) + await seed.close() + // Independent fixture control: retain the writer's genuine native record, with only legacy body ownership. + db() + .prepare("DELETE FROM journal_outcomes WHERE field='queue.cards' AND entry_key=?") + .run(JSON.stringify('D')) + seedLegacy(session, [['D', 'waiting', 'saved D']]) + const restored = loadTestJournal(fixture.rig.root, session)! + expect(restored.damage).toBeNull() + expect(restored.state.queue.cards.has('D')).toBe(false) + expect(restored.state.inputEntries.size).toBe(0) + expect(restored.state.inputCards.get('D')).toEqual(saved) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queue-import-scaling.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queue-import-scaling.test.ts index e0b17e43649..cbca679dff6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queue-import-scaling.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queue-import-scaling.test.ts @@ -1,3 +1,10 @@ +import * as provenance from '../agent-session-journal/journal-input-provenance' +import { JournalDraftMap } from '../agent-session-journal/journal-transition-collections' +import * as transitions from '../agent-session-journal/journal-reducer-transition' +import * as outcomes from '../agent-session-journal/journal-outcome-codec' +import { JournalInputCardIndex } from '../agent-session-journal/journal-input-card-index' +import { JournalInputLineageOwner } from '../agent-session-journal/journal-input-lineage' +import { encodeJournalState } from '../agent-session-journal/journal-state-codec' import { expect, it, vi } from 'vitest' import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' @@ -5,6 +12,8 @@ import { AgentSessionSubscribers } from './structured-agent-session-subscribers' import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' import { + fixture, + threadWrite, type Legacy, db, handOff, @@ -134,9 +143,9 @@ it.each([100, 1_000].flatMap((size) => [false, true].map((adopt) => ({ size, ado expect(journal.queuedMessages.displayState() === display).toBe(false) expect(journal.snapshot()).not.toBe(snapshot) feed.publish(SESSION, journal) - // A reopen that retries the failed import writes one card; its outcome write lists that card - // once in the card map and once in the order set: a constant 2, whatever the history size. - expect(work()).toEqual([corrections + (adopt ? 0 : 2), 1, 1]) + // Import normalizes the one affected submission and refreshes its derived entered index. + // Each touched map key is listed for encoding and adoption; no other history key is visited. + expect(work()).toEqual([corrections + (adopt ? 0 : 2 * 2), 1, 1]) reset() await journal.appendItem( { provider: 'codex', threadId: 'progress', turnId: 't', ordinal: 0 }, @@ -165,3 +174,314 @@ it.each([100, 1_000].flatMap((size) => [false, true].map((adopt) => ({ size, ado await journal.close() } ) + +it.each([10, 100])('bounds actual canonical fallback reads/writes N=%i', async (n) => { + const session = 'overlay-scale' + const seed = journalFor(session) + await seed.open() + await seed.queuedMessages.insert({ messageId: 'C', body: message('/compact') }) + await seed.close() + seedLegacy( + session, + Array.from({ length: n }, (_, i): Legacy => [`D${i}`, 'waiting', `body${i}`]) + ) + db().exec(`CREATE TEMP TRIGGER overlay_scale_import_failure BEFORE DELETE ON queued_messages + BEGIN SELECT RAISE(ABORT, 'import failure'); END`) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const journal = journalFor(session) + let indexed = 0 + let draftKeys = 0 + let receiptQueries = 0 + let cohortVisits = 0 + let introduced = 0 + let frontiers = 0 + let mutations = 0 + let entries = 0 + let cards = 0 + let bytes = 0 + let drafts: ReturnType[] = [] + const index = JournalInputCardIndex.prototype.indexCard + const indexSpy = vi + .spyOn(JournalInputCardIndex.prototype, 'indexCard') + .mockImplementation(function (this: JournalInputCardIndex, ...args) { + indexed++ + return index.apply(this, args) + }) + const introduce = JournalInputCardIndex.prototype.introduceOverlays + const introduction = vi + .spyOn(JournalInputCardIndex.prototype, 'introduceOverlays') + .mockImplementation(function (this: JournalInputCardIndex, change) { + introduce.call(this, (card) => { + introduced++ + change(card) + }) + }) + const frontier = JournalInputLineageOwner.prototype.restoreFrontier + const frontierSpy = vi + .spyOn(JournalInputLineageOwner.prototype, 'restoreFrontier') + .mockImplementation(function (this: JournalInputLineageOwner, ...args) { + frontiers++ + return frontier.apply(this, args) + }) + const isolate = transitions.isolateJournalTransition + const isolateSpy = vi + .spyOn(transitions, 'isolateJournalTransition') + .mockImplementation((...args) => { + const draft = isolate(...args) + drafts.push(draft) + return draft + }) + const encode = outcomes.encodeJournalOutcomes + const encodeSpy = vi.spyOn(outcomes, 'encodeJournalOutcomes').mockImplementation((...args) => { + const encoded = encode(...args) + for (const value of encoded.values) { + if (value.field === 'inputEntries') { + entries++ + } + if (value.field === 'inputCards') { + cards++ + } + if (value.field === 'inputEntries' || value.field === 'inputCards') { + bytes += Buffer.byteLength(value.json) + } + } + return encoded + }) + const map = Array.prototype.map + const flatMap = Array.prototype.flatMap + const isCohort = (array: unknown[]) => { + const first = array[0] + return ( + typeof first === 'object' && + first !== null && + 'messageId' in first && + typeof first.messageId === 'string' && + first.messageId.startsWith('D') + ) + } + const maps = vi.spyOn(Array.prototype, 'map').mockImplementation(function ( + this: unknown[], + callback, + thisArg + ) { + const cohort = isCohort(this) + return map.call(this, (...args) => { + if (cohort) { + cohortVisits++ + } + return callback.apply(thisArg, args) + }) + }) + const flatMaps = vi.spyOn(Array.prototype, 'flatMap').mockImplementation(function ( + this: unknown[], + callback, + thisArg + ) { + const cohort = isCohort(this) + return flatMap.call(this, (...args) => { + if (cohort) { + cohortVisits++ + } + return callback.apply(thisArg, args) + }) + }) + const keys = JournalDraftMap.prototype.keys + const keySpy = vi.spyOn(JournalDraftMap.prototype, 'keys').mockImplementation(function* ( + this: JournalDraftMap + ) { + for (const key of keys.call(this)) { + draftKeys++ + yield key + } + return undefined + }) + const acceptance = provenance.journalInputAcceptance + const receiptSpy = vi + .spyOn(provenance, 'journalInputAcceptance') + .mockImplementation((...args) => { + receiptQueries++ + return acceptance(...args) + }) + const measurements: Record = {} + const measure = () => ({ + indexed, + draftKeys, + receiptQueries, + cohortVisits, + introduced, + frontiers, + mutations, + entries, + cards, + bytes, + nodes: drafts.reduce((sum, draft) => sum + draft.tracked(), 0), + derived: drafts.reduce((sum, draft) => sum + draft.trackedDerived(), 0) + }) + const reset = () => { + indexed = + draftKeys = + receiptQueries = + cohortVisits = + introduced = + frontiers = + mutations = + entries = + cards = + bytes = + 0 + drafts = [] + } + try { + await journal.open() + measurements.discovery = measure() + expect(indexed).toBeLessThanOrEqual(5 * n + 1) + expect(cohortVisits).toBeLessThanOrEqual(8 * n) + expect(frontiers).toBe(1) + expect(cards).toBe(2 * n - 1) // Attempted failed import, never adopted. + expect(introduced).toBe(n) // Failed import prepared this actual cohort, but adopted none. + expect(journal['state'].inputCards.size).toBe(1) + reset() + const set = journal['state'].inputCards.set.bind(journal['state'].inputCards) + const mutation = vi.spyOn(journal['state'].inputCards, 'set').mockImplementation((...args) => { + mutations++ + return set(...args) + }) + const remove = journal['state'].inputCards.delete.bind(journal['state'].inputCards) + const clear = journal['state'].inputCards.clear.bind(journal['state'].inputCards) + const deletion = vi + .spyOn(journal['state'].inputCards, 'delete') + .mockImplementation((...args) => { + mutations++ + return remove(...args) + }) + const clearing = vi.spyOn(journal['state'].inputCards, 'clear').mockImplementation(() => { + mutations++ + return clear() + }) + const before = encodeJournalState(journal['state']) + for (let i = 0; i < n; i++) { + journal.publicRead() + expect(journal.queuedMessages.nextInputCard()?.messageId).toBe('C') + } + expect(encodeJournalState(journal['state'])).toEqual(before) + expect(measure()).toEqual({ + indexed: 0, + draftKeys: 0, + receiptQueries: 0, + cohortVisits: 0, + introduced: 0, + frontiers: 0, + mutations: 0, + entries: 0, + cards: 0, + bytes: 0, + nodes: 0, + derived: 0 + }) + mutation.mockRestore() + deletion.mockRestore() + clearing.mockRestore() + measurements.reads = measure() + await journal.appendItem( + { provider: 'orca', clientMessageId: 'unrelated' }, + message('unrelated'), + threadWrite + ) + measurements.firstWrite = measure() + expect(introduced).toBe(n) + expect(cards).toBe(n) + expect(indexed).toBe(n) + expect(cohortVisits).toBe(0) + expect(journal['state'].inputCards.size).toBe(n + 1) + reset() + for (let i = 0; i < n; i++) { + await journal.appendSubmission({ + clientMessageId: `M${i}`, + body: message(`M${i}`), + payloadFingerprint: `M${i}`, + origin: 'client', + handoverRecorded: true, + fence: 0 + }) + expect(journal.inputOrder.forced(`M${i}`)).toBe(true) + expect( + journal.inputOrder.precedes( + journal.queuedMessages.get(`D${n - 1}`)!, + journal.submission(`M${i}`)! + ) + ).toBe(true) + } + measurements.admissions = measure() + expect({ + indexed, + draftKeys, + receiptQueries, + cohortVisits, + introduced, + frontiers, + cards, + entries + }).toEqual({ + indexed: 0, + draftKeys: 0, + receiptQueries: 0, + cohortVisits: 0, + introduced: 0, + frontiers: 0, + cards: 0, + entries: n + }) + // One input's fixed graph plus at most two AVL paths, each with bounded wrapper fields. + expect(measure().nodes).toBeLessThanOrEqual(160 * n * (1 + Math.ceil(Math.log2(n + 2)))) + // Each native entry contains fixed IDs/cursors and one witness, never card bodies/populations. + expect(bytes).toBeLessThanOrEqual(400 * n) + reset() + db().exec('DROP TRIGGER overlay_scale_import_failure') + const old = journal.queuedMessages.get('D0')! + expect( + await journal.queuedMessages.update({ + messageId: 'D0', + expectedBodyFingerprint: old.fingerprint, + text: 'latest D' + }) + ).toMatchObject({ status: 'updated' }) + expect(journal.queuedMessages.get('D0')?.body).toEqual(message('latest D')) + expect(journal['state'].inputCards.size).toBe(n + 1) + measurements.edit = measure() + expect({ indexed, cards, cohortVisits, introduced }).toEqual({ + indexed: 1, + cards: 0, + cohortVisits: 0, + introduced: 0 + }) + reset() + await journal.queuedMessages.withdraw(Array.from({ length: n }, (_, i) => `D${i}`)) + expect(journal['state'].inputCards.size).toBe(1) + expect(legacyIds(session)).toEqual([]) + measurements.retirement = measure() + // One bulk adoption, withdrawals, then current-cohort retirement; not n separate flushes. + expect(cards).toBe(n) + expect(bytes).toBe(4 * n) // Native tombstone null values. + expect(indexed).toBeLessThanOrEqual(3 * n) + expect(cohortVisits).toBeLessThanOrEqual(6 * n) + console.info('overlay-operation-counts', JSON.stringify({ n, measurements })) + expect(fixture.rig.dispatch).not.toHaveBeenCalled() + } finally { + db().exec('DROP TRIGGER IF EXISTS overlay_scale_import_failure') + for (const spy of [ + indexSpy, + introduction, + frontierSpy, + isolateSpy, + encodeSpy, + maps, + flatMaps, + keySpy, + receiptSpy, + warn + ]) { + spy.mockRestore() + } + await journal.close() + } +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index f4478cd2abc..4a2b5c8a132 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -21,25 +21,22 @@ import type { AgentSessionTurnContext } from './structured-agent-session-turns' import { QueuedMessageNotConsumableError } from '../agent-session-journal/queued-message-consume-error' import type { QueuedMessageRow } from '../agent-session-journal/queued-message-table' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' -import { structuredQueuePauses } from './structured-agent-session-queued-pause' -import { nextSendableQueuedCard } from '../agent-session-journal/queued-message-pause' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' import { agentSessionAttachmentExpiredRefusal } from './structured-agent-session-turns' import { isAgentSessionAttachmentExpiredError } from '../agent-session-attachments/agent-session-attachment-claims' import { captureStructuredExecutionOptions } from './structured-agent-session-execution-options' import type { JournalOperationReceipt } from '../agent-session-journal/journal-row-writer' +import { + selectStructuredAgentSessionInput, + structuredInputExecutionWorking +} from './structured-agent-session-input-selection' /** Preserve order and holds; only automatic drain stops at edit leases, which still count as admission backlog. */ function oldestActionableQueuedMessage( journal: Pick, automatic: boolean ): QueuedMessageRow | null { - const rows = journal.queuedMessages.list() - if (!rows.some((row) => row.state === 'waiting')) { - return null - } - const edited = automatic ? journal.queuedMessages.editLeases.heldIds(rows) : undefined - return nextSendableQueuedCard(structuredQueuePauses(journal), rows, edited) + return journal.queuedMessages.nextInputCard(automatic) } /** Host-local gate shared by admission, drain and Send-now; only Send-now overrides working. */ @@ -66,12 +63,20 @@ export function structuredQueueHold(input: StructuredQueueGateInput): Structured export function nextStructuredQueuedMessage( input: StructuredQueueGateInput & { journal: AgentSessionJournal } ): QueuedMessageRow | null { - const next = oldestActionableQueuedMessage(input.journal, true) - // Check cheap working state before the prompt scan on every streamed frame. - if (next === null || input.work.working()) { + if (structuredInputExecutionWorking(input.journal, input.work)) { return null } - return structuredQueueHold(input) === null ? next : null + const selected = selectStructuredAgentSessionInput(input.journal) + const next = selected.kind === 'card' ? selected.card : null + // Check cheap working state before the prompt scan on every streamed frame. + if (next === null) { + return null + } + return !input.record || + structuredAgentSessionSendBlock(input.record) || + input.work.hasActionablePrompt() + ? null + : next } /** Queue behind work/actionable backlog; held or returned cards cannot trap a fresh send that lifts their pause. */ @@ -215,9 +220,11 @@ export class StructuredAgentSessionQueuedMessageDrain { } // Skip serialized work during token streams; settlement/generation-end schedules a fresh gate check. try { + const work = this.deps.currentWork(sessionId) if ( - oldestActionableQueuedMessage(journal, true) === null || - this.deps.currentWork(sessionId)?.working() !== false + !work || + nextStructuredQueuedMessage({ journal, record: this.deps.getRecord(sessionId), work }) === + null ) { return } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-ready-operation.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-ready-operation.ts index 9e80a1934ae..1057512590d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-ready-operation.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-ready-operation.ts @@ -224,7 +224,7 @@ export async function runWithReadyStructuredAgentSessionOperation( } context.deliveryOrder.reconcile( envelope.sessionId, - context.sessions.get(envelope.sessionId)?.journal.submissions() ?? [] + context.sessions.get(envelope.sessionId)?.journal ) const place = context.deliveryOrder.reserve(envelope.sessionId) return { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts index a20958875ba..7c8a33f5458 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts @@ -10,6 +10,7 @@ import { isStructuredAgentSessionCommandTurnId } from './structured-agent-sessio /** Scope active turns to the live generation so ended commands cannot hold delivery waits. */ export type SendSettlementJournal = Pick & { activeTurnId: () => string | null + behindCommand?: (clientMessageId: string) => boolean } type SettledSend = { @@ -59,7 +60,10 @@ function settledSend( until === 'answered' ? submission.dispatchState === 'pending' : isQueuedAgentJournalSubmission(submission) && - !(until === 'handed-over-or-behind-command' && runningCommand(journal)) + !( + until === 'handed-over-or-behind-command' && + (runningCommand(journal) || journal.behindCommand?.(clientMessageId)) + ) return waiting ? 'pending' : { cursor: journal.cursor(), value: { clientMessageId, submission } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts index b607856de07..c63b67de945 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts @@ -1,3 +1,4 @@ +import { selectedStructuredSubmission } from './structured-agent-session-input-selection' import type { AgentSessionFailureWordsContext } from '../../../shared/agent-session-failure-words' import { structuredAgentSessionStartFailure } from './structured-agent-session-failure-text' import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' @@ -145,16 +146,7 @@ export function acceptedNoLaterThan( export function oldestQueuedSubmission( session: Pick ): ReturnType[number] | undefined { - let oldest: ReturnType - for (const submission of session.journal.submissions()) { - if ( - isQueuedAgentJournalSubmission(submission) && - (oldest === undefined || (submission.acceptedSequence ?? 0) < (oldest.acceptedSequence ?? 0)) - ) { - oldest = submission - } - } - return oldest + return selectedStructuredSubmission(session.journal) } export async function recordStructuredAgentSessionDeliveryFailure( diff --git a/src/main/runtime/agent-session-publication-scale.test.ts b/src/main/runtime/agent-session-publication-scale.test.ts index 7c9b87da283..203b3dec938 100644 --- a/src/main/runtime/agent-session-publication-scale.test.ts +++ b/src/main/runtime/agent-session-publication-scale.test.ts @@ -216,13 +216,17 @@ describe('real-store status and queue publications', () => { subscribers.publish(id, session.journal) } } - // Each publish derives queue pauses twice and checks current work independently. + // Three publications retain the former four-read ceiling; cached reads may disappear. + const lists = counts.lists - warm.lists + const submissions = counts.submissions - warm.submissions + expect(lists).toBeLessThanOrEqual(12 * n) + expect(submissions).toBeLessThanOrEqual(12 * n) expect(counts).toEqual({ ...warm, - lists: warm.lists + 12 * n, - listRows: warm.listRows + 12 * n * cards, - submissions: warm.submissions + 12 * n, - submissionRows: warm.submissionRows + 12 * n * sends + lists: warm.lists + lists, + listRows: warm.listRows + lists * cards, + submissions: warm.submissions + submissions, + submissionRows: warm.submissionRows + submissions * sends }) expect(work.counts).toMatchObject({ views: 9 * n, inventories: 0, clones: 0, decodes: 0 }) expect(statusEvents).toEqual([]) diff --git a/src/main/runtime/orchestration/chat-worker-legacy-failure-read.test.ts b/src/main/runtime/orchestration/chat-worker-legacy-failure-read.test.ts index a3382807014..3f45aefd398 100644 --- a/src/main/runtime/orchestration/chat-worker-legacy-failure-read.test.ts +++ b/src/main/runtime/orchestration/chat-worker-legacy-failure-read.test.ts @@ -18,7 +18,10 @@ import { words, batch } from './chat-worker-request-failure-rig.test-fixture' -import { countLegacyFailureDisplay } from './chat-worker-legacy-operation-counts.test-support' +import { + countLegacyFailureDisplay, + emptyLegacyCounts +} from './chat-worker-legacy-operation-counts.test-support' it.each([100, 1000])( 'bounds the added failure display with %i readable legacy cards', @@ -91,6 +94,31 @@ it.each([100, 1000])( db.prepare('SELECT message_id FROM queued_messages WHERE session_id = ?').all(SESSION) ).toHaveLength(n) expect(journal.queuedMessages.list()).toHaveLength(n) + const cold = structuredClone(ops.counts) + // Three decoded populations: restore, cached source and failed import. The import builds + // N queue rows on an isolated draft before its DELETE fails; none of that work is free. + expect(cold).toEqual({ + upstream: { + ...emptyLegacyCounts(), + decodedRows: 3 * n, + restoredRows: n, + ownerClassifications: 2 * n, + cardConstructions: 5 * n, + handoffVisits: n, + identityEntries: n, + sourceIdEntries: n, + overlayEntries: 2 * n, + overlayVisits: 2 * n, + priorOverlayVisits: n, + indexUpdates: 4 * n, + overlayIndexUpdates: 2 * n, + pointIndexUpdates: 2 * n, + indexRebuilds: 1 + }, + publication: emptyLegacyCounts(), + failure: emptyLegacyCounts() + }) + ops.reset() const projection = new StructuredAgentSessionJournalProjections() r.deliver.mockImplementation(() => ops.publication(() => projection.read(journal, null))) r.sessions.set(SESSION, { ...r.session, journal }) @@ -109,7 +137,24 @@ it.each([100, 1000])( }) ]) await tick() + const changed = structuredClone(ops.counts) + // First successful native introduction charges N point updates, without reclassification. + expect(changed).toEqual({ + upstream: { ...emptyLegacyCounts(), indexUpdates: n, pointIndexUpdates: n }, + publication: { + ...emptyLegacyCounts(), + displays: 2, + handoffApplications: 2 * n, + itemCopies: 2, + itemEntries: 2 * n + 3, + submissionCopies: 2, + submissionEntries: 2 * (n + 1) + }, + failure: emptyLegacyCounts() + }) + expect(r.deliver).toHaveBeenCalledTimes(2) ops.reset() + r.deliver.mockClear() await batch(journal, 'ordinary-output', [ mutation( 'ordinary-output', @@ -124,25 +169,22 @@ it.each([100, 1000])( await tick() const ordinary = structuredClone(ops.counts) expect(ordinary).toEqual({ + upstream: emptyLegacyCounts(), publication: { + ...emptyLegacyCounts(), displays: 1, - cardVisits: n, + handoffApplications: n, itemCopies: 1, itemEntries: n + 3, submissionCopies: 1, submissionEntries: n + 1 }, - failure: { - displays: 0, - cardVisits: 0, - itemCopies: 0, - itemEntries: 0, - submissionCopies: 0, - submissionEntries: 0 - } + failure: emptyLegacyCounts() }) + expect(r.deliver).toHaveBeenCalledTimes(1) expect(r.mail()).toHaveLength(0) ops.reset() + r.deliver.mockClear() await batch(journal, 'eligible-failure', [ mutation('eligible-turn', { kind: 'turn', @@ -156,24 +198,27 @@ it.each([100, 1000])( await tick() const failure = structuredClone(ops.counts) const oneDisplay = { + ...emptyLegacyCounts(), displays: 1, - cardVisits: n, + handoffApplications: n, itemCopies: 1, itemEntries: n + 4, submissionCopies: 1, submissionEntries: n + 1 } expect(failure).toEqual({ + upstream: emptyLegacyCounts(), publication: oneDisplay, failure: { + ...emptyLegacyCounts(), displays: 1, - cardVisits: 0, itemCopies: 0, itemEntries: 0, submissionCopies: 0, submissionEntries: 0 } }) + expect(r.deliver).toHaveBeenCalledTimes(1) expect(r.mail()).toHaveLength(1) expect(r.mail()[0]?.body).toBe( [ @@ -185,6 +230,8 @@ it.each([100, 1000])( expect(r.db.getDispatchContextById(r.dispatch().id)?.status).toBe('dispatched') expect(r.db.getTask(r.dispatch().task_id)?.status).toBe('dispatched') expect(journal.submission('legacy:0')?.dispatchState).toBe('rejected') - process.stdout.write(`LEGACY_FAILURE_OPERATIONS ${JSON.stringify({ n, ordinary, failure })}\n`) + process.stdout.write( + `LEGACY_FAILURE_OPERATIONS ${JSON.stringify({ n, cold, changed, ordinary, failure })}\n` + ) } ) diff --git a/src/main/runtime/orchestration/chat-worker-legacy-operation-counts.test-support.ts b/src/main/runtime/orchestration/chat-worker-legacy-operation-counts.test-support.ts index 23d1e902624..988ae7c91cb 100644 --- a/src/main/runtime/orchestration/chat-worker-legacy-operation-counts.test-support.ts +++ b/src/main/runtime/orchestration/chat-worker-legacy-operation-counts.test-support.ts @@ -1,9 +1,28 @@ import { vi } from 'vitest' import * as legacy from '../../native-chat/agent-session-journal/journal-queue-import' +import * as fold from '../../native-chat/agent-session-journal/journal-queue-fold' +import * as stored from '../../native-chat/agent-session-journal/queued-message-stored-row' +import { JournalInputCardIndex } from '../../native-chat/agent-session-journal/journal-input-card-index' -const emptyCounts = () => ({ +export const emptyLegacyCounts = () => ({ + decodedRows: 0, + restoredRows: 0, + ownerClassifications: 0, + cardConstructions: 0, + handoffVisits: 0, + identityEntries: 0, + sourceIdEntries: 0, + overlayEntries: 0, + overlayVisits: 0, + priorOverlayVisits: 0, + rebuildVisits: 0, + indexUpdates: 0, + overlayIndexUpdates: 0, + rebuildIndexUpdates: 0, + pointIndexUpdates: 0, + indexRebuilds: 0, displays: 0, - cardVisits: 0, + handoffApplications: 0, itemCopies: 0, itemEntries: 0, submissionCopies: 0, @@ -27,38 +46,227 @@ function countMapCopies(map: Map, copy: () => void, visit: () => voi }) } -/** Delegates the real legacy display, counting its cached array and source Map iterators only. */ +function countIteration(array: readonly T[], visit: () => void) { + const iterate = array[Symbol.iterator].bind(array) + Object.defineProperty(array, Symbol.iterator, { + configurable: true, + value: function* () { + for (const value of { [Symbol.iterator]: iterate }) { + visit() + yield value + } + } + }) +} + +function countValues(map: Map, visit: () => void) { + const values = map.values.bind(map) + Object.defineProperty(map, 'values', { + configurable: true, + value: function* () { + for (const value of { [Symbol.iterator]: values }) { + visit() + yield value + } + } + }) +} + +function countMap(array: readonly T[], visit: () => void) { + const map = array.map.bind(array) + Object.defineProperty(array, 'map', { + configurable: true, + value: ( + callback: (value: T, index: number, array: readonly T[]) => U, + thisArg?: unknown + ): U[] => + map((value, index, source) => { + visit() + return callback.call(thisArg, value, index, source) + }) + }) +} + +function countFlatMap( + array: readonly T[], + visit: () => void, + result?: (values: readonly unknown[]) => void +) { + const flatMap = array.flatMap.bind(array) + Object.defineProperty(array, 'flatMap', { + configurable: true, + value: ( + callback: (this: This, value: T, index: number, array: readonly T[]) => U | readonly U[], + thisArg?: This + ): U[] => { + const values = flatMap(function (value, index, source) { + visit() + return callback.call(this, value, index, source) + }, thisArg) + result?.(values) + return values + } + }) +} + +/** Count delegated passes at their real boundaries, including upstream before publication. */ export function countLegacyFailureDisplay() { - const counts = { publication: emptyCounts(), failure: emptyCounts() } - let phase: keyof typeof counts = 'failure' - let displaying = false - const arrays = new WeakSet() + const counts = { + upstream: emptyLegacyCounts(), + publication: emptyLegacyCounts(), + failure: emptyLegacyCounts() + } + let phase: keyof typeof counts = 'upstream' + let classifying = false + let overlaying = false + let ensuring = false + const overlays = new WeakSet() + const handoffs = new WeakSet() + const decode = stored.readStoredQueuedMessageRow + vi.spyOn(stored, 'readStoredQueuedMessageRow').mockImplementation((row) => { + counts[phase].decodedRows++ + return decode(row) + }) + const restore = legacy.readJournalLegacyCards + vi.spyOn(legacy, 'readJournalLegacyCards').mockImplementation((...args) => { + const measured = counts[phase] + const before = measured.decodedRows + const cards = restore(...args) + measured.restoredRows += measured.decodedRows - before + return cards + }) + const construct = fold.queuedCardFromChange + vi.spyOn(fold, 'queuedCardFromChange').mockImplementation((...args) => { + counts[phase].cardConstructions++ + return construct(...args) + }) + const indexMaps = new WeakSet() + const watchIndexMap = (map: Map) => { + if (!indexMaps.has(map)) { + indexMaps.add(map) + countValues(map, () => { + if (ensuring) { + counts[phase].rebuildVisits++ + } else if (overlaying) { + counts[phase].priorOverlayVisits++ + } + }) + } + } + const index = JournalInputCardIndex.prototype + const overlay = index.overlay + vi.spyOn(index, 'overlay').mockImplementation(function ( + this: JournalInputCardIndex, + state, + rows + ) { + watchIndexMap(this['overlays']) + if (!overlays.has(rows)) { + overlays.add(rows) + countMap(rows, () => { + if (overlaying) { + counts[phase].overlayEntries++ + } + }) + countIteration(rows, () => { + if (overlaying) { + counts[phase].overlayVisits++ + } + }) + } + overlaying = true + try { + return overlay.call(this, state, rows) + } finally { + overlaying = false + } + }) + const update = index.indexCard + vi.spyOn(index, 'indexCard').mockImplementation(function ( + this: JournalInputCardIndex, + state, + card + ) { + counts[phase].indexUpdates++ + if (ensuring) { + counts[phase].rebuildIndexUpdates++ + } else if (overlaying) { + counts[phase].overlayIndexUpdates++ + } else { + counts[phase].pointIndexUpdates++ + } + return update.call(this, state, card) + }) + const ensure = index.ensure + vi.spyOn(index, 'ensure').mockImplementation(function ( + this: JournalInputCardIndex, + state, + floor + ) { + watchIndexMap(this['overlays']) + watchIndexMap(state.queue.cards) + const before = this['index'] + ensuring = true + try { + ensure.call(this, state, floor) + } finally { + ensuring = false + } + if (this['index'] !== before) { + counts[phase].indexRebuilds++ + } + }) const Original = legacy.LegacyQueuedCards vi.spyOn(legacy, 'LegacyQueuedCards').mockImplementation( class extends Original { constructor(private readonly source: ConstructorParameters[0]) { super(source) + const read = this['read'].bind(this) + this['read'] = () => { + const rows = read() + countMap(rows, () => counts[phase].sourceIdEntries++) + countFlatMap( + rows, + () => counts[phase].ownerClassifications++, + (cards) => { + if (classifying) { + countFlatMap(cards, () => { + if (classifying) { + counts[phase].handoffVisits++ + } + }) + countMap(cards, () => { + if (classifying) { + counts[phase].identityEntries++ + } + }) + } + } + ) + return rows + } } override cards() { - const cards = super.cards() - if (!arrays.has(cards)) { - arrays.add(cards) - cards.forEach((card, index) => { - Object.defineProperty(cards, index, { - get: () => { - if (displaying) { - counts[phase].cardVisits++ - } - return card - } - }) - }) + classifying = true + try { + const cards = super.cards() + const handedOff = this['viewed']?.handedOff + if (handedOff && !handoffs.has(handedOff)) { + handoffs.add(handedOff) + countIteration(handedOff, () => counts[phase].handoffApplications++) + } + return cards + } finally { + classifying = false } - return cards } override displayState() { + const previous = phase + if (phase !== 'publication') { + phase = 'failure' + } const measured = counts[phase] measured.displays++ const state = this.source.state() @@ -72,13 +280,12 @@ export function countLegacyFailureDisplay() { () => measured.submissionCopies++, () => measured.submissionEntries++ ) - displaying = true try { return super.displayState() } finally { - displaying = false items.mockRestore() submissions.mockRestore() + phase = previous } } } @@ -86,16 +293,18 @@ export function countLegacyFailureDisplay() { return { counts, publication: (publish: () => T): T => { + const previous = phase phase = 'publication' try { return publish() } finally { - phase = 'failure' + phase = previous } }, reset: () => { - Object.assign(counts.publication, emptyCounts()) - Object.assign(counts.failure, emptyCounts()) + for (const value of Object.values(counts)) { + Object.assign(value, emptyLegacyCounts()) + } } } } diff --git a/src/main/runtime/rpc/methods/structured-agent-session-legacy-command-wait.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-legacy-command-wait.test.ts index 4cee22e028b..706271e8506 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-legacy-command-wait.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-legacy-command-wait.test.ts @@ -115,3 +115,65 @@ it.each(['accepted', 'pending', 'oldest'] as const)( } } ) + +it('a pending-result reader gets the original direct receipt behind a waiting command before W ends', async () => { + const rig = await createQueuedMessageTestRig() + setStructuredAgentSessionHost(rig.host) + try { + const working = await rig.workingSend() + const commandId = hostTestOperationId() + const fields = { command: 'compact' as const, delivery: 'queue-if-active' as const } + expect( + await rig.host.conversationCommand(QUEUED_RIG_CALLER, { + ...fields, + envelope: rig.envelope(fields, 'agentSession.conversationCommand', commandId), + userSend: true + }) + ).toMatchObject({ ok: true, value: { queued: { messageId: commandId } } }) + const body = hostTestMessage('M behind waiting compact') + const messageId = hostTestOperationId() + const params = { + body, + envelope: rig.envelope( + { body: agentSessionMessagePayload(body) }, + 'agentSession.send', + messageId + ) + } + const result = await sendStructuredAgentSessionForClient(params, { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the installed-host registry path never reads runtime methods. + runtime: {} as RpcContext['runtime'], + clientId: QUEUED_RIG_CALLER.callerKey, + clientKind: 'runtime', + clientCapabilities: [ + STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY, + AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY + ] + }) + expect(result).toMatchObject({ + ok: true, + value: { clientMessageId: messageId, submission: { dispatchState: 'pending' } } + }) + expect(await rig.submission(messageId)).not.toHaveProperty('handedOverAt') + const replay = await rig.host.send(QUEUED_RIG_CALLER, { ...params, userSend: true }) + expect(replay).toMatchObject({ + ok: true, + replayed: true, + value: { submission: { clientMessageId: messageId } } + }) + expect(rig.compact).not.toHaveBeenCalled() + await rig.settleAccepted(working, 'W') + await eventually(() => expect(rig.compact).toHaveBeenCalledOnce()) + rig.finishCompact() + await eventually(async () => + expect((await rig.submission(messageId))?.handedOverAt).toBeDefined() + ) + await rig.settleAccepted(messageId, 'M') + expect(await rig.submission(messageId)).toMatchObject({ dispatchState: 'accepted' }) + expect( + rig.dispatch.mock.calls.filter(([input]) => input.clientMessageId === messageId) + ).toHaveLength(1) + } finally { + await rig.dispose() + } +}) diff --git a/src/main/runtime/rpc/methods/structured-agent-session-wire-contracts.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-wire-contracts.test.ts index db696da5993..8a12cbf23d1 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-wire-contracts.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-wire-contracts.test.ts @@ -33,7 +33,8 @@ import { } from './structured-agent-session-rpc.test-fixture' import { createQueuedMessageTestRig, - eventually + eventually, + QUEUED_RIG_CALLER } from '../../../native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture' import { HOST_TEST_SESSION as HOST_SESSION, @@ -492,6 +493,15 @@ describe('agentSession.send reply timing', () => { let release = () => {} try { const working = await rig.workingSend() + const compactId = hostTestOperationId() + const compactFields = { command: 'compact' as const, delivery: 'queue-if-active' as const } + expect( + await rig.host.conversationCommand(QUEUED_RIG_CALLER, { + ...compactFields, + envelope: rig.envelope(compactFields, 'agentSession.conversationCommand', compactId), + userSend: true + }) + ).toMatchObject({ ok: true, value: { queued: { messageId: compactId } } }) const image = imageSend() expect( await call('agentSession.send', image.request, STRUCTURED_MOBILE_CLIENT) @@ -499,8 +509,12 @@ describe('agentSession.send reply timing', () => { ok: true, result: { value: { submission: { clientMessageId: image.id } } } }) - expect(await rig.drafts()).toEqual([]) + expect((await rig.drafts()).map((card) => card.messageId)).toEqual([compactId]) + expect((await rig.submission(image.id))?.handedOverAt).toBeUndefined() await rig.settleAccepted(working, 'working') + await eventually(() => expect(rig.compact).toHaveBeenCalledOnce()) + expect((await rig.submission(image.id))?.handedOverAt).toBeUndefined() + rig.finishCompact() await eventually(async () => { expect((await rig.submission(image.id))?.handedOverAt).toBeDefined() expect( diff --git a/src/renderer/src/components/native-chat/structured-agent-session-queue-request.ts b/src/renderer/src/components/native-chat/structured-agent-session-queue-request.ts index bb36bd32410..1a1657a18d0 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-queue-request.ts +++ b/src/renderer/src/components/native-chat/structured-agent-session-queue-request.ts @@ -1,4 +1,4 @@ -import type { StructuredAgentSessionHostCapabilityState } from '@/runtime/structured-agent-session-host-capability' +import type { StructuredAgentSessionHostCapabilityState } from '../../../../shared/structured-agent-session-host-capability' import { structuredAgentSessionFollowUpsQueue, structuredAgentSessionQueueEnabled diff --git a/src/renderer/src/runtime/structured-agent-session-host-capability.ts b/src/renderer/src/runtime/structured-agent-session-host-capability.ts index e37d033cd66..eb8ebea83c0 100644 --- a/src/renderer/src/runtime/structured-agent-session-host-capability.ts +++ b/src/renderer/src/runtime/structured-agent-session-host-capability.ts @@ -1,4 +1,5 @@ import { AGENT_SESSION_PERMISSION_OWN_RUNTIME_CAPABILITY } from '../../../shared/agent-session-permission-modes-capability' +import type { StructuredAgentSessionHostCapabilityState } from '../../../shared/structured-agent-session-host-capability' import { useEffect, useState, useSyncExternalStore } from 'react' import { useAppStore } from '@/store' import { @@ -29,8 +30,7 @@ export function structuredAgentSessionHostKey(target: RuntimeClientTarget): stri return target.kind === 'local' ? 'local' : `environment:${target.environmentId}` } -/** Unknown before verification or after revocation; a contact outage retains the known build. */ -export type StructuredAgentSessionHostCapabilityState = 'unknown' | 'supported' | 'unsupported' +export type { StructuredAgentSessionHostCapabilityState } /** Displays the existing host status; connection lifecycle owns probing. */ export function useStructuredAgentSessionOwnPermissionState( diff --git a/src/shared/structured-agent-session-host-capability.ts b/src/shared/structured-agent-session-host-capability.ts new file mode 100644 index 00000000000..09976231b03 --- /dev/null +++ b/src/shared/structured-agent-session-host-capability.ts @@ -0,0 +1,2 @@ +/** Unknown before verification or after revocation; outages retain the known build. */ +export type StructuredAgentSessionHostCapabilityState = 'unknown' | 'supported' | 'unsupported'