Preserve queued command and card order through recovery (#36430)

* Advertise mid-turn queueing as queued-messages.v2

Released clients read the v1 name with an unfinished queue UI; they never
see v2 and keep sending immediately.

* Let each desktop send choose queue or steer while the agent works

The follow-up choice is now a Queue / Steer setting saved under
nativeChatFollowUpBehavior, Steer by default; the old
nativeChatQueueFollowUps switch, which every install saved as true
while queueing was dark, is retired on load. Every reader goes through
native-chat-follow-up-behavior.

While the agent works, Enter follows the setting and Cmd/Ctrl+Enter
sends the draft the other way; the alternate flips the setting before
the queue's own rules (a waiting command still queues, prompts no one
here can answer still send now) and never applies while Stopping.
Cmd/Ctrl+Shift+Enter steers the head card only and leaves the draft.
An idle agent's card reads Send, and the card menu no longer offers
"Turn off queueing".

* Hold Send on the phone to queue or steer a message mid-turn

While the agent works on a host that queues, holding Send opens a
Queue / Steer sheet; a choice other than the host's setting sends the
alternate, resolved against the chat as it is when it lands, by the
desktop's rule. Release and dismiss send nothing. The phone reads the
host's new follow-up setting (absent reads as Steer), and an idle
agent's card reads Send.

* Read the follow-up default from one place on desktop and phone

* Keep phone follow-up choices aligned with live host state

* Update stream and wire fixtures for live follow-up preferences

* Document minimal runtime test fixture assertions

* Keep mobile preference tests in the mobile compiler project

* Preserve queued input order through handover and recovery

* Consolidate input selection type imports

* Keep queue request capability types in the shared compiler graph

* Preserve reopen admission context and bound queue projection work

* feat(native-chat): queue follow-ups by default with a toggle

* fix(tests): preserve projection contracts in the node compiler

* fix(native-chat): retain receipt lookup after journal replacement

* test(mobile): assert shared preference renders on real changes

* Remove obsolete follow-up option translations

* Preserve input places across journal recovery without predecessor snapshots

* Keep recovered card placement tied to the exact handoff

* Preserve replay source types and correct verification fixtures

* Complete indexed queue selection and recovery ordering

* Preserve queued input order across recovery and source replay

* Fix legacy queue recovery fixture types

* Use a specific name for queue pause inputs

* Preserve fallback queue identity across journal recovery

* Place nested storage negative directive on its constrained call

* Constrain journal draft property type parameter

* Remove journal input order import cycle

* Preserve lazy journal counter reads within line limit

* test: preserve legacy conversion and display work contracts

* test: align journal refusal and restore accounting contracts
This commit is contained in:
Brennan Benson
2026-10-31 10:17:56 -07:00
committed by GitHub
parent de192f1e96
commit c64e1f615a
82 changed files with 6828 additions and 379 deletions
+1
View File
@@ -26,6 +26,7 @@
"../src/renderer/src/components/native-chat/structured-agent-question-projection.ts",
"../src/renderer/src/components/native-chat/native-chat-subagent-delegation.ts",
"../src/renderer/src/components/native-chat/structured-agent-session-pending-sends.ts",
"../src/renderer/src/components/native-chat/structured-agent-session-queue-request.ts",
"../src/renderer/src/components/native-chat/native-chat-composer-primary-action.ts",
"../src/renderer/src/components/terminal-pane/legacy-relay-bridge-panes.ts",
"../src/preload/**/*",
@@ -20,6 +20,7 @@ import {
} from '../native-chat/agent-session-journal/journal-host-database-test-support'
import type { JournalLoad } from '../native-chat/agent-session-journal/journal-open'
import { publishJournalOutcomeEpoch } from '../native-chat/agent-session-journal/journal-outcome-write'
import { JOURNAL_OUTCOME_FORMAT } from '../native-chat/agent-session-journal/journal-outcome-format'
import type { JournalRow } from '../native-chat/agent-session-journal/journal-row-schema'
import { createJournalReducerState } from '../native-chat/agent-session-journal/journal-reducer'
import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key'
@@ -226,8 +227,13 @@ describe('ACP launch resolution', () => {
// The turn on acp-1 is found, so the exchange is saved; the loss has no warning yet.
expect(await readDecisions()).toEqual({ unsaved: false, losses: [lostKey] })
// A newer build's records are unreadable here, never a missing chat.
database.db.prepare('UPDATE journal_outcome_epochs SET format = 2').run()
expect(restoreJournal(database.db, identity.sessionId)?.newer).toBeTruthy()
database.db
.prepare('UPDATE journal_outcome_epochs SET format = ?')
.run(JOURNAL_OUTCOME_FORMAT + 1)
const newer = restoreJournal(database.db, identity.sessionId)
expect(newer).not.toBeNull()
expect(newer?.newer).toStrictEqual({ sequence: 1 })
expect(newer?.damage).toBeNull()
expect(await readDecisions()).toEqual({ unsaved: false, losses: [] })
} finally {
await closeTestJournalHostDatabase(root)
@@ -0,0 +1,12 @@
import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types'
import type { JournalReducerState } from './journal-reducer'
export function retireCoveredJournalFloor(
state: JournalReducerState,
reopenFloor: AgentJournalCursor | null
): AgentJournalCursor | null {
return reopenFloor?.epoch === state.epoch &&
state.queuePauseMarks.reopenedSequence >= reopenFloor.sequence
? null
: reopenFloor
}
@@ -1,3 +1,4 @@
import { journalInputOrder } from './journal-input-order'
import { journalOutcomeSource } from './journal-outcome-format'
import { publishJournalOutcomeEpoch } from './journal-outcome-write'
import {
@@ -53,6 +54,8 @@ import { AgentSessionJournalError, assertJournalFence } from './journal-write-gu
import type { JournalOperationReceipt } from './journal-row-writer'
import type { JournalAttachmentClaim } from './journal-submission-hook'
import { copyJournalNativeState } from './journal-input-conversion'
import { journalInputAcceptance, journalQueueRebase } from './journal-input-provenance'
export type JournalReplacementItem = AgentJournalProducerLinkage & {
echoOf?: JournalItemEcho['echoOf']
@@ -88,7 +91,13 @@ export function replaceJournalEpoch(input: {
onPublished: (loaded: JournalLoad) => void
}): void {
const epoch = input.mintEpoch()
const state = createJournalReducerState(input.identity.sessionId, epoch)
const state = createJournalReducerState(input.identity.sessionId, epoch, (id) =>
journalInputAcceptance(input.database.db, input.identity.sessionId, id)
)
if (input.previous) {
copyJournalNativeState(input.previous, state)
}
journalInputOrder(state).beginReconstruction()
const providerHandle = agentSessionJournalProviderHandle(input.identity)
const previousOrigin = input.previous?.historyOrigin
const restate =
@@ -96,6 +105,9 @@ export function replaceJournalEpoch(input: {
input.reason !== 'session_created' &&
previousOrigin &&
sameJournalConversation(previousOrigin.providerHandle, providerHandle)
const queued = [...(input.previous?.submissions.values() ?? [])]
.filter(isQueuedAgentJournalSubmission)
.sort((a, b) => a.submittedSequence! - b.submittedSequence!)
const epochRow: JournalRow = {
kind: 'epoch',
reason:
@@ -133,23 +145,12 @@ export function replaceJournalEpoch(input: {
applyJournalRow(state, row)
rows.push(row)
}
for (const { messageId, change } of input.queue.cards) {
if (change) {
const row = buildJournalQueueRow(
state,
messageId,
change,
state.lastSequence + 1,
input.now()
)
applyJournalRow(state, row)
rows.push(row)
}
const appendQueue = (messageId: string, change: Parameters<typeof buildJournalQueueRow>[2]) => {
const row = buildJournalQueueRow(state, messageId, change, state.lastSequence + 1, input.now())
applyJournalRow(state, row)
rows.push(row)
}
// Accepted-but-unhanded messages must survive provider history replacement.
const queued = [...(input.previous?.submissions.values() ?? [])]
.filter(isQueuedAgentJournalSubmission)
.sort((a, b) => (a.acceptedSequence ?? 0) - (b.acceptedSequence ?? 0))
const rebuilt = new Map<string, number>()
for (const submission of queued) {
const body = input.previous?.items.get(
agentJournalSubmissionKey(submission.clientMessageId)
@@ -168,6 +169,34 @@ export function replaceJournalEpoch(input: {
})
applyJournalRow(state, row)
rows.push(row)
rebuilt.set(submission.clientMessageId, row.seq)
}
const rebase = journalQueueRebase(
input.database.db,
input.identity.sessionId,
epoch,
input.previous?.epoch,
queued,
rebuilt
)
for (const { messageId, change } of input.queue.cards) {
if (!change) {
continue
}
const prior = input.previous?.queue.cards.get(messageId)
const queuedAt = change.op === 'queued' ? rebase(change.queuedAt) : null
appendQueue(
messageId,
change.op === 'queued'
? {
...change,
queuedAt,
...(prior?.state === 'dispatched' && prior.consumedAs
? { attempt: prior.consumedAs }
: {})
}
: change
)
}
const { lifted, liveStop, reopened } = input.queuePause
const place = () => ({ state, seq: state.lastSequence + 1, fence: input.fence, ts: input.now() })
@@ -192,6 +221,8 @@ export function replaceJournalEpoch(input: {
rows.push(row)
}
journalInputOrder(state).restore(state)
journalInputOrder(state).finishReconstruction(state)
const { sessionId } = input.identity
const receipt = input.receipt?.({ epoch, sequence: state.lastSequence })
const claimsCommitted: (() => void)[] = []
@@ -225,7 +256,7 @@ export function replaceJournalEpoch(input: {
input.queue.receipt.write(db)
publishJournalSessionEpoch(db, input.identity, epoch)
receipt?.write(db)
return publishJournalOutcomeEpoch(db, sessionId, state.epoch, rows)
return publishJournalOutcomeEpoch(db, sessionId, state.epoch, rows, input.previous)
},
false,
(publication) => {
@@ -1,3 +1,4 @@
import type { AgentSessionJournalProviderHandle } from '../../../shared/agent-session-journal-types'
// Share one connection per test profile to match production host ownership.
import { resolve } from 'node:path'
@@ -197,6 +198,44 @@ export function deleteTestJournalRow(db: Database.Database, sessionId: string, s
)
}
/** Unannotated submissions and terminal rows emitted by the legacy writer in one source epoch. */
export function writeTestLegacyTerminalGaps(
db: Database.Database,
sessionId: string,
base: {
epoch: string
sequence: number
fence: number
providerHandle: AgentSessionJournalProviderHandle
},
n: number
): number {
let seq = base.sequence
const row = () => ({ v: 3, epoch: base.epoch, seq: ++seq, fence: base.fence, ts: 1 })
for (let i = 0; i < n; i++) {
const id = `terminal-${i}`
insertTestJournalRow(db, sessionId, {
...row(),
kind: 'submission',
clientMessageId: id,
payloadFingerprint: id,
body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: id }] },
providerHandle: base.providerHandle,
handoverRecorded: true,
origin: 'client'
})
insertTestJournalRow(db, sessionId, {
...row(),
kind: 'dispatch',
clientMessageId: id,
state: 'rejected',
providerItemId: null,
reason: 'not sent'
})
}
return seq
}
/** Read current canonical contributions in commit order for persisted-outcome assertions. */
export function readTestJournalOutcomes(
db: Database.Database,
@@ -0,0 +1,279 @@
import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types'
import type { JournalSubmissionRow } from './journal-row-schema'
import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission'
import type { JournalReducerState } from './journal-reducer'
import type { QueuedMessageRow } from './queued-message-table'
import type { JournalInputPlace } from './journal-input-order'
import {
queuePauseHolding,
journalQueueStopHolds,
type DerivedQueuePause
} from './queued-message-pause'
import {
JournalInputPlaceIndex,
mergeInputPlaces,
type InputPlaceQuery
} from './journal-input-place-index'
import {
type JournalInputLineageOwner,
type JournalInputCard,
queryInputContext
} from './journal-input-lineage'
import { JournalInputDomainIndex } from './journal-input-domain-index'
export class JournalInputCardIndex {
private index: JournalInputPlaceIndex | undefined
private currentCoordinates: JournalInputPlaceIndex | undefined
private domains: JournalInputDomainIndex | undefined
private readonly indexed = new Map<
string,
{
card: QueuedMessageRow
actionable: boolean
handoff: boolean
source?: JournalInputCard
}
>()
// Derived current membership only; the prepared transition owns introduction values.
private readonly unintroduced = new Set<string>()
private overlayRows: readonly QueuedMessageRow[] = []
private overlays = new Map<string, QueuedMessageRow>()
overlay(state: JournalReducerState, rows: readonly QueuedMessageRow[]): void {
if (rows === this.overlayRows || (rows.length === 0 && this.overlays.size === 0)) {
return
}
this.ensure(state)
const next = new Map(rows.map((card) => [card.messageId, card]))
for (const card of this.overlays.values()) {
if (!next.has(card.messageId) && !state.queue.cards.has(card.messageId)) {
this.indexCard(state, { ...card, state: 'withdrawn' })
this.unintroduced.delete(card.messageId)
}
}
this.overlayRows = rows
this.overlays = next
for (const card of rows) {
if (!state.queue.cards.has(card.messageId)) {
if (!this.lineage.cards.has(card.messageId)) {
this.unintroduced.add(card.messageId)
}
this.indexCard(state, card)
}
}
}
introduceOverlays(change: (card: QueuedMessageRow) => void): void {
for (const id of this.unintroduced) {
change(this.overlays.get(id)!)
}
this.unintroduced.clear()
}
private pauseKey: readonly unknown[] = []
private floor: { epoch: string; sequence: number } | null = null
private pauses: readonly DerivedQueuePause[] = []
private cleared = new Set<string>()
constructor(private readonly lineage: JournalInputLineageOwner) {}
hasOverlay(id: string): boolean {
return this.overlays.has(id)
}
command(): QueuedMessageRow | undefined {
return this.currentCoordinates?.allPlaces().command
}
releaseHistory(): void {
this.domains = undefined
}
head(): QueuedMessageRow | undefined {
return this.index?.allPlaces().head
}
paused(card: QueuedMessageRow): boolean {
return (
card.state === 'waiting' &&
card.holdReason === null &&
(this.cleared.has(card.messageId) || Boolean(queuePauseHolding(this.pauses, card)))
)
}
admission(
state: JournalReducerState,
row: JournalSubmissionRow,
originalAcceptance?: (id: string) => AgentJournalCursor | undefined
): JournalInputPlace {
this.ensure(state)
const coordinate = { epoch: row.epoch, sequence: row.seq }
const context = row.inputOrderContext ?? {
source: coordinate,
basis: 'legacy' as const,
cut: this.lineage.at(row.seq)
}
const original =
context.basis === 'legacy' ? originalAcceptance?.(row.clientMessageId) : undefined
const place: JournalInputPlace = { coordinate, context, ...(original ? { original } : {}) }
const exposed = row.inputOrderContext?.basis === 'legacy' ? this.command() : undefined
const selected =
row.inputOrderContext?.basis === 'legacy'
? exposed && this.relation(exposed, place)
? exposed
: undefined
: !row.inputOrderContext
? this.queryCurrent(place).command
: this.query(place).command
return { ...place, ...(selected ? { witnessCardId: selected.messageId } : {}) }
}
queryCurrent(place: JournalInputPlace): InputPlaceQuery {
return queryInputContext(
place.context,
place.original,
(coordinate) => this.currentCoordinates!.query(coordinate.epoch, coordinate.sequence),
() => this.currentCoordinates!.allPlaces(),
mergeInputPlaces
)
}
query(place: JournalInputPlace): InputPlaceQuery {
if (place.context.basis === 'observed') {
return this.index!.query('introduction', place.context.cut)
}
if (!this.domains) {
this.domains = new JournalInputDomainIndex()
for (const { card, source, actionable, handoff } of this.indexed.values()) {
const known = this.lineage.cards.get(card.messageId) ?? source
if (known) {
this.domains.set(card, known, actionable, handoff)
}
}
}
return this.domains.query(place.context, place.original)
}
ensure(state: JournalReducerState, floor?: { epoch: string; sequence: number } | null): void {
if (floor !== undefined) {
this.floor = floor
}
floor = this.floor
const marks = state.queuePauseMarks
const lifted = Math.max(state.latestAcceptedTurnSequence, marks.resumedSequence)
const reopened = Math.max(
marks.reopenedSequence,
floor?.epoch === state.epoch ? floor.sequence : 0
)
const pauseInputs = [
journalQueueStopHolds(marks, state.latestAcceptedTurnSequence) ? marks.latestStop : null,
reopened > lifted ? reopened : 0,
marks.cleared && !marks.cleared.lifted && marks.cleared.sequence > lifted
? marks.cleared
: null
]
if (!this.index || pauseInputs.some((value, i) => value !== this.pauseKey[i])) {
this.pauseKey = pauseInputs
this.cleared = new Set(pauseInputs[2] ? marks.cleared?.messageIds : [])
this.index = new JournalInputPlaceIndex('introduction')
this.currentCoordinates = new JournalInputPlaceIndex(state.epoch)
this.domains = undefined
this.indexed.clear()
this.pauses = [
...(pauseInputs[0] && marks.latestStop
? [
{
reason: 'stopped' as const,
since: { epoch: state.epoch, sequence: marks.latestStop.sequence }
}
]
: []),
...(pauseInputs[1]
? [
{
reason: 'restarted' as const,
since: { epoch: state.epoch, sequence: reopened }
}
]
: [])
]
for (const card of this.overlays.values()) {
if (!state.queue.cards.has(card.messageId)) {
this.indexCard(state, card)
}
}
for (const card of state.queue.cards.values()) {
this.indexCard(state, card)
}
}
}
queueChanged(
state: JournalReducerState,
prior: QueuedMessageRow | undefined,
card: QueuedMessageRow
): void {
this.ensure(state)
const sameCoordinate =
prior?.queuedAt?.epoch === card.queuedAt?.epoch &&
prior?.queuedAt?.sequence === card.queuedAt?.sequence
if (
prior &&
prior.position === card.position &&
prior.state === card.state &&
prior.holdReason === card.holdReason &&
prior.consumedAs === card.consumedAs &&
sameCoordinate &&
Boolean(prior.body.command) === Boolean(card.body.command)
) {
return
}
this.lineage.change(card.messageId, card.queuedAt, state.lastSequence)
this.indexCard(state, card)
}
indexCard(state: JournalReducerState, card: QueuedMessageRow): void {
const attempt = card.consumedAs ? state.submissions.get(card.consumedAs) : undefined
const cleared = this.cleared.has(card.messageId)
const actionable =
!cleared &&
card.state === 'waiting' &&
card.holdReason === null &&
!queuePauseHolding(this.pauses, card)
const handoff = attempt !== undefined && isQueuedAgentJournalSubmission(attempt)
const lineage = this.lineage.cards.get(card.messageId) ?? {
introduced: this.lineage.at(state.lastSequence),
baseline: { coordinate: card.queuedAt }
}
this.currentCoordinates!.set(card, actionable, handoff)
this.index!.set(card, actionable, handoff, {
epoch: 'introduction',
sequence: lineage.introduced
})
if (this.domains && lineage) {
this.domains.set(card, lineage, actionable, handoff)
}
if (card.state === 'withdrawn') {
this.indexed.delete(card.messageId)
return
}
this.indexed.set(card.messageId, {
card,
actionable,
handoff,
...(!this.lineage.cards.has(card.messageId) ? { source: lineage } : {})
})
}
relation(card: QueuedMessageRow, place: JournalInputPlace): boolean {
const lineage =
this.lineage.cards.get(card.messageId) ?? this.indexed.get(card.messageId)?.source
if (!lineage || lineage.introduced >= place.context.cut) {
return false
}
if (place.context.basis === 'observed') {
return true
}
if (!lineage.baseline) {
return false
}
const coordinate = lineage.baseline.coordinate
return queryInputContext(
place.context,
place.original,
(boundary) => coordinate?.epoch === boundary.epoch && coordinate.sequence < boundary.sequence,
() => true,
(a, b) => a || b
)
}
}
@@ -0,0 +1,108 @@
import { readJournalLegacyCards } from './journal-queue-import'
import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission'
import type { JournalReducerState } from './journal-reducer'
import { initializeJournalInputOrder, journalInputOrder } from './journal-input-order'
import type { JournalOutcomeFold } from './journal-outcome-codec'
import type { JournalInputCard, JournalInputEntry } from './journal-input-lineage'
import type Database from '../../sqlite/sync-database'
import { journalInputAcceptance } from './journal-input-provenance'
export type JournalNativePrefix = {
entries: Map<string, JournalInputEntry>
cards: Map<string, JournalInputCard>
base: number
}
/** Marker-5 ordinary keys remain frozen; native evidence is lifted only after tail overrides. */
export function detachJournalNativePrefix(state?: JournalReducerState): JournalNativePrefix {
const prefix = {
entries: new Map(state?.inputEntries),
cards: new Map(state?.inputCards),
base: state ? journalInputOrder(state).lineage.at(0) : 0
}
state?.inputEntries.clear()
state?.inputCards.clear()
return prefix
}
export function restoreJournalNativeState(
db: Database.Database,
state: JournalReducerState,
format: number,
fold: JournalOutcomeFold,
prefix: JournalNativePrefix
): void {
if (format < 2) {
for (const [id, own] of prefix.entries) {
if (
state.submissions.get(id)?.dispatchState === 'pending' &&
!fold.replaced('submissions', id)
) {
state.inputEntries.set(id, own)
}
}
for (const [id, card] of prefix.cards) {
if (!state.queue.cards.has(id) && !fold.replaced('queue.cards', id)) {
state.inputCards.set(id, card)
}
}
for (const [id, card] of state.queue.cards) {
const known = !fold.replaced('queue.cards', id) ? prefix.cards.get(id) : undefined
const introducedAt = fold.introduction('queue.cards', id)
if (known) {
state.inputCards.set(id, known)
} else if (introducedAt !== undefined) {
state.inputCards.set(id, {
introduced: prefix.base + introducedAt,
baseline: { coordinate: card.queuedAt }
})
}
}
for (const [id, entry] of state.submissions) {
const sequence = entry.acceptedSequence ?? entry.submittedSequence
if (
!state.inputEntries.has(id) &&
isQueuedAgentJournalSubmission(entry) &&
!entry.queuedMessageId &&
sequence !== undefined
) {
state.inputEntries.set(id, {
context: {
source: { epoch: state.epoch, sequence },
basis: 'legacy',
cut: prefix.base + sequence
}
})
}
}
}
const owner = initializeJournalInputOrder(state, (id) =>
journalInputAcceptance(db, state.sessionId, id)
)
owner.beginReconstruction()
owner.restore(state)
owner.overlay(state, readJournalLegacyCards(db, state))
owner.finishReconstruction(state)
}
/** Replacement carries facts directly; restatement is not a new admission or introduction. */
export function copyJournalNativeState(
previous: JournalReducerState,
state: JournalReducerState
): void {
for (const [id, own] of previous.inputEntries) {
state.inputEntries.set(id, {
...own,
context: { ...own.context, source: { ...own.context.source } }
})
}
for (const [id, card] of previous.inputCards) {
state.inputCards.set(id, {
...card,
...(card.baseline
? { baseline: { coordinate: card.baseline.coordinate && { ...card.baseline.coordinate } } }
: {})
})
}
journalInputOrder(state).lineage.restoreFrontier(state.lastSequence, state.inputEntries.values())
}
@@ -0,0 +1,85 @@
import {
queryInputContext,
type JournalInputCard,
type JournalInputContext
} from './journal-input-lineage'
import {
JournalInputPlaceIndex,
mergeInputPlaces,
type InputPlaceQuery
} from './journal-input-place-index'
import type { QueuedMessageRow } from './queued-message-table'
import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types'
type Node = { index: JournalInputPlaceIndex; left?: Node; right?: Node }
const LIMIT = Number.MAX_SAFE_INTEGER
/** A fixed-width introduction prefix, with coordinate summaries on each of its 53 levels. */
export class JournalInputDomainIndex {
private readonly root: Node = { index: new JournalInputPlaceIndex('') }
set(
card: QueuedMessageRow,
source: JournalInputCard,
actionable: boolean,
handoff: boolean
): void {
if (!source.baseline) {
return
}
let node = this.root,
low = 0,
high = LIMIT
while (true) {
node.index.set(card, actionable, handoff, source.baseline.coordinate)
if (high - low <= 1) {
break
}
const mid = low + Math.floor((high - low) / 2)
if (source.introduced < mid) {
node = node.left ??= { index: new JournalInputPlaceIndex('') }
high = mid
} else {
node = node.right ??= { index: new JournalInputPlaceIndex('') }
low = mid
}
}
}
query(context: JournalInputContext, original?: AgentJournalCursor): InputPlaceQuery {
let result: InputPlaceQuery = { position: -Infinity }
const append = (node: Node) => {
result = mergeInputPlaces(
result,
queryInputContext(
context,
original,
(coordinate) => node.index.query(coordinate.epoch, coordinate.sequence),
() => node.index.allPlaces(),
mergeInputPlaces
)
)
}
let node: Node | undefined = this.root,
low = 0,
high = LIMIT
while (node && low < context.cut) {
if (high <= context.cut) {
append(node)
break
}
const mid = low + Math.floor((high - low) / 2)
if (context.cut <= mid) {
node = node.left
high = mid
} else {
if (node.left) {
append(node.left)
}
node = node.right
low = mid
}
}
return result
}
}
@@ -0,0 +1,74 @@
import { vi } from 'vitest'
import { JournalInputLineageOwner } from './journal-input-lineage'
/** Instrument real index nodes without charging the test's discovery walk as production work. */
export function journalInputIndexProbe(
keys: readonly string[],
count: { allocated: () => void; read: () => void }
): ((node: unknown) => void) & { restore: () => void } {
const seen = new WeakMap<object, Map<string, unknown>>()
const originals: { node: object; key: string; descriptor?: PropertyDescriptor }[] = []
const inspect = (node: unknown): void => {
if (typeof node !== 'object' || node === null) {
return
}
const prior = seen.get(node)
if (prior) {
inspect(prior.get('left'))
inspect(prior.get('right'))
return
}
const values = new Map<string, unknown>()
seen.set(node, values)
count.allocated()
for (const key of keys) {
const descriptor = Object.getOwnPropertyDescriptor(node, key)
originals.push({ node, key, descriptor })
values.set(key, descriptor?.value)
if (key === 'left' || key === 'right') {
inspect(values.get(key))
}
Object.defineProperty(node, key, {
configurable: true,
get: () => {
count.read()
return values.get(key)
},
set: (next) => {
values.set(key, next)
}
})
}
}
return Object.assign(inspect, {
restore: () => {
for (const { node, key, descriptor } of originals) {
const value = seen.get(node)?.get(key)
if (descriptor || value !== undefined) {
Object.defineProperty(node, key, {
...descriptor,
value,
writable: true,
configurable: true
})
} else {
Reflect.deleteProperty(node, key)
}
}
originals.length = 0
}
})
}
export function journalInputHistoryProbe() {
let visits = 0
const release = JournalInputLineageOwner.prototype.releaseHistory
const spy = vi
.spyOn(JournalInputLineageOwner.prototype, 'releaseHistory')
.mockImplementation(function (this: JournalInputLineageOwner, keep) {
release.call(this, (id) => {
visits++
return keep(id)
})
})
return { spy, visits: () => visits }
}
@@ -0,0 +1,296 @@
import { journalInputOrder } from './journal-input-order'
import {
journalInputIndexProbe,
journalInputHistoryProbe
} from './journal-input-index-probe.test-support'
import { JournalInputDomainIndex } from './journal-input-domain-index'
import { JournalInputPlaceIndex } from './journal-input-place-index'
import { insertCommandReceiptIfAbsent } from './command-receipt-table'
import { writeCommandReceiptTestRecord } from './command-receipt-test-support'
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { expect, it, vi } from 'vitest'
import {
claudeProviderHandle,
agentSessionJournalProviderHandle
} from '../../../shared/agent-session-provider-handle-encoding'
import type { JournalRow } from './journal-row-schema'
import type { JournalReducerState } from './journal-reducer'
import type { AgentJournalMessageItem } from '../../../shared/agent-session-journal-types'
import { holdUnsentSends } from './journal-unsent-send-hold'
import {
createTrackedJournalOpener,
loadTestJournal,
openTestJournalHostDatabase,
readTestJournalOutcomes,
insertTestJournalRow,
writeTestLegacyTerminalGaps
} from './journal-host-database-test-support'
const message = (text: string): AgentJournalMessageItem => ({
kind: 'message',
role: 'user',
blocks: [{ type: 'text', text }]
})
it.each([10, 100])(
'indexes the initial legacy writer shape without promoting foreign/null comparison N=%i',
async (n) => {
const root = await mkdtemp(join(tmpdir(), 'orca-legacy-baseline-'))
const journals = createTrackedJournalOpener({ legacyInputs: true })
const identity = {
sessionId: 'baseline',
workspaceId: 'folder',
hostId: 'ssh',
agent: 'claude' as const,
providerHandle: claudeProviderHandle('thread', null)
}
let journal = await journals.open({ identity, stateDirectory: root })
const database = openTestJournalHostDatabase(root)
const cost = { sets: 0, queries: 0, nodes: 0, reads: 0 }
const inspect = journalInputIndexProbe(['index', 'left', 'right'], {
allocated: () => {
cost.nodes++
},
read: () => {
cost.reads++
}
})
let inspecting = true
const set = JournalInputDomainIndex.prototype.set
const domainSets = vi
.spyOn(JournalInputDomainIndex.prototype, 'set')
.mockImplementation(function (this: JournalInputDomainIndex, ...args) {
cost.sets++
if (inspecting) {
inspect(this['root'])
}
set.apply(this, args)
if (inspecting) {
inspect(this['root'])
}
})
const query = JournalInputDomainIndex.prototype.query
const domainQueries = vi
.spyOn(JournalInputDomainIndex.prototype, 'query')
.mockImplementation(function (this: JournalInputDomainIndex, ...args) {
cost.queries++
return query.apply(this, args)
})
const indexSets = vi.spyOn(JournalInputPlaceIndex.prototype, 'set')
const release = journalInputHistoryProbe()
try {
const { epoch, sequence } = journal.cursor()
let seq = sequence
await journal.close()
const base = () => ({ v: 3, epoch, seq: ++seq, fence: 0, ts: 1 })
const queue = (
id: string,
queuedAt: { epoch: string; sequence: number } | null,
command = false
) =>
insertTestJournalRow(database.db, identity.sessionId, {
...base(),
kind: 'queue',
messageId: id,
change: {
op: 'queued',
body: command ? { ...message('/compact'), command: { name: 'compact' } } : message(id),
position: id === 'C' ? 1 : id === 'D' ? 2 : 3,
queuedAt
}
})
queue('C', { epoch, sequence: seq }, true)
queue('F', { epoch: 'original-foreign', sequence: 1 })
queue('N', null)
database.transaction((db) => writeCommandReceiptTestRecord(db, identity.sessionId))
const ids = Array.from({ length: n }, (_, i) => `M${i + 1}`)
for (let i = 0; i < n; i++) {
queue(`ordinary-${i}`, { epoch, sequence: seq })
}
queue('D', { epoch: 'original-foreign', sequence: 1 })
seq = writeTestLegacyTerminalGaps(
database.db,
identity.sessionId,
{
epoch,
sequence: seq,
fence: 0,
providerHandle: agentSessionJournalProviderHandle(identity)
},
n
)
for (const id of ids) {
const target: JournalRow = {
...base(),
kind: 'submission',
clientMessageId: id,
payloadFingerprint: id,
body: message(id),
providerHandle: agentSessionJournalProviderHandle(identity),
handoverRecorded: true,
origin: 'client'
}
insertTestJournalRow(database.db, identity.sessionId, target)
if (id !== 'M2') {
database.transaction((db) =>
insertCommandReceiptIfAbsent(
db,
{ kind: 'global' },
{
operationId: id,
sessionId: identity.sessionId,
callerKey: 'fixture',
method: 'agentSession.send',
fingerprint: id,
status: 'accepted',
acceptedAt: 1,
result: { kind: 'journal-row', epoch: 'original-foreign', sequence: 100 }
}
)
)
}
}
queue('later-F', { epoch: 'original-foreign', sequence: 1 })
journal = await journals.open({ identity, stateDirectory: root })
expect(release.spy).not.toHaveBeenCalled()
const check = (state: JournalReducerState) => {
for (const card of ['F', 'D']) {
expect(
['M1', 'M2', 'M3'].map((id) =>
journalInputOrder(state).precedes(
state.queue.cards.get(card)!,
state.submissions.get(id)!
)
)
).toEqual([true, false, true])
}
for (const card of ['N', 'later-F']) {
expect(
['M1', 'M2', 'M3'].map((id) =>
journalInputOrder(state).precedes(
state.queue.cards.get(card)!,
state.submissions.get(id)!
)
)
).toEqual([false, false, false])
}
expect(['M1', 'M2', 'M3'].every((id) => journalInputOrder(state).forced(id))).toBe(true)
}
check(journal.queuedMessages.displayState())
for (const id of ids) {
expect(
journal.inputOrder.predecessors(journal.queuedMessages.displayState(), id).position
).toBe(3)
}
// Accessor probes count the old raw fold only; remove them before keyed transaction drafts.
const rawCost = { ...cost }
expect(rawCost.nodes).toBeGreaterThan(0)
expect(rawCost.nodes).toBeLessThanOrEqual(54 * rawCost.sets)
expect(rawCost.reads).toBeLessThanOrEqual(4 * 54 * (rawCost.sets + rawCost.queries))
inspect.restore()
inspecting = false
for (let pass = 0; pass < 3; pass++) {
domainSets.mockClear()
domainQueries.mockClear()
indexSets.mockClear()
if (pass) {
await journal.replaceEpochItems('unreconcilable_prefix', 0, [])
}
check(journal.queuedMessages.displayState())
const replay = loadTestJournal(root, identity.sessionId)!
expect(replay.damage).toBeNull()
check(replay.state)
if (pass) {
const outcomes = readTestJournalOutcomes(database.db, identity.sessionId)
const rows = [...new Set(outcomes.map((value) => value.seq))]
expect(rows.length).toBeLessThanOrEqual(2 * n + 12)
const baseline = [...journal.queuedMessages.displayState().inputCards.values()].filter(
(card) => card.baseline !== undefined
).length
expect(baseline).toBeLessThanOrEqual(n + 6)
for (const id of ids) {
journal.inputOrder.predecessors(journal.queuedMessages.displayState(), id)
journalInputOrder(replay.state).predecessors(replay.state, id)
}
expect(domainQueries).toHaveBeenCalledTimes(2 * n)
// Each independent fold adds at most one command-witness transition beyond the frozen baseline; each update traverses at most 54 introduction nodes.
expect(domainSets.mock.calls.length).toBeLessThanOrEqual(4 * (n + 6) + 2 * rows.length)
expect(indexSets.mock.calls.length).toBeLessThanOrEqual(
54 * domainSets.mock.calls.length + 8 * rows.length
)
console.info(
'legacy-normalized-output',
JSON.stringify({
n,
pass,
rows: rows.length,
baseline,
domainSets: domainSets.mock.calls.length,
domainQueries: domainQueries.mock.calls.length,
indexSets: indexSets.mock.calls.length
})
)
}
}
domainSets.mockClear()
domainQueries.mockClear()
indexSets.mockClear()
await holdUnsentSends(journal, { fence: 0, hold: { cause: 'hostRestarted' } })
expect(domainQueries).toHaveBeenCalledTimes(n)
// One finite-rank move per current card and one keep per input, each updating its one frozen coordinate through at most 54 introduction nodes.
expect(domainSets.mock.calls.length).toBeLessThanOrEqual(16 * (n + 6))
expect(indexSets.mock.calls.length).toBeLessThanOrEqual(
54 * domainSets.mock.calls.length + 16 * (n + 6)
)
console.info(
'legacy-recovery-index',
JSON.stringify({
n,
domainSets: domainSets.mock.calls.length,
domainQueries: domainQueries.mock.calls.length,
indexSets: indexSets.mock.calls.length
})
)
for (const id of ids) {
expect(journal.submission(id)).toMatchObject({
dispatchState: 'rejected',
keptAsQueuedMessageId: id
})
}
const ranks = journal.queuedMessages
.list()
.filter((card) => card.state === 'waiting')
.map((card) => card.position)
expect(
ranks.every((rank, i) => Number.isFinite(rank) && (i === 0 || rank > ranks[i - 1]))
).toBe(true)
console.info('frozen-baseline-raw-nodes', JSON.stringify({ n, ...rawCost }))
expect(release.spy).toHaveBeenCalledOnce()
expect(release.visits()).toBe(2 * n + 5)
const finished = loadTestJournal(root, identity.sessionId)!
expect(finished.damage).toBeNull()
expect(release.spy).toHaveBeenCalledTimes(2)
expect(release.visits()).toBe(2 * (2 * n + 5))
expect(
[...journalInputOrder(finished.state).lineage.cards.values()].every(
(card) => !card.baseline
)
).toBe(true)
expect(journal.inputOrder['cards']['domains']).toBeUndefined()
expect(journal.queuedMessages.displayState().inputEntries.size).toBe(0)
expect(finished.state.inputEntries.size).toBe(0)
expect(
[...journal.inputOrder.lineage.cards.values()].every((card) => card.baseline === undefined)
).toBe(true)
} finally {
inspect.restore()
domainSets.mockRestore()
domainQueries.mockRestore()
indexSets.mockRestore()
release.spy.mockRestore()
await journals.closeAll()
await rm(root, { recursive: true, force: true })
}
}
)
@@ -0,0 +1,127 @@
import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types'
export type JournalInputContext = {
source: AgentJournalCursor
basis: 'observed' | 'legacy'
cut: number
}
export type JournalInputCard = {
introduced: number
baseline?: { coordinate: AgentJournalCursor | null }
}
export type JournalCardLineage = JournalInputCard & { messageId: string }
export type JournalInputEntry = { context: JournalInputContext; witnessCardId?: string }
export type JournalInputLineage = { clock: number; cards: JournalCardLineage[] }
const object = (value: unknown): value is Record<string, unknown> =>
typeof value === 'object' && value !== null && !Array.isArray(value)
const ordinal = (value: unknown): value is number =>
Number.isSafeInteger(value) && Number(value) >= 0
export function isInputCoordinate(value: unknown): value is AgentJournalCursor {
return (
object(value) &&
typeof value.epoch === 'string' &&
value.epoch.length > 0 &&
ordinal(value.sequence)
)
}
export function isInputContext(value: unknown): value is JournalInputContext {
return (
object(value) &&
isInputCoordinate(value.source) &&
ordinal(value.cut) &&
(value.basis === 'observed' || value.basis === 'legacy')
)
}
export function isInputLineage(value: unknown): value is JournalInputLineage {
if (!object(value) || !ordinal(value.clock) || !Array.isArray(value.cards)) {
return false
}
const clock = value.clock
const ids = new Set<string>()
return value.cards.every((card) => {
if (
!object(card) ||
typeof card.messageId !== 'string' ||
!card.messageId ||
ids.has(card.messageId) ||
!ordinal(card.introduced) ||
card.introduced > clock
) {
return false
}
ids.add(card.messageId)
return (
card.baseline === undefined ||
(object(card.baseline) &&
(card.baseline.coordinate === null || isInputCoordinate(card.baseline.coordinate)))
)
})
}
export function queryInputContext<T>(
context: JournalInputContext,
original: AgentJournalCursor | undefined,
before: (coordinate: AgentJournalCursor) => T,
all: () => T,
merge: (left: T, right: T) => T
): T {
if (context.basis === 'observed' || original?.epoch === context.source.epoch) {
return all()
}
const source = before(context.source)
return original ? merge(source, before(original)) : source
}
/** Initial source coordinates and introduction cuts never follow physical restatement rows. */
export class JournalInputLineageOwner {
constructor(readonly cards: Map<string, JournalInputCard>) {}
private base = 0
clock = 0
load(lineage: JournalInputLineage): void {
this.base = lineage.clock
this.clock = lineage.clock
for (const card of lineage.cards) {
this.cards.set(card.messageId, {
introduced: card.introduced,
...(card.baseline ? { baseline: card.baseline } : {})
})
}
}
restoreFrontier(sequence: number, entries: Iterable<JournalInputEntry>): void {
let frontier = 0
for (const entry of entries) {
frontier = Math.max(frontier, entry.context.cut)
}
for (const card of this.cards.values()) {
frontier = Math.max(frontier, card.introduced)
}
this.clock = frontier
this.base = frontier - sequence
}
at(sequence: number): number {
return this.base + sequence
}
observe(sequence: number): void {
this.clock = Math.max(this.clock, this.at(sequence))
}
change(messageId: string, coordinate: AgentJournalCursor | null, sequence: number): void {
if (!this.cards.has(messageId)) {
this.cards.set(messageId, {
introduced: this.at(sequence),
baseline: { coordinate }
})
}
}
releaseHistory(keep: (id: string) => boolean): void {
for (const [id, card] of this.cards) {
if (!keep(id)) {
this.cards.delete(id)
} else {
delete card.baseline
}
}
}
}
@@ -0,0 +1,84 @@
import type {
AgentJournalCursor,
AgentJournalSubmission
} from '../../../shared/agent-session-journal-types'
import type { JournalReducerState } from './journal-reducer'
import type { JournalInputEntry } from './journal-input-lineage'
import type { JournalRow } from './journal-row-schema'
export type JournalInputPlace = JournalInputEntry & {
coordinate: AgentJournalCursor
original?: AgentJournalCursor
}
export function readJournalInputPlace(
state: JournalReducerState,
id: string,
originals: Map<string, AgentJournalCursor | undefined>,
originalAcceptance?: (id: string) => AgentJournalCursor | undefined
): JournalInputPlace | undefined {
const own = state.inputEntries.get(id)
const entry = state.submissions.get(id)
if (!own || !entry) {
return undefined
}
if (own.context.basis === 'legacy' && !originals.has(id)) {
originals.set(id, originalAcceptance?.(id))
}
const original = originals.get(id)
return {
...own,
coordinate: {
epoch: state.epoch,
sequence: entry.acceptedSequence ?? entry.submittedSequence ?? 0
},
...(original ? { original } : {})
}
}
export function currentJournalInputIdentity(
state: JournalReducerState,
id: string,
hasOverlay: () => boolean
): boolean {
const card = state.queue.cards.get(id)
return card
? card.state === 'waiting' ||
card.state === 'returned' ||
(card.state === 'dispatched' &&
card.consumedAs !== null &&
state.submissions.get(card.consumedAs)?.dispatchState === 'pending')
: hasOverlay()
}
export function restoreJournalInputPlaces(state: JournalReducerState): number {
let legacyPlaces = 0
for (const [id, own] of state.inputEntries) {
if (state.submissions.get(id)?.dispatchState !== 'pending') {
state.inputEntries.delete(id)
} else if (own.context.basis === 'legacy') {
legacyPlaces++
}
}
return legacyPlaces
}
export function journalInputOrderChanged(
state: JournalReducerState,
row: JournalRow,
readCount: (key: 'receiptCount' | 'queueRevision') => number
): boolean {
return (
row.kind === 'submission' ||
row.kind === 'dispatch' ||
row.kind === 'tombstone' ||
readCount('receiptCount') !== state.receipts.size ||
readCount('queueRevision') !== state.queue.revision
)
}
export function isJournalInputOverride(
entry: AgentJournalSubmission,
recoveryPending: boolean
): boolean {
return !recoveryPending && entry.origin === 'client' && entry.queuedMessageId !== undefined
}
@@ -0,0 +1,566 @@
import { journalInputOrder } from './journal-input-order'
import { insertCommandReceiptIfAbsent, readCommandReceipt } from './command-receipt-table'
import { writeCommandReceiptTestRecord } from './command-receipt-test-support'
import type { JournalRow } from './journal-row-schema'
import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key'
import type { AgentSessionJournal } from './journal-store'
import type { JournalReducerState } from './journal-reducer'
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { expect, it, vi } from 'vitest'
import {
agentSessionJournalProviderHandle,
claudeProviderHandle
} from '../../../shared/agent-session-provider-handle-encoding'
import type { AgentJournalMessageItem } from '../../../shared/agent-session-journal-types'
import { holdUnsentSends } from './journal-unsent-send-hold'
import {
createTrackedJournalOpener,
loadTestJournal,
openTestJournalHostDatabase,
readTestJournalOutcomes,
insertTestJournalRow
} from './journal-host-database-test-support'
import {
selectedStructuredSubmission,
selectStructuredAgentSessionInput
} from '../agent-session-wire/structured-agent-session-input-selection'
const message = (text: string): AgentJournalMessageItem => ({
kind: 'message',
role: 'user',
blocks: [{ type: 'text', text }]
})
function knownPredecessors(owner: AgentSessionJournal | JournalReducerState, id: string): string[] {
const state = 'queuedMessages' in owner ? owner.queuedMessages.displayState() : owner
const submission = state.submissions.get(id)
return submission
? [...state.queue.cards.values()]
.filter((card) => journalInputOrder(state).precedes(card, submission))
.map((card) => card.messageId)
: []
}
it('preserves the forced queue place through command removal, two epoch replacements, replay and recovery', async () => {
const root = await mkdtemp(join(tmpdir(), 'orca-input-order-'))
const journals = createTrackedJournalOpener()
const identity = {
sessionId: 'order',
workspaceId: 'folder',
hostId: 'ssh',
agent: 'claude' as const,
providerHandle: claudeProviderHandle('thread', null)
}
const journal = await journals.open({ identity, stateDirectory: root })
try {
const c = await journal.queuedMessages.insert({
messageId: 'C',
body: { ...message('/compact'), command: { name: 'compact' } }
})
await journal.queuedMessages.insert({ messageId: 'D', body: message('earlier card') })
await journal.appendSubmission({
clientMessageId: 'M',
body: message('stale direct'),
payloadFingerprint: 'M',
fence: 0,
handoverRecorded: true,
origin: 'client'
})
await journal.queuedMessages.insert({ messageId: 'later', body: message('later card') })
await journal.queuedMessages.withdraw([c.messageId])
expect(selectStructuredAgentSessionInput(journal)).toMatchObject({
kind: 'card',
card: { messageId: 'D' }
})
for (let round = 0; round < 2; round++) {
await journal.replaceEpochItems('unreconcilable_prefix', 0, [])
expect(knownPredecessors(journal, 'M')).toEqual(['D'])
expect(journal.inputOrder.forced('M')).toBe(true)
expect(journal.queuedMessages.get('C')).toBeNull()
expect(selectStructuredAgentSessionInput(journal)).toMatchObject({
kind: 'card',
card: { messageId: 'D' }
})
}
await journal.close()
const reopened = await journals.open({ identity, stateDirectory: root })
expect(knownPredecessors(reopened, 'M')).toEqual(['D'])
expect(reopened.inputOrder.forced('M')).toBe(true)
await holdUnsentSends(reopened, { fence: 0, hold: { cause: 'hostRestarted' } })
expect(
reopened.queuedMessages
.list()
.filter((card) => card.state === 'waiting')
.map((card) => card.messageId)
).toEqual(['D', 'M', 'later'])
expect(reopened.submission('M')).toMatchObject({
dispatchState: 'rejected',
keptAsQueuedMessageId: 'M'
})
expect(selectedStructuredSubmission(reopened)).toBeUndefined()
} finally {
await journals.closeAll()
await rm(root, { recursive: true, force: true })
}
})
it.each(['resume', 'accepted'] as const)(
'preserves failed reopen admission facts across replay/replacement and %s release',
async (release) => {
const root = await mkdtemp(join(tmpdir(), 'orca-input-order-reopen-'))
const journals = createTrackedJournalOpener()
const identity = {
sessionId: 'order',
workspaceId: 'folder',
hostId: 'ssh',
agent: 'claude' as const,
providerHandle: claudeProviderHandle('thread', null)
}
let journal = await journals.open({ identity, stateDirectory: root })
const append = (
id: string,
receipt?: Parameters<typeof journal.appendSubmission>[2],
fence = 1
) =>
journal.appendSubmission(
{
clientMessageId: id,
body: message(id),
payloadFingerprint: id,
fence,
handoverRecorded: true,
origin: 'client',
options: { effort: 'high' }
},
undefined,
receipt
)
const accept = (id: string) =>
journal.resolveDispatch({
clientMessageId: id,
state: 'accepted',
providerIdentity: null,
fence: 1
})
try {
await journal.queuedMessages.insert({
messageId: 'C',
body: { ...message('/compact'), command: { name: 'compact' } }
})
await append('forced')
const mark = vi
.spyOn(journal, 'appendQueueReopen')
.mockRejectedValueOnce(new Error('disk full'))
await expect(journal.markQueueReopen(1)).rejects.toThrow('disk full')
mark.mockRestore()
const floor = journal.reopenFloor()!
const before = journal.cursor()
let committed = 0
await expect(
append('M', {
write: () => {
throw new Error('receipt transaction fails')
},
committed: () => {
committed++
}
})
).rejects.toThrow('receipt transaction fails')
expect(journal.cursor()).toEqual(before)
expect(journal.submission('M')).toBeUndefined()
expect(journal.reopenFloor()).toEqual(floor)
expect(
loadTestJournal(root, identity.sessionId)!.state.queuePauseMarks.reopenedSequence
).toBe(0)
expect(committed).toBe(0)
await expect(append('stale', undefined, 0)).rejects.toMatchObject({
code: 'journal_stale_fence'
})
expect(journal.cursor()).toEqual(before)
expect(journal.reopenFloor()).toEqual(floor)
if (release === 'resume') {
await journal.appendQueueResume(1)
}
let target: { epoch: string; sequence: number } | undefined
const cursor = await append('M', {
write: (_db, row) => {
if (row) {
target = { epoch: row.epoch, sequence: row.seq }
}
expect(row).toMatchObject({
kind: 'submission',
clientMessageId: 'M',
payloadFingerprint: 'M',
options: { effort: 'high' },
body: message('M')
})
},
committed: () => {
committed++
}
})
expect(target).toEqual(cursor)
expect(committed).toBe(1)
expect(journal.reopenFloor()).toBeNull()
const marks = readTestJournalOutcomes(
openTestJournalHostDatabase(root).db,
identity.sessionId
).filter((record) => record.field === '$' && record.key === '"queuePauseMarks"')
expect(
marks.some(
(record) =>
record.seq < cursor.sequence &&
JSON.parse(record.json).reopenedSequence === floor.sequence
)
).toBe(true)
expect(knownPredecessors(journal, 'M')).toEqual(release === 'resume' ? ['C'] : [])
if (release === 'accepted') {
expect(journal.queuedMessages.pauses().map((p) => p.reason)).toEqual(['restarted'])
await append('N')
expect(knownPredecessors(journal, 'N')).toEqual([])
await accept('N')
expect(journal.queuedMessages.pauses()).toEqual([])
await append('ordinary-held')
// The accepted N released C; this later input genuinely belongs behind it.
expect(knownPredecessors(journal, 'ordinary-held')).toEqual(['C'])
} else {
expect(journal.queuedMessages.pauses()).toEqual([])
}
const beforePrefixes = ['forced', 'M', 'ordinary-held'].map((id) =>
knownPredecessors(journal, id)
)
const replay = loadTestJournal(root, identity.sessionId)!
expect(replay.damage).toBeNull()
expect(replay.newer).toBeNull()
expect(
['forced', 'M', 'ordinary-held'].map((id) => knownPredecessors(replay.state, id))
).toEqual(beforePrefixes)
const lateMark = vi
.spyOn(journal, 'appendQueueReopen')
.mockRejectedValueOnce(new Error('later reopen write fails'))
await expect(journal.markQueueReopen(1)).rejects.toThrow('later reopen write fails')
lateMark.mockRestore()
const foreignFloor = journal.reopenFloor()!
await journal.replaceEpochItems('unreconcilable_prefix', 1, [])
expect(journal.reopenFloor()).toEqual(foreignFloor)
expect(foreignFloor.epoch).not.toBe(journal.epoch)
const beforeForeign = journal.cursor()
await append('new-epoch')
expect(journal.cursor().sequence).toBe(beforeForeign.sequence + 1)
expect(knownPredecessors(journal, 'new-epoch')).toEqual([])
expect(['forced', 'M', 'ordinary-held'].map((id) => knownPredecessors(journal, id))).toEqual(
beforePrefixes
)
await journal.close()
journal = await journals.open({ identity, stateDirectory: root })
expect(['forced', 'M', 'ordinary-held'].map((id) => knownPredecessors(journal, id))).toEqual(
beforePrefixes
)
await accept('forced')
expect(knownPredecessors(journal, 'forced')).toEqual([])
} finally {
await journals.closeAll()
await rm(root, { recursive: true, force: true })
}
}
)
it('keeps the receipt callback live after replacement of a legacy null-boundary card', async () => {
const root = await mkdtemp(join(tmpdir(), 'orca-input-replacement-'))
const journals = createTrackedJournalOpener({ legacyInputs: true })
const identity = {
sessionId: 'null-order',
workspaceId: 'folder',
hostId: 'ssh',
agent: 'claude' as const,
providerHandle: claudeProviderHandle('thread', null)
}
const body: AgentJournalMessageItem = {
kind: 'message',
role: 'user',
blocks: [{ type: 'text', text: '/compact' }],
command: { name: 'compact' }
}
let journal = await journals.open({ identity, stateDirectory: root })
try {
const cursor = journal.cursor()
await journal.close()
insertTestJournalRow(openTestJournalHostDatabase(root).db, identity.sessionId, {
v: 3,
epoch: cursor.epoch,
seq: cursor.sequence + 1,
fence: 0,
ts: 1,
kind: 'queue',
messageId: 'legacy-C',
change: { op: 'queued', body, position: 1, queuedAt: null }
})
journal = await journals.open({ identity, stateDirectory: root })
expect(journal.queuedMessages.get('legacy-C')?.queuedAt).toBeNull()
await journal.replaceEpochItems('unreconcilable_prefix', 0, [])
expect(journal.queuedMessages.get('legacy-C')?.queuedAt).toBeNull()
await expect(
journal.appendSubmission({
clientMessageId: 'M-after-replace',
body: { ...body, command: undefined },
payloadFingerprint: 'M-after-replace',
fence: 0,
handoverRecorded: true,
origin: 'client'
})
).resolves.toMatchObject({ epoch: journal.epoch })
expect(journal.submission('M-after-replace')).toMatchObject({ dispatchState: 'pending' })
const replay = loadTestJournal(root, identity.sessionId)!
expect(replay.damage).toBeNull()
expect(replay.state.queue.cards.get('legacy-C')?.queuedAt).toBeNull()
expect(knownPredecessors(replay.state, 'M-after-replace')).toEqual(
knownPredecessors(journal, 'M-after-replace')
)
} finally {
await journals.closeAll()
await rm(root, { recursive: true, force: true })
}
})
it.each(['legacy', 'observed'] as const)(
'retains %s import evidence through two production replacements and replay',
async (basis) => {
const root = await mkdtemp(join(tmpdir(), 'orca-source-domain-'))
const journals = createTrackedJournalOpener({ legacyInputs: true })
const identity = {
sessionId: 'domain',
workspaceId: 'folder',
hostId: 'ssh',
agent: 'claude' as const,
providerHandle: claudeProviderHandle('thread', null)
}
let journal = await journals.open({ identity, stateDirectory: root })
const database = openTestJournalHostDatabase(root)
try {
const a = journal.cursor()
await journal.close()
let seq = a.sequence
const row = (
fields: Pick<Extract<JournalRow, { kind: 'queue' }>, 'kind' | 'messageId' | 'change'>
): JournalRow => ({ v: 3, epoch: a.epoch, seq: ++seq, fence: 0, ts: 1, ...fields })
for (const [id, queuedAt] of [
['D', { epoch: 'foreign', sequence: 1 }],
['N', null],
['C', { epoch: a.epoch, sequence: seq }]
] as const) {
insertTestJournalRow(
database.db,
identity.sessionId,
row({
kind: 'queue',
messageId: id,
change: {
op: 'queued',
body:
id === 'C' ? { ...message('/compact'), command: { name: 'compact' } } : message(id),
position: seq,
queuedAt
}
})
)
}
database.transaction((db) => writeCommandReceiptTestRecord(db, identity.sessionId))
journal = await journals.open({ identity, stateDirectory: root })
const originals = new Map<string, string>()
for (const id of ['M1', 'M2', 'M3']) {
const receipt =
id === 'M2'
? undefined
: {
write: (
db: Parameters<typeof insertCommandReceiptIfAbsent>[0],
target?: JournalRow
) => {
expect(target).toMatchObject({ kind: 'submission', clientMessageId: id })
const result = {
kind: 'journal-row' as const,
epoch: target!.epoch,
sequence: target!.seq
}
insertCommandReceiptIfAbsent(
db,
{ kind: 'global' },
{
operationId: id,
sessionId: identity.sessionId,
callerKey: 'fixture',
method: 'agentSession.send',
fingerprint: id,
status: 'accepted',
acceptedAt: 1,
result
}
)
originals.set(id, JSON.stringify(result))
},
committed: () => {}
}
if (basis === 'observed') {
await journal.appendSubmission(
{
clientMessageId: id,
payloadFingerprint: id,
body: message(id),
handoverRecorded: true,
origin: 'client',
options: { effort: 'high' },
fence: 0
},
undefined,
receipt
)
} else {
await journal.close()
const target: JournalRow = {
kind: 'submission',
v: 3,
epoch: a.epoch,
seq: journal.cursor().sequence + 1,
ts: 1,
fence: 0,
clientMessageId: id,
payloadFingerprint: id,
body: message(id),
providerHandle: agentSessionJournalProviderHandle(identity),
handoverRecorded: true,
origin: 'client',
options: { effort: 'high' }
}
insertTestJournalRow(database.db, identity.sessionId, target)
if (receipt) {
database.transaction((db) => receipt.write(db, target))
}
journal = await journals.open({ identity, stateDirectory: root })
}
}
const expected = basis === 'legacy' ? [true, false, true] : [true, true, true]
const check = (state: JournalReducerState) => {
for (const card of ['D', 'N']) {
expect(
['M1', 'M2', 'M3'].map((id) =>
journalInputOrder(state).precedes(
state.queue.cards.get(card)!,
state.submissions.get(id)!
)
)
).toEqual(expected)
}
expect(['M1', 'M2', 'M3'].map((id) => journalInputOrder(state).forced(id))).toEqual([
true,
true,
true
])
expect(
['M1', 'M2', 'M3'].map((id) =>
journalInputOrder(state).precedes(
state.queue.cards.get('L')!,
state.submissions.get(id)!
)
)
).toEqual([false, false, false])
}
await journal.queuedMessages.insert({ messageId: 'L', body: message('later') })
for (let pass = 0; pass < 3; pass++) {
if (pass) {
await journal.replaceEpochItems('unreconcilable_prefix', 0, [])
}
check(journal.queuedMessages.displayState())
const replay = loadTestJournal(root, identity.sessionId)!
expect(replay.damage).toBeNull()
expect(replay.newer).toBeNull()
check(replay.state)
for (const id of ['M1', 'M3']) {
expect(JSON.stringify(readCommandReceipt(database.db, { kind: 'global' }, id))).toContain(
originals.get(id)!
)
}
expect(readCommandReceipt(database.db, { kind: 'global' }, 'M2').verdict).toBe('absent')
expect(journal.itemBody(agentJournalSubmissionKey('M2'))).toEqual(message('M2'))
expect(journal.submission('M2')?.options).toEqual({ effort: 'high' })
}
} finally {
await journals.closeAll()
await rm(root, { recursive: true, force: true })
}
}
)
it('retries partial keeps from committed facts and carries rank moves on the first successful keep', async () => {
const root = await mkdtemp(join(tmpdir(), 'orca-partial-order-'))
const journals = createTrackedJournalOpener()
const identity = {
sessionId: 'partial',
workspaceId: 'folder',
hostId: 'ssh',
agent: 'claude' as const,
providerHandle: claudeProviderHandle('thread', null)
}
let journal = await journals.open({ identity, stateDirectory: root })
try {
await journal.queuedMessages.insert({
messageId: 'C',
body: { ...message('/compact'), command: { name: 'compact' } }
})
await journal.queuedMessages.insert({ messageId: 'D', body: message('D') })
for (const id of ['M1', 'M2']) {
await journal.appendSubmission({
clientMessageId: id,
payloadFingerprint: id,
body: message(id),
handoverRecorded: true,
origin: 'client',
fence: 0
})
}
await journal.queuedMessages.insert({ messageId: 'L', body: message('L') })
await journal.close()
journal = await journals.open({ identity, stateDirectory: root })
const resolve = journal.resolveDispatch.bind(journal)
const fault = vi
.spyOn(journal, 'resolveDispatch')
.mockImplementation((...args) =>
args[0].clientMessageId === 'M1'
? Promise.reject(new Error('keep and fallback unavailable'))
: resolve(...args)
)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
await expect(
holdUnsentSends(journal, { fence: 0, hold: { cause: 'hostRestarted' } })
).rejects.toThrow('settling unsent sends failed')
expect(journal.submission('M1')?.dispatchState).toBe('pending')
expect(journal.submission('M2')?.keptAsQueuedMessageId).toBe('M2')
expect(journal.queuedMessages.get('L')?.position).toBeGreaterThan(
journal.queuedMessages.get('M2')!.position
)
fault.mockRestore()
warn.mockRestore()
await holdUnsentSends(journal, { fence: 0, hold: { cause: 'hostRestarted' } })
const saved = journal.queuedMessages.list().filter((card) => card.state === 'waiting')
expect(saved.map((card) => card.messageId)).toEqual(['C', 'D', 'M1', 'M2', 'L'])
expect(
saved.every(
(card, i) =>
Number.isFinite(card.position) && (i === 0 || card.position > saved[i - 1].position)
)
).toBe(true)
await journal.close()
journal = await journals.open({ identity, stateDirectory: root })
await holdUnsentSends(journal, { fence: 0, hold: { cause: 'hostRestarted' } })
expect(
journal.queuedMessages
.list()
.filter((card) => card.state === 'waiting')
.map((card) => card.messageId)
).toEqual(saved.map((card) => card.messageId))
} finally {
await journals.closeAll()
await rm(root, { recursive: true, force: true })
}
})
@@ -0,0 +1,323 @@
import {
readJournalInputPlace,
restoreJournalInputPlaces,
currentJournalInputIdentity,
journalInputOrderChanged,
isJournalInputOverride
} from './journal-input-order-state'
export type { JournalInputPlace } from './journal-input-order-state'
import { JournalSubmissionLinks } from './journal-submission-links'
import type {
AgentJournalCursor,
AgentJournalSubmission
} from '../../../shared/agent-session-journal-types'
import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key'
import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission'
import type { JournalReducerState } from './journal-reducer'
import type { JournalRow } from './journal-row-schema'
import type { QueuedMessageRow } from './queued-message-table'
import type { InputPlaceQuery } from './journal-input-place-index'
import { JournalInputLineageOwner } from './journal-input-lineage'
import { JournalInputCardIndex } from './journal-input-card-index'
const owners = new WeakMap<JournalReducerState, JournalInputOrder>()
export function initializeJournalInputOrder(
state: JournalReducerState,
originalAcceptance?: (id: string) => AgentJournalCursor | undefined
): JournalInputOrder {
const owner = new JournalInputOrder(state, originalAcceptance)
owners.set(state, owner)
return owner
}
export function journalInputOrder(state: JournalReducerState): JournalInputOrder {
return owners.get(state) ?? initializeJournalInputOrder(state)
}
export function setJournalInputOrder(state: JournalReducerState, owner: JournalInputOrder): void {
owners.set(state, owner)
}
/** Own admission places and shared command witnesses are reconstructed by the journal fold. */
export class JournalInputOrder {
private readonly links = new JournalSubmissionLinks()
private reconstructing = false
private legacyPlaces = 0
private readonly recoveryOwed = new Set<string>()
private readonly originals = new Map<string, AgentJournalCursor | undefined>()
private readonly unsettled = new Map<string, AgentJournalSubmission>()
private readonly queued = new Map<string, AgentJournalSubmission>()
private readonly entered = new Map<string, AgentJournalSubmission>()
private readonly overrides = new Map<string, AgentJournalSubmission>()
private pending: readonly AgentJournalSubmission[] | undefined
readonly lineage: JournalInputLineageOwner
private readonly cards: JournalInputCardIndex
private receiptCount = 0
private queueRevision = 0
private revision = 0
constructor(
private readonly state: JournalReducerState,
private readonly originalAcceptance?: (id: string) => AgentJournalCursor | undefined
) {
this.lineage = new JournalInputLineageOwner(state.inputCards)
this.cards = new JournalInputCardIndex(this.lineage)
}
beforeRow(state: JournalReducerState, row: JournalRow): void {
if (row.kind === 'dispatch' && row.keptAsQueuedMessageId) {
const place = this.place(row.clientMessageId)
if (place && !this.lineage.cards.has(row.keptAsQueuedMessageId)) {
this.lineage.cards.set(row.keptAsQueuedMessageId, {
introduced: place.context.cut,
baseline: { coordinate: place.coordinate }
})
}
}
this.receiptCount = state.receipts.size
this.queueRevision = state.queue.revision
if (row.kind === 'epoch' && row.inputOrderLineage) {
this.lineage.load(row.inputOrderLineage)
}
this.lineage.observe(row.seq)
if (row.kind !== 'submission' || !row.handoverRecorded || row.queuedMessageId !== undefined) {
return
}
if (this.state.inputEntries.has(row.clientMessageId)) {
return
}
const place = this.cards.admission(state, row, this.originalAcceptance)
if (place.context.basis === 'legacy') {
this.legacyPlaces++
}
this.state.inputEntries.set(row.clientMessageId, {
context: place.context,
...(place.witnessCardId ? { witnessCardId: place.witnessCardId } : {})
})
}
// A streaming prefix cannot establish the last legacy consumer.
beginReconstruction(): void {
this.reconstructing = true
this.legacyPlaces = 0
for (const entry of this.state.inputEntries.values()) {
if (entry.context.basis === 'legacy') {
this.legacyPlaces++
}
}
}
finishReconstruction(state: JournalReducerState): void {
this.reconstructing = false
if (this.legacyPlaces === 0) {
this.releaseHistory(state)
}
}
restore(state: JournalReducerState): void {
this.legacyPlaces = restoreJournalInputPlaces(state)
this.cards.ensure(state)
for (const id of state.submissions.keys()) {
this.refresh(state, id)
}
this.lineage.restoreFrontier(state.lastSequence, state.inputEntries.values())
}
linkedVersion(): number {
return this.links.version()
}
linkedTo(id: string): Iterable<AgentJournalSubmission> {
return this.links.to(id)
}
overlay(state: JournalReducerState, rows: readonly QueuedMessageRow[]): void {
this.cards.overlay(state, rows)
}
introduceOverlays(state: JournalReducerState): void {
this.cards.introduceOverlays((card) => this.queueChanged(state, undefined, card))
}
markOpeningRecovery(): void {
for (const entry of this.unsettled.values()) {
if (isQueuedAgentJournalSubmission(entry)) {
this.markRecoveryPending(entry.clientMessageId)
}
}
}
markRecoveryPending(id: string): void {
this.recoveryOwed.add(id)
const queued = this.queued.delete(id)
const override = this.overrides.delete(id)
if (queued || override) {
this.revision++
}
}
recoveryPending(id: string): boolean {
return this.recoveryOwed.has(id)
}
carryRecovery(previous: JournalInputOrder): void {
for (const id of previous.recoveryOwed) {
if (this.queued.has(id)) {
this.markRecoveryPending(id)
}
}
}
place(id: string): ReturnType<typeof readJournalInputPlace> {
return readJournalInputPlace(this.state, id, this.originals, this.originalAcceptance)
}
forced(id: string): boolean {
return this.state.inputEntries.get(id)?.witnessCardId !== undefined
}
firstQueued(): AgentJournalSubmission | undefined {
return this.queued.values().next().value
}
explicitSend(): AgentJournalSubmission | undefined {
return this.overrides.values().next().value
}
enteredSubmissions(): Iterable<AgentJournalSubmission> {
return this.entered.values()
}
unsettledSubmissions(): readonly AgentJournalSubmission[] {
return (this.pending ??= [...this.unsettled.values()])
}
version(): number {
return this.revision
}
head(state: JournalReducerState, floor: AgentJournalCursor | null): QueuedMessageRow | undefined {
this.cards.ensure(state, floor)
return this.cards.head()
}
paused(
state: JournalReducerState,
card: QueuedMessageRow,
floor: AgentJournalCursor | null
): boolean {
this.cards.ensure(state, floor)
return this.cards.paused(card)
}
predecessors(state: JournalReducerState, id: string): InputPlaceQuery {
this.cards.ensure(state)
const place = this.place(id)
return place?.witnessCardId ? this.cards.query(place) : { position: -Infinity }
}
queueChanged(
state: JournalReducerState,
prior: QueuedMessageRow | undefined,
card: QueuedMessageRow
): void {
this.cards.queueChanged(state, prior, card)
if (!this.reconstructing && this.legacyPlaces === 0) {
const source = this.lineage.cards.get(card.messageId)
if (source) {
delete source.baseline
}
}
if (card.state === 'withdrawn' && !this.reconstructing && this.legacyPlaces === 0) {
this.lineage.cards.delete(card.messageId)
}
}
afterRow(state: JournalReducerState, row: JournalRow): void {
if (journalInputOrderChanged(state, row, (key) => this[key])) {
this.revision++
}
if (row.kind === 'submission' || row.kind === 'dispatch') {
this.refresh(state, row.clientMessageId)
} else if (row.kind === 'tombstone' && row.retireSubmission) {
this.refreshItem(state, row.itemId)
} else if (row.kind === 'item') {
this.refreshItem(state, state.aliases.get(row.itemId) ?? row.itemId)
} else if (row.kind === 'lifecycle-batch') {
for (const mutation of row.mutations) {
this.refreshItem(state, state.aliases.get(mutation.itemId) ?? mutation.itemId)
}
} else if (row.kind === 'queue') {
const id = state.queue.cards.get(row.messageId)?.consumedAs
if (id) {
this.refresh(state, id)
}
}
}
private refreshItem(state: JournalReducerState, itemId: string): void {
const identity = parseAgentJournalItemKey(itemId)
if (identity?.provider === 'orca') {
this.refresh(state, identity.clientMessageId)
}
}
private refresh(state: JournalReducerState, id: string): void {
const entry = state.submissions.get(id)
this.links.refresh(id, entry)
if (entry?.dispatchState === 'pending' || entry?.dispatchState === 'unknown') {
if (this.unsettled.get(id) !== entry) {
this.unsettled.set(id, entry)
this.pending = undefined
this.revision++
}
} else if (this.unsettled.delete(id)) {
this.pending = undefined
this.revision++
}
if (entry?.dispatchState !== 'pending') {
this.releasePlace(state, id)
}
if (entry && isQueuedAgentJournalSubmission(entry)) {
if (!this.recoveryOwed.has(id)) {
this.queued.set(id, entry)
}
this.entered.delete(id)
if (isJournalInputOverride(entry, this.recoveryOwed.has(id))) {
this.overrides.set(id, entry)
}
} else {
this.recoveryOwed.delete(id)
this.queued.delete(id)
this.overrides.delete(id)
if (entry && (entry.dispatchState === 'pending' || entry.dispatchState === 'unknown')) {
this.entered.set(id, entry)
} else {
this.entered.delete(id)
}
}
if (entry?.queuedMessageId) {
const card = state.queue.cards.get(entry.queuedMessageId)
if (card?.consumedAs === id) {
this.cards.indexCard(state, card)
if (
!this.reconstructing &&
this.legacyPlaces === 0 &&
!this.currentIdentity(state, card.messageId)
) {
this.lineage.cards.delete(card.messageId)
}
}
}
}
private currentIdentity(state: JournalReducerState, id: string): boolean {
return currentJournalInputIdentity(state, id, () => this.cards.hasOverlay(id))
}
private releasePlace(state: JournalReducerState, id: string): void {
const place = this.state.inputEntries.get(id)
if (place && this.state.inputEntries.delete(id)) {
this.originals.delete(id)
if (place.context.basis === 'legacy' && --this.legacyPlaces === 0 && !this.reconstructing) {
this.releaseHistory(state)
}
}
}
private releaseHistory(state: JournalReducerState): void {
this.lineage.releaseHistory((id) => this.currentIdentity(state, id))
this.cards.releaseHistory()
}
precedes(card: QueuedMessageRow, submission: AgentJournalSubmission): boolean {
const place = this.place(submission.clientMessageId)
return Boolean(place?.witnessCardId && this.cards.relation(card, place))
}
}
@@ -0,0 +1,169 @@
import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types'
import type { QueuedMessageRow } from './queued-message-table'
export type InputPlaceQuery = {
head?: QueuedMessageRow
command?: QueuedMessageRow
waiting?: QueuedMessageRow
handoff?: QueuedMessageRow
position: number
}
type Entry = InputPlaceQuery & { card: QueuedMessageRow; sequence: number; domain: string }
type Node = {
entry: Entry
height: number
result: InputPlaceQuery
left?: Node
right?: Node
}
const empty = (): InputPlaceQuery => ({ position: -Infinity })
const first = (a?: QueuedMessageRow, b?: QueuedMessageRow) =>
!a || (b && b.position < a.position) ? b : a
export function mergeInputPlaces(a: InputPlaceQuery, b: InputPlaceQuery): InputPlaceQuery {
return {
head: first(a.head, b.head),
command: first(a.command, b.command),
waiting: first(a.waiting, b.waiting),
handoff: first(a.handoff, b.handoff),
position: Math.max(a.position, b.position)
}
}
const height = (node?: Node) => node?.height ?? 0
const compare = (a: Entry, b: Entry) =>
a.sequence - b.sequence || a.card.messageId.localeCompare(b.card.messageId)
function update(node: Node): Node {
node.height = 1 + Math.max(height(node.left), height(node.right))
node.result = mergeInputPlaces(
mergeInputPlaces(node.left?.result ?? empty(), node.entry),
node.right?.result ?? empty()
)
return node
}
function left(node: Node): Node {
const next = node.right!
node.right = next.left
next.left = update(node)
return update(next)
}
function right(node: Node): Node {
const next = node.left!
node.left = next.right
next.right = update(node)
return update(next)
}
function balance(node: Node): Node {
update(node)
if (height(node.left) - height(node.right) > 1) {
if (height(node.left!.right) > height(node.left!.left)) {
node.left = left(node.left!)
}
return right(node)
}
if (height(node.right) - height(node.left) > 1) {
if (height(node.right!.left) > height(node.right!.right)) {
node.right = right(node.right!)
}
return left(node)
}
return node
}
function put(node: Node | undefined, entry: Entry, remove: boolean): Node | undefined {
if (!node) {
return remove ? undefined : { entry, height: 1, result: entry }
}
const order = compare(entry, node.entry)
if (order < 0) {
node.left = put(node.left, entry, remove)
} else if (order > 0) {
node.right = put(node.right, entry, remove)
} else if (!remove) {
node.entry = entry
} else {
if (!node.left) {
return node.right
}
if (!node.right) {
return node.left
}
let next = node.right
while (next.left) {
next = next.left
}
node.entry = next.entry
node.right = put(node.right, next.entry, true)
}
return balance(node)
}
/** Current card coordinates only; subtree summaries answer admission/selection/recovery cuts. */
export class JournalInputPlaceIndex {
private readonly epochs = new Map<string, Node>()
private all: Node | undefined
private readonly entries = new Map<string, Entry>()
constructor(_epoch: string) {}
set(
card: QueuedMessageRow,
actionable: boolean,
handoff: boolean,
coordinate: AgentJournalCursor | null = card.queuedAt
): void {
const prior = this.entries.get(card.messageId)
if (prior) {
this.change(prior, true)
}
this.entries.delete(card.messageId)
if (card.state === 'withdrawn') {
return
}
const entry: Entry = {
card,
sequence: coordinate?.sequence ?? 0,
domain: coordinate?.epoch ?? '',
position: card.position,
...((card.state === 'waiting' && card.holdReason === null) || card.state === 'returned'
? { head: card }
: {}),
...(actionable ? { waiting: card, ...(card.body.command ? { command: card } : {}) } : {}),
...(handoff ? { handoff: card } : {})
}
this.entries.set(card.messageId, entry)
this.change(entry, false)
}
private change(entry: Entry, remove: boolean): void {
const epoch = entry.domain
const root = put(this.epochs.get(epoch), entry, remove)
if (root) {
this.epochs.set(epoch, root)
} else {
this.epochs.delete(epoch)
}
this.all = put(this.all, entry, remove)
}
isEmpty(): boolean {
return this.entries.size === 0
}
query(epoch: string, sequence = Infinity): InputPlaceQuery {
const before = (node?: Node): InputPlaceQuery => {
if (!node) {
return empty()
}
if (node.entry.sequence >= sequence) {
return before(node.left)
}
return mergeInputPlaces(
mergeInputPlaces(node.left?.result ?? empty(), node.entry),
before(node.right)
)
}
return before(this.epochs.get(epoch))
}
allPlaces(): InputPlaceQuery {
return this.all?.result ?? empty()
}
}
@@ -0,0 +1,113 @@
import type Database from '../../sqlite/sync-database'
import type {
AgentJournalCursor,
AgentJournalSubmission
} from '../../../shared/agent-session-journal-types'
import { readCommandReceipt } from './command-receipt-table'
/** Only the original send's accepted coordinate can compare a retired enqueue boundary. */
export function journalInputAcceptance(
db: Database.Database,
sessionId: string,
id: string
): AgentJournalCursor | undefined {
const read = readCommandReceipt(db, { kind: 'global' }, id)
if (
read.verdict === 'readable' &&
read.receipt.status === 'accepted' &&
read.receipt.sessionId === sessionId &&
read.receipt.method === 'agentSession.send' &&
read.receipt.result.kind === 'journal-row'
) {
return { epoch: read.receipt.result.epoch, sequence: read.receipt.result.sequence }
}
return undefined
}
/** Reuse comparable admission coordinates; original receipts remain the provenance authority. */
export function journalQueueRebase(
db: Database.Database,
sessionId: string,
epoch: string,
previousEpoch: string | undefined,
queued: readonly AgentJournalSubmission[],
rebuilt: ReadonlyMap<string, number>
): (queuedAt: AgentJournalCursor | null) => AgentJournalCursor | null {
type Position = { sequence: number; rebased: number }
const comparableByEpoch = new Map<string, Position[]>()
let originalPositions: Map<string, Position[]> | undefined
const originals = () => {
if (!originalPositions) {
originalPositions = new Map()
for (const submission of queued) {
const original = journalInputAcceptance(db, sessionId, submission.clientMessageId)
if (original) {
const positions = originalPositions.get(original.epoch) ?? []
positions.push({
sequence: original.sequence,
rebased: rebuilt.get(submission.clientMessageId) ?? 1
})
originalPositions.set(original.epoch, positions)
}
}
}
return originalPositions
}
return (queuedAt) => {
if (queuedAt && previousEpoch !== undefined) {
const boundary = queuedAt
let comparable = comparableByEpoch.get(boundary.epoch)
if (!comparable) {
if (boundary.epoch === previousEpoch) {
const current = queued.flatMap((submission) =>
submission.acceptedSequence === undefined
? []
: [
{
sequence: submission.acceptedSequence,
rebased: rebuilt.get(submission.clientMessageId) ?? 1
}
]
)
if (current.length === queued.length) {
comparable = current
} else {
// Legacy missing admission places retain only comparable original coordinates.
const missing = new Set(
queued
.filter((submission) => submission.acceptedSequence === undefined)
.map((submission) => rebuilt.get(submission.clientMessageId))
)
comparable = [
...current,
...(originals().get(boundary.epoch) ?? []).filter((position) =>
missing.has(position.rebased)
)
]
}
} else {
comparable = originals().get(boundary.epoch) ?? []
}
comparable.sort((a, b) => a.sequence - b.sequence)
for (let i = 1; i < comparable.length; i++) {
comparable[i].rebased = Math.max(comparable[i].rebased, comparable[i - 1].rebased)
}
comparableByEpoch.set(boundary.epoch, comparable)
}
if (comparable.length === queued.length) {
let low = 0
let high = comparable.length
while (low < high) {
const mid = (low + high) >>> 1
if (comparable[mid].sequence <= boundary.sequence) {
low = mid + 1
} else {
high = mid
}
}
queuedAt = { epoch, sequence: low === 0 ? 1 : comparable[low - 1].rebased }
}
}
return queuedAt
}
}
@@ -9,11 +9,15 @@ import {
type AgentSessionJournalIdentity
} from '../../../shared/agent-session-journal-types'
import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding'
import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key'
import {
createTrackedJournalOpener,
insertTestJournalRow,
liveTestJournalRows,
openTestJournalHostDatabase
openTestJournalHostDatabase,
seedLegacyTestJournal
} from './journal-host-database-test-support'
import type { JournalRow } from './journal-row-schema'
import { journalOutcomeSource } from './journal-outcome-format'
import { sweepJournalOutcomePage } from './journal-outcome-cleanup'
import type { AgentSessionJournal } from './journal-store'
@@ -67,15 +71,69 @@ const recordsOf = (generation: number) =>
/** Old rows (items 0 and 1, a waiting card, a user Stop), then this build's records (item 0
* revised, item 2, a second card). */
async function mixedChat(): Promise<AgentSessionJournal> {
const older = createTrackedJournalOpener({ legacyInputs: true })
const earlier = await older.open({ identity: IDENTITY, stateDirectory: root })
await earlier.appendItem(item(0), status('first'), scope)
await earlier.appendItem(item(1), status('second'), scope)
await earlier.queuedMessages.insert({ messageId: 'card-old', body: message('old card') })
await earlier.appendStopEvent({ reason: 'user-stop' }, FENCE)
await older.closeAll()
// Freeze the old writer's rows before opening a current native coordinate owner.
const database = openTestJournalHostDatabase(root)
const epoch = 'old-input-epoch'
seedLegacyTestJournal(database, { identity: IDENTITY, mintEpoch: () => epoch, now: () => 1000 })
const base = { v: 2, epoch, fence: FENCE, ts: 1000 }
const prefix = [
...[status('first'), status('second')].map((body, ordinal): JournalRow => ({
...base,
kind: 'item',
seq: ordinal + 2,
itemId: agentJournalItemKey(item(ordinal)),
revision: 1,
body,
turnScope: AGENT_JOURNAL_THREAD_SCOPE
})),
{
...base,
v: 4,
kind: 'queue',
seq: 4,
messageId: 'card-old',
change: {
op: 'queued',
body: message('old card'),
position: 0,
queuedAt: { epoch, sequence: 4 }
}
},
{
...base,
kind: 'tombstone',
seq: 5,
itemId: agentJournalItemKey({ provider: 'orca', clientMessageId: 'stop-event' }),
revision: 1,
stopEvent: { reason: 'user-stop', at: 1000 }
}
] satisfies JournalRow[]
database.transaction((db) => {
for (const row of prefix) {
insertTestJournalRow(db, IDENTITY.sessionId, row)
}
})
expect(
rowsOfEpoch(epoch)
.slice(1)
.map((row) => JSON.parse(String(row.row_json)))
).toEqual(prefix)
expect(journalOutcomeSource(db(), IDENTITY.sessionId, epoch)).toBeNull()
const journal = await journals.open({ identity: IDENTITY, stateDirectory: root })
expect(seen(journal)).toEqual({
items: [status('first'), status('second')],
cards: [['card-old', message('old card'), 'waiting']],
stop: 'user-stop'
})
await journal.appendItem(item(0), status('first, revised'), scope)
const converted = journalOutcomeSource(db(), IDENTITY.sessionId, epoch)
expect(converted).toMatchObject({ prefix: 5, format: 2 })
expect(
db()
.prepare(`SELECT entry_key FROM journal_outcomes
WHERE generation = ? AND field = 'inputCards' AND deleted = 0`)
.all(converted!.generation)
).toEqual([{ entry_key: JSON.stringify('card-old') }])
await journal.appendItem(item(2), status('third'), scope)
await journal.queuedMessages.insert({ messageId: 'card-new', body: message('new card') })
const source = journalOutcomeSource(db(), IDENTITY.sessionId, journal.epoch)
@@ -1,3 +1,5 @@
import { readJournalLegacyCards } from './journal-queue-import'
import { journalInputOrder } from './journal-input-order'
// Replay live epochs without rewriting history; damage/newer rows fail load and remain on disk.
import { journalOutcomeSource } from './journal-outcome-format'
@@ -6,6 +8,7 @@ import { JournalOutcomeDecodeError } from './journal-outcome-codec'
import { readJournalOutcomeGroups, type JournalReadRow } from './journal-read-row'
import { AgentSessionJournalError } from './journal-write-guards'
import type Database from '../../sqlite/sync-database'
import { journalInputAcceptance } from './journal-input-provenance'
import {
applyJournalRow,
createJournalReducerState,
@@ -38,7 +41,10 @@ export function replayJournal(db: Database.Database, sessionId: string): Journal
if (epoch === null) {
return null
}
return foldJournalFromState(db, createJournalReducerState(sessionId, epoch))
return foldJournalFromState(
db,
createJournalReducerState(sessionId, epoch, (id) => journalInputAcceptance(db, sessionId, id))
)
}
/** The read is synchronous, like main's replay: it blocks once for one chat's saved records (its
@@ -49,7 +55,9 @@ export function restoreJournal(db: Database.Database, sessionId: string): Journa
if (epoch === null) {
return null
}
const empty = createJournalReducerState(sessionId, epoch)
const empty = createJournalReducerState(sessionId, epoch, (id) =>
journalInputAcceptance(db, sessionId, id)
)
try {
const source = journalOutcomeSource(db, sessionId, epoch)
if (!source) {
@@ -57,7 +65,7 @@ export function restoreJournal(db: Database.Database, sessionId: string): Journa
}
let baseline: JournalReducerState | undefined
if (source.prefix > 0) {
const prefix = foldJournalFromState(db, empty, source.prefix)
const prefix = foldJournalFromState(db, empty, source.prefix, true)
if (prefix.newer || prefix.damage || prefix.state.lastSequence !== source.prefix) {
const gap = { sequence: prefix.state.lastSequence + 1, cause: 'sequence-gap' as const }
return { ...prefix, damage: prefix.newer ? null : (prefix.damage ?? gap) }
@@ -82,10 +90,11 @@ export function restoreJournal(db: Database.Database, sessionId: string): Journa
function foldJournalFromState(
db: Database.Database,
state: JournalReducerState,
through = Number.POSITIVE_INFINITY
through = Number.POSITIVE_INFINITY,
deferNativeCleanup = false
): JournalLoad {
const { sessionId, epoch, lastSequence } = state
const fold = startJournalRowFold({ sessionId, epoch }, state)
const fold = startJournalRowFold({ sessionId, epoch, deferNativeCleanup }, state)
for (const entry of iterateJournalEpochRows(
db,
sessionId,
@@ -96,17 +105,27 @@ function foldJournalFromState(
break
}
}
if (!deferNativeCleanup) {
journalInputOrder(state).overlay(state, readJournalLegacyCards(db, state))
}
return fold.finish()
}
/** untilDamage stops at the first damage, for callers that only ask whether a fold publishes. */
export function startJournalRowFold(
input: { sessionId: string; epoch: string; untilDamage?: boolean },
state = createJournalReducerState(input.sessionId, input.epoch)
input: {
sessionId: string
epoch: string
untilDamage?: boolean
deferNativeCleanup?: boolean
originalAcceptance?: Parameters<typeof createJournalReducerState>[2]
},
state = createJournalReducerState(input.sessionId, input.epoch, input.originalAcceptance)
): {
add: (entry: { seq: number; rowJson: string; ts?: number }) => boolean
finish: () => JournalLoad
} {
journalInputOrder(state).beginReconstruction()
let expectedSequence = state.lastSequence + 1
let damage: JournalDamage | null = null
let anchored = state.lastSequence > 0
@@ -149,6 +168,13 @@ export function startJournalRowFold(
return true
}
const finish = (): JournalLoad => {
if (!input.deferNativeCleanup) {
journalInputOrder(state).finishReconstruction(state)
journalInputOrder(state).lineage.restoreFrontier(
state.lastSequence,
state.inputEntries.values()
)
}
state.oldestSequence = FIRST_JOURNAL_SEQUENCE
// Empty epochs contain no recoverable history; founding a fresh epoch loses nothing.
const found =
@@ -1,3 +1,9 @@
import {
admitOutcomeFormat2,
JOURNAL_OUTCOME_FORMAT_2,
isNativeOutcomeField,
decodeNativeOutcome
} from './journal-outcome-format-2'
import { z } from 'zod'
import type { SqliteRow } from '../../sqlite/sqlite-statement'
import type { JournalReducerState } from './journal-reducer'
@@ -46,7 +52,7 @@ export function encodeJournalOutcomes(
seq: state.lastSequence,
field: entry.path,
key: JSON.stringify(entry.key),
format: JOURNAL_OUTCOME_FORMAT_1,
format: JOURNAL_OUTCOME_FORMAT_2,
json: entry.deleted
? 'null'
: entry.path === 'items'
@@ -61,7 +67,7 @@ export function encodeJournalOutcomes(
seq: state.lastSequence,
field: '$',
key: JSON.stringify(key),
format: JOURNAL_OUTCOME_FORMAT_1,
format: JOURNAL_OUTCOME_FORMAT_2,
json: JSON.stringify(value),
deleted: false
})
@@ -69,8 +75,8 @@ export function encodeJournalOutcomes(
}
for (const value of values) {
// A reducer field format 1 has no home for needs a new outcome format, never a silent change.
if (admitOutcomeFormat1(value) === 'unknown') {
throw new Error(`journal outcome format 1 does not store ${value.field} ${value.key}`)
if (admitOutcomeFormat2(value) === 'unknown') {
throw new Error(`journal outcome format 2 does not store ${value.field} ${value.key}`)
}
}
return { values, scalars: next }
@@ -114,7 +120,11 @@ function admitJournalOutcome(row: SqliteRow): { value: JournalOutcome; admitted:
deleted: row.deleted === 1
}
const admitted =
value.format === JOURNAL_OUTCOME_FORMAT_1 ? admitOutcomeFormat1(value) : 'unknown'
value.format === JOURNAL_OUTCOME_FORMAT_1
? admitOutcomeFormat1(value)
: value.format === JOURNAL_OUTCOME_FORMAT_2
? admitOutcomeFormat2(value)
: 'unknown'
if (admitted === 'unknown') {
throw new AgentSessionJournalError('journal_read_only', 'the chat holds a newer outcome')
}
@@ -152,6 +162,16 @@ export class JournalOutcomeFold {
return value
}
introduction(field: string, id: string): number | undefined {
const ordinal = this.values.get(field)?.get(JSON.stringify(id))?.ordinal
return ordinal?.startsWith('s') ? Number(ordinal.slice(1, 17)) : undefined
}
replaced(field: string, id: string): boolean {
const value = this.values.get(field)?.get(JSON.stringify(id))
return Boolean(value && (value.deleted || value.ordinal !== JOURNAL_PREFIX_ORDINAL))
}
/** Applies the latest values over `baseline` (the folded input prefix) or over the scalars alone:
* a deletion removes, a prefix-ordinal value replaces in place, and every other value takes its
* canonical place after the prefix. Each record was admitted as format 1 on read. A prefix-ordinal
@@ -173,7 +193,7 @@ export class JournalOutcomeFold {
]
const decode = decodingOutcome(() => decodeOutcomeFormat1Onto(scalars, baseline))
for (const [field, entries] of this.values) {
if (field === '$') {
if (field === '$' || isNativeOutcomeField(field)) {
continue
}
// Canonical ordinals compare bytewise, as SQLite's BINARY collation orders them; the prefix
@@ -191,6 +211,19 @@ export class JournalOutcomeFold {
}
}
const state = decode.finish()
for (const [field, entries] of this.values) {
if (!isNativeOutcomeField(field)) {
continue
}
const ordered = [...entries.values()].sort((a, b) =>
a.ordinal < b.ordinal ? -1 : a.ordinal > b.ordinal ? 1 : 0
)
for (const value of ordered) {
decodingOutcome(() =>
decodeNativeOutcome(state, field, value.entryKey, value.kept, value.deleted)
)
}
}
if (state.sessionId !== sessionId || state.epoch !== epoch) {
throw new JournalOutcomeDecodeError('invalid journal outcome coverage')
}
@@ -72,7 +72,9 @@ afterEach(async () => {
})
function plain(state: JournalReducerState): unknown {
const fields: Record<string, unknown> = { ...state }
const fields: Record<string, unknown> = Object.fromEntries(
Object.entries(state).filter(([key]) => key !== 'inputEntries' && key !== 'inputCards')
)
return JSON.parse(
JSON.stringify({
...Object.fromEntries(
@@ -163,7 +165,7 @@ it('refuses a field format 1 does not define as written by a newer Orca', () =>
})
it('refuses a record of a newer format even when the epoch summary was not raised', () => {
insertOutcome('receipts', '"client-9"', '["client-9",{}]', 2)
insertOutcome('receipts', '"client-9"', '["client-9",{}]', 3)
expect(restoreJournal(database.db, sessionId)).toMatchObject(newer)
})
@@ -199,3 +201,32 @@ it('admits and decodes every declared shape as stored, and refuses one key more
'unknown'
)
})
it('distinguishes new canonical damage from newer keys, without changing frozen format-1 admission', () => {
const own = {
context: { source: { epoch, sequence: 1 }, basis: 'observed', cut: 1 },
witnessCardId: 'C'
}
const write = (value: unknown) => {
database.db.prepare("DELETE FROM journal_outcomes WHERE field='inputEntries'").run()
insertOutcome('inputEntries', '"client-1"', JSON.stringify(['client-1', value]), 2)
}
write(own)
expect(restoreJournal(database.db, sessionId)).toMatchObject({ newer: null, damage: null })
for (const context of [
{ ...own.context, cut: -1 },
{ ...own.context, cut: Number.MAX_SAFE_INTEGER + 1 },
{ ...own.context, source: { epoch: '', sequence: 1 } }
]) {
write({ ...own, context })
expect(restoreJournal(database.db, sessionId)).toMatchObject({
newer: null,
damage: { cause: 'unparseable-row' }
})
}
write({ ...own, futureNativeKey: true })
expect(restoreJournal(database.db, sessionId)).toMatchObject(newer)
// A new native family is never accepted as an additive format-1 record.
database.db.prepare("UPDATE journal_outcomes SET format=1 WHERE field='inputEntries'").run()
expect(restoreJournal(database.db, sessionId)).toMatchObject(newer)
})
@@ -0,0 +1,105 @@
import { z } from 'zod'
import type { JournalReducerState } from './journal-reducer'
import { admitOutcomeFormat1, type AdmittedOutcome } from './journal-outcome-format-1'
import {
strictStorage,
storageVerdict,
type StorageDeclaresAll,
type UndeclaredStorageKeys
} from './journal-storage-strict'
export const JOURNAL_OUTCOME_FORMAT_2 = 2
const ordinal = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER)
const coordinate = z.object({ epoch: z.string().min(1), sequence: ordinal })
export const inputEntrySchema = z.object({
context: z.object({ source: coordinate, basis: z.enum(['observed', 'legacy']), cut: ordinal }),
witnessCardId: z.string().min(1).optional()
})
export const inputCardSchema = z.object({
introduced: ordinal,
baseline: z.object({ coordinate: coordinate.nullable() }).optional()
})
const entries = z.tuple([z.string().min(1), inputEntrySchema])
const cards = z.tuple([z.string().min(1), inputCardSchema])
const native = new Map<string, z.ZodType>([
['inputEntries', strictStorage(entries)],
['inputCards', strictStorage(cards)]
])
export function isNativeOutcomeField(field: string): boolean {
return native.has(field)
}
/** Ordinary format-1 values keep their frozen meaning; format 2 adds only the native joins. */
export function admitOutcomeFormat2(value: {
field: string
key: string
json: string
deleted: boolean
}): 'unknown' | AdmittedOutcome {
const schema = native.get(value.field)
if (!schema) {
return admitOutcomeFormat1(value)
}
const key: unknown = JSON.parse(value.key)
if (typeof key !== 'string' || !key) {
throw new Error('invalid native journal key')
}
if (value.deleted) {
return value.json === 'null' ? { entryKey: key, kept: null } : 'unknown'
}
const kept: unknown = JSON.parse(value.json)
const verdict = storageVerdict(schema, kept)
if (verdict === 'newer') {
return 'unknown'
}
if (verdict !== 'stored' || !Array.isArray(kept) || kept[0] !== key) {
throw new Error('invalid native journal outcome')
}
return { entryKey: key, kept }
}
const admittedEntry =
z.custom<[string, JournalReducerState['inputEntries'] extends Map<string, infer V> ? V : never]>()
const admittedCard =
z.custom<[string, JournalReducerState['inputCards'] extends Map<string, infer V> ? V : never]>()
// Admission walked the declared schema once; these views keep the exact admitted bytes' value.
export function decodeNativeOutcome(
state: JournalReducerState,
field: string,
key: unknown,
kept: unknown,
deleted: boolean
): void {
if (field === 'inputEntries') {
if (deleted) {
state.inputEntries.delete(z.string().parse(key))
} else {
const [id, value] = admittedEntry.parse(kept)
state.inputEntries.set(id, value)
}
} else if (field === 'inputCards') {
if (deleted) {
state.inputCards.delete(z.string().parse(key))
} else {
const [id, value] = admittedCard.parse(kept)
state.inputCards.set(id, value)
}
}
}
export type OutcomeFormat2DeclaresNativeKeys = [
StorageDeclaresAll<
UndeclaredStorageKeys<
JournalReducerState['inputEntries'] extends Map<string, infer V> ? V : never,
z.output<typeof inputEntrySchema>
>
>,
StorageDeclaresAll<
UndeclaredStorageKeys<
JournalReducerState['inputCards'] extends Map<string, infer V> ? V : never,
z.output<typeof inputCardSchema>
>
>
]
@@ -1,12 +1,12 @@
import type Database from '../../sqlite/sync-database'
import { readJournalSessionEpoch } from './journal-row-table'
import { AgentSessionJournalError } from './journal-write-guards'
import { JOURNAL_OUTCOME_FORMAT_1 } from './journal-outcome-format-1'
import { JOURNAL_OUTCOME_FORMAT_2 } from './journal-outcome-format-2'
import { JournalOutcomeDecodeError } from './journal-outcome-codec'
/** The newest record format this build reads, and the one it writes. Each stored record carries its
* own format; older formats keep their frozen decoders and are never rewritten. */
export const JOURNAL_OUTCOME_FORMAT = JOURNAL_OUTCOME_FORMAT_1
export const JOURNAL_OUTCOME_FORMAT = JOURNAL_OUTCOME_FORMAT_2
/** Builds before outcome records refuse a row of this version wherever it sits in the epoch.
* Under this version, how the input rows before the marker fold into keys and values is frozen: the
* tail's records were written over that reading. A build that reads old rows into different keys or
@@ -28,6 +28,7 @@ import { encodeJournalState } from './journal-state-codec'
import type * as RowStorage from './journal-storage-row-schema'
import type * as Strict from './journal-storage-strict'
import type * as OutcomeFormat1 from './journal-outcome-format-1'
import type * as OutcomeFormat2 from './journal-outcome-format-2'
import type * as Transition from './journal-reducer-transition'
import {
projectionBase as base,
@@ -44,7 +45,7 @@ import {
// Admission's own work: strict walks of input rows and of outcome records, counted at the calls,
// and every strict walk at all, counted at each strict schema storage builds.
const admission = vi.hoisted(() => ({ rows: 0, records: 0, walks: 0 }))
const admission = vi.hoisted(() => ({ rows: 0, records: 0, walks: 0, tombstones: 0 }))
vi.mock('./journal-storage-strict', async (original) => {
const actual = await original<typeof Strict>()
// A schema storage cannot close further comes back as it is, so one instance can recur.
@@ -84,13 +85,31 @@ vi.mock('./journal-outcome-format-1', async (original) => {
...actual,
admitOutcomeFormat1: (value: Parameters<typeof actual.admitOutcomeFormat1>[0]) => {
admission.records += 1
if (value.deleted) {
admission.tombstones++
}
return actual.admitOutcomeFormat1(value)
}
}
})
vi.mock('./journal-outcome-format-2', async (original) => {
const actual = await original<typeof OutcomeFormat2>()
return {
...actual,
admitOutcomeFormat2: (value: Parameters<typeof actual.admitOutcomeFormat2>[0]) => {
if (actual.isNativeOutcomeField(value.field)) {
admission.records++
if (value.deleted) {
admission.tombstones++
}
}
return actual.admitOutcomeFormat2(value)
}
}
})
// Every draft transition a write or a replacement folds through, to count the nodes it reaches.
const transitions = vi.hoisted(() => {
const opened: { tracked: () => number }[] = []
const opened: { tracked: () => number; trackedDerived: () => number }[] = []
return opened
})
vi.mock('./journal-reducer-transition', async (original) => {
@@ -146,7 +165,16 @@ const send = (seq: number, clientMessageId: string, handoverRecorded?: true): Jo
clientMessageId,
body: message(clientMessageId),
providerHandle,
...(handoverRecorded ? { handoverRecorded } : {}),
...(handoverRecorded
? {
handoverRecorded,
inputOrderContext: {
source: { epoch, sequence: seq },
basis: 'observed' as const,
cut: seq
}
}
: {}),
payloadFingerprint: agentSessionSendBodyFingerprint(sessionId, message(clientMessageId))
})
const revision =
@@ -466,9 +494,23 @@ it.each(['rewind', 'replacement', 'rollover'] as const)(
rows
]),
...work.writes.map(([target, rows]): [string, number] => [`${target}`, Number(rows)]),
['draft nodes', transitions.reduce((sum, transition) => sum + transition.tracked(), 0)]
[
'canonical draft nodes',
transitions.reduce(
(sum, transition) => sum + transition.tracked() - transition.trackedDerived(),
0
)
]
])
return { statements: work.statements, counts }
const derived = transitions.reduce((sum, transition) => sum + transition.trackedDerived(), 0)
// Two coordinate indexes each have an all-card and a domain AVL; a point path
// visits at most twice log2(K+2), reaching at most eight objects at each level.
expect(derived).toBeLessThanOrEqual(64 * (size + 1) * Math.ceil(Math.log2(size + 2)))
console.info(
'replacement-draft',
JSON.stringify({ path, size, canonical: counts.get('canonical draft nodes'), derived })
)
return { statements: work.statements, counts, derived }
}
// A chat's first item and card start collections an empty chat lacks, so the line is taken
// between non-empty chats: 10N more restated items cost ten times what N more do.
@@ -484,7 +526,10 @@ it.each(['rewind', 'replacement', 'rollover'] as const)(
grown: 10 * ((small.counts.get(counted) ?? 0) - from)
})
}
expect(large.counts.get('draft nodes')).toBeGreaterThan(small.counts.get('draft nodes') ?? 0)
expect(large.counts.get('canonical draft nodes')).toBeGreaterThan(
small.counts.get('canonical draft nodes') ?? 0
)
expect(large.derived).toBeGreaterThan(small.derived)
}
)
@@ -522,8 +567,15 @@ it("opens a new chat reading each key's newest record once, the same with N and
journal_outcome_epochs: 1,
journal_outcomes: small.keys,
journal_rows: 1,
journal_sessions: 2
journal_sessions: 2,
other: 0
})
// Empty legacy ownership still costs one session-scoped, position-ordered lookup per restore.
for (const { work } of [small, large]) {
expect(work.statements.filter((sql) => !/\bFROM\s+journal_\w+/.test(sql))).toStrictEqual([
'SELECT message_id, position, body_json, fingerprint, state, hold_reason, returned_reason, returned_rejection, consumed_as, queued_epoch, queued_sequence, options_json FROM queued_messages WHERE session_id = ? ORDER BY position ASC'
])
}
// Each read is one seek down the key index from the last key: no scan, no sort, no revision walk.
const plan = small.database.db
.prepare(`EXPLAIN QUERY PLAN ${JOURNAL_NEWEST_OUTCOME_SQL}`)
@@ -553,14 +605,22 @@ async function prefixChat(size: number, label: string) {
it('writes to an existing chat without reading its prefix, the same at N and 10N', async () => {
const small = await prefixChat(N, 'prefix-append-small')
const large = await prefixChat(10 * N, 'prefix-append-large')
for (const key of ['first', 'streamed', 'accepted'] as const) {
for (const key of ['streamed', 'accepted'] as const) {
expectSameWork(small[key], large[key])
}
// The first write appends the marker, then the item's record and the two scalars it changed.
expect(small.first.writes).toStrictEqual([
['journal_outcomes INSERT', 3],
['journal_rows INSERT', 1]
])
// Conversion writes each existing identity and the one own entry once, plus the
// item's record and two scalars; every subsequent ordinary append stays point work.
for (const [size, fixture] of [
[N, small],
[10 * N, large]
] as const) {
expect(fixture.first.writes).toStrictEqual([
['journal_outcomes INSERT', size + 4],
['journal_rows INSERT', 1]
])
expect(fixture.first.read.journal_rows).toBe(1) // Newly inserted marker read-back, no prefix scan.
}
expect(large.first.statements).toEqual(small.first.statements) // Prepared statements are reused for each converted key.
expect(small.streamed.writes).toStrictEqual([['journal_outcomes INSERT', 1]])
expect(large.rig.state().submissions.get('send')?.dispatchState).toBe('accepted')
})
@@ -587,7 +647,8 @@ it("opens an existing chat reading main's prefix rows and each tail key's newest
expect(work.read.journal_rows).toBe(prefix + 2)
expect(work.read.journal_outcomes).toBe(keys)
}
expect(large.keys).toBe(small.keys)
// The first mandatory conversion persisted one native identity per original card.
expect(large.keys - small.keys).toBe(9 * N)
})
/** The text of each keyed record an open decodes: every key's newest live record. */
@@ -613,6 +674,7 @@ it('admits each row and record an existing chat reads or writes once, at N and 1
admission.rows = 0
admission.records = 0
admission.walks = 0
admission.tombstones = 0
const parses = vi.spyOn(JSON, 'parse')
restoreJournal(database.db, sessionId)
const parsed = parses.mock.calls.filter(([text]) => entries.texts.has(text)).length
@@ -624,6 +686,7 @@ it('admits each row and record an existing chat reads or writes once, at N and 1
admission.rows = 0
admission.records = 0
admission.walks = 0
admission.tombstones = 0
await rig.writer.enqueue(revision('answer', message('z'.repeat(4_096), 'assistant'), 3))
const written = { ...admission, writes: watched.take().writes }
watched.stop()
@@ -634,7 +697,7 @@ it('admits each row and record an existing chat reads or writes once, at N and 1
expect(open.read.journal_rows).toBe(prefix + 2)
expect(open.rows).toBe(prefix)
expect(open.records).toBe(keys)
expect(open.walks).toBe(open.rows + open.records)
expect(open.walks).toBe(open.rows + open.records - open.tombstones)
expect(open.entries).toBeGreaterThan(0)
expect(open.parsed).toBe(open.entries)
expect(paged.rows).toHaveLength(100)
@@ -642,7 +705,7 @@ it('admits each row and record an existing chat reads or writes once, at N and 1
// The written row's read-back, and each record the write inserts.
expect(written.rows).toBe(1)
expect(written.writes).toStrictEqual([['journal_outcomes INSERT', written.records]])
expect(written.walks).toBe(written.rows + written.records)
expect(written.walks).toBe(written.rows + written.records - written.tombstones)
}
})
@@ -1,3 +1,5 @@
import { journalInputOrder } from './journal-input-order'
import { detachJournalNativePrefix, restoreJournalNativeState } from './journal-input-conversion'
import type Database from '../../sqlite/sync-database'
import type { JournalReducerState } from './journal-reducer'
import { applyJournalRow } from './journal-reducer'
@@ -28,12 +30,15 @@ export function readJournalOutcomes(
baseline?: JournalReducerState
): JournalReducerState {
const newest = db.prepare(JOURNAL_NEWEST_OUTCOME_SQL)
const prefix = detachJournalNativePrefix(baseline)
const fold = new JournalOutcomeFold()
let after = { field: '', key: '' }
for (;;) {
const row = newest.get(source.generation, after.field, after.key, source.through)
if (!row) {
return fold.finish(sessionId, epoch, source.through, baseline)
const state = fold.finish(sessionId, epoch, source.through, baseline)
restoreJournalNativeState(db, state, source.format, fold, prefix)
return state
}
const value = fold.add(row)
after = { field: value.field, key: value.key }
@@ -47,10 +52,13 @@ export function persistJournalOutcomeRows(
state: JournalReducerState,
source: Pick<JournalOutcomeSource, 'generation' | 'prefix'>,
rows: readonly JournalRow[],
savedAt?: number
savedAt?: number,
upgradeNative = false,
finish?: (state: JournalReducerState) => void
): { state: JournalReducerState; adopt: () => JournalReducerState } {
let scalar = state.lastSequence === 0 ? {} : journalOutcomeScalars(state)
const transition = isolateJournalTransition(state, () => undefined)
journalInputOrder(transition.state).introduceOverlays(transition.state)
const previous = db.prepare(`SELECT ordinal, deleted FROM journal_outcomes
WHERE generation = ? AND field = ? AND entry_key = ? ORDER BY seq DESC LIMIT 1`)
const insert = db.prepare(`INSERT INTO journal_outcomes
@@ -61,7 +69,23 @@ export function persistJournalOutcomeRows(
// unusable never reaches the model, so it is never refused by the strict record encoder.
const { stored } = journalRowReadBack(state.sessionId, row)
applyJournalRow(transition.state, stored, savedAt ?? row.ts)
if (row === rows.at(-1)) {
finish?.(transition.state)
}
const changes = transition.changes(true)
if (upgradeNative) {
for (const path of ['inputEntries', 'inputCards'] as const) {
const changed = new Set(
changes.filter((change) => change.path === path).map((change) => change.key)
)
for (const [key, value] of transition.state[path]) {
if (!changed.has(key)) {
changes.push({ path, key, value, deleted: false, reinsert: true })
}
}
}
upgradeNative = false
}
// Only this row's own transition is recorded, never a snapshot of a prefix value it left alone.
const encoded = encodeJournalOutcomes(transition.state, changes, scalar, transition.untracked)
scalar = encoded.scalars
@@ -1,3 +1,6 @@
import { journalInputAcceptance } from './journal-input-provenance'
import { copyJournalNativeState } from './journal-input-conversion'
import { journalInputOrder } from './journal-input-order'
import type Database from '../../sqlite/sync-database'
import { createJournalReducerState, type JournalReducerState } from './journal-reducer'
import type { JournalRow } from './journal-row-schema'
@@ -16,13 +19,20 @@ export function prepareJournalOutcomeWrite(
rows: readonly JournalRow[],
savedAt: number
): { adopt: () => JournalReducerState } {
const previous = journalOutcomeSource(db, live.sessionId, live.epoch)
const source =
journalOutcomeSource(db, live.sessionId, live.epoch) ??
startJournalOutcomeLog(db, live.sessionId, live.epoch, live.lastSequence, savedAt)
previous ?? startJournalOutcomeLog(db, live.sessionId, live.epoch, live.lastSequence, savedAt)
if (source.through !== live.lastSequence) {
throw new Error('journal outcome writer has stale coverage')
}
const transition = persistJournalOutcomeRows(db, live, source, rows, savedAt)
const transition = persistJournalOutcomeRows(
db,
live,
source,
rows,
savedAt,
!previous || previous.format < 2
)
advanceJournalOutcomeSource(db, live.sessionId, live.epoch, transition.state.lastSequence)
return { adopt: transition.adopt }
}
@@ -32,15 +42,33 @@ export function publishJournalOutcomeEpoch(
db: Database.Database,
sessionId: string,
epoch: string,
rows: readonly JournalRow[]
rows: readonly JournalRow[],
previous?: JournalReducerState
): { adopt: () => JournalReducerState } {
const first = rows[0]
if (!first || first.seq !== 1 || first.kind !== 'epoch') {
throw new Error('missing founding epoch')
}
const source = startJournalOutcomeLog(db, sessionId, epoch, 0, first.ts)
const initial = createJournalReducerState(sessionId, epoch)
const transition = persistJournalOutcomeRows(db, initial, source, rows)
const initial = createJournalReducerState(sessionId, epoch, (id) =>
journalInputAcceptance(db, sessionId, id)
)
if (previous) {
copyJournalNativeState(previous, initial)
}
journalInputOrder(initial).beginReconstruction()
const transition = persistJournalOutcomeRows(
db,
initial,
source,
rows,
undefined,
true,
(state) => {
journalInputOrder(state).restore(state)
journalInputOrder(state).finishReconstruction(state)
}
)
advanceJournalOutcomeSource(db, sessionId, epoch, transition.state.lastSequence)
return { adopt: transition.adopt }
}
@@ -1,3 +1,5 @@
import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key'
import { journalInputOrder } from './journal-input-order'
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -171,3 +173,155 @@ it('refuses to publish a new epoch over one the caller did not see, writing noth
expect(database.db.prepare('SELECT * FROM journal_rows ORDER BY seq').all()).toEqual(before)
expect(restoreJournal(database.db, sessionId)?.state.epoch).toBe(epoch)
})
// A frozen marker-5 prefix and an outcome-only format-1 chat carry different evidence.
it.each(['prefix', 'zero', 'recreated'] as const)(
'lifts actual %s evidence once, then treats native tombstones as authoritative',
async (path) => {
const user = {
kind: 'message' as const,
role: 'user' as const,
blocks: [{ type: 'text' as const, text: 'M' }]
}
const rows: JournalRow[] = [
projectionEpochRow,
...['C', 'D'].map((id, i): JournalRow => ({
...base(i + 2),
kind: 'queue',
messageId: id,
change: {
op: 'queued',
body: id === 'C' ? { ...user, command: { name: 'compact' } } : user,
position: i,
queuedAt: { epoch, sequence: i + 2 }
}
})),
{
...base(4),
kind: 'submission',
clientMessageId: 'S',
body: user,
payloadFingerprint: 'S',
providerHandle: { kind: 'codex', threadId: 'thread-1' },
handoverRecorded: true
},
{
...base(5),
kind: 'dispatch',
clientMessageId: 'S',
state: 'accepted',
providerItemId: null,
reason: null
},
{
...base(6),
kind: 'submission',
clientMessageId: 'M',
body: user,
payloadFingerprint: 'M',
providerHandle: { kind: 'codex', threadId: 'thread-1' },
handoverRecorded: true,
inputOrderContext: { source: { epoch, sequence: 6 }, basis: 'observed', cut: 6 }
}
]
if (path !== 'zero') {
seedProjectionRows(database, rows)
} else {
seedOutcomeRows(database, rows)
}
let state = restoreJournal(database.db, sessionId)!.state
const first = projectionWriter(database, state)
rigs.push(first)
await first.writer.enqueue((seq, ts): JournalRow => ({
...base(seq),
ts,
kind: 'queue',
messageId: 'D',
change: { op: 'edited', body: { ...user, blocks: [{ type: 'text', text: 'latest D' }] } }
}))
if (path === 'recreated') {
await first.writer.enqueue((seq): JournalRow => ({
...base(seq),
kind: 'tombstone',
itemId: agentJournalSubmissionKey('M'),
revision: seq,
retireSubmission: true
}))
await first.writer.enqueue((seq): JournalRow => ({
...base(seq),
kind: 'submission',
clientMessageId: 'M',
body: user,
payloadFingerprint: 'M-recreated',
providerHandle: { kind: 'codex', threadId: 'thread-1' },
handoverRecorded: true
}))
}
// These ordinary records are admitted by the frozen format-1 contract; old writers had no native families.
database.db.exec(
"DELETE FROM journal_outcomes WHERE field IN ('inputEntries','inputCards'); UPDATE journal_outcomes SET format=1; UPDATE journal_outcome_epochs SET format=1"
)
const prefix = liveTestJournalRows(database.db, sessionId)
const restored = restoreJournal(database.db, sessionId)!
expect(restored).toMatchObject({ newer: null, damage: null })
state = restored.state
expect(state.queue.cards.get('D')?.body.blocks).toEqual([{ type: 'text', text: 'latest D' }])
expect(journalInputOrder(state).firstQueued()?.clientMessageId).toBe('M')
expect(journalInputOrder(state).forced('M')).toBe(path === 'prefix')
expect(state.inputEntries.get('M')?.context.basis).toBe(
path === 'prefix' ? 'observed' : 'legacy'
)
if (path === 'prefix') {
expect(
['C', 'D'].map((id) =>
journalInputOrder(state).precedes(state.queue.cards.get(id)!, state.submissions.get('M')!)
)
).toEqual([true, true])
}
const rig = projectionWriter(database, state)
rigs.push(rig)
database.db.exec(
"CREATE TRIGGER no_upgrade BEFORE UPDATE ON journal_outcome_epochs BEGIN SELECT RAISE(ABORT, 'upgrade failed'); END"
)
await expect(rig.writer.enqueue((seq) => item(seq, 'upgrade'))).rejects.toThrow(
'upgrade failed'
)
database.db.exec('DROP TRIGGER no_upgrade')
expect(
database.db
.prepare(
"SELECT COUNT(*) AS n FROM journal_outcomes WHERE field IN ('inputEntries','inputCards')"
)
.get()?.n
).toBe(0)
expect(rig.state().items.has('upgrade')).toBe(false)
expect(liveTestJournalRows(database.db, sessionId)).toEqual(prefix)
await rig.writer.enqueue((seq) => item(seq, 'upgrade'))
expect(database.db.prepare('SELECT format FROM journal_outcome_epochs').get()?.format).toBe(2)
expectProjectionMatches(database, rig.state())
const native = database.db
.prepare(
"SELECT field, entry_key FROM journal_outcomes WHERE field IN ('inputEntries','inputCards')"
)
.all()
expect(native).toHaveLength(3)
expect(native).toContainEqual({ field: 'inputEntries', entry_key: '"M"' })
expect(liveTestJournalRows(database.db, sessionId)).toEqual(prefix)
await rig.writer.enqueue((seq): JournalRow => ({
...base(seq),
kind: 'dispatch',
clientMessageId: 'M',
state: 'accepted',
providerItemId: null,
reason: null
}))
expectProjectionMatches(database, rig.state())
expect(restoreJournal(database.db, sessionId)!.state.inputEntries.size).toBe(0)
expect(
journalInputOrder(restoreJournal(database.db, sessionId)!.state).firstQueued()
).toBeUndefined()
// A later unrelated write cannot revive the old prefix's own component.
await rig.writer.enqueue((seq) => item(seq, 'after-retirement'))
expect(restoreJournal(database.db, sessionId)!.state.inputEntries.size).toBe(0)
}
)
@@ -1,3 +1,4 @@
import { readJournalLegacyCards } from './journal-queue-import'
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -8,6 +9,7 @@ import {
closeTestJournalHostDatabases,
openTestJournalHostDatabase
} from './journal-host-database-test-support'
import { journalInputOrder } from './journal-input-order'
import { encodeJournalState } from './journal-state-codec'
import { JournalStopMarks } from './journal-stop-marks'
import type { JournalHostDatabase } from './journal-host-database'
@@ -54,6 +56,26 @@ it('adopts nothing on an outcome, coverage, receipt, outer-abort or COMMIT failu
itemId: 'stop',
revision: 1,
stopEvent: { reason: 'user-stop', at: 1_002 }
},
...['C', 'D'].map((id, i): JournalRow => ({
...base(i + 3),
kind: 'queue',
messageId: id,
change: {
op: 'queued',
body: id === 'C' ? { ...user, command: { name: 'compact' } } : user,
position: i,
queuedAt: { epoch, sequence: i + 3 }
}
})),
{
...base(5),
kind: 'submission',
clientMessageId: 'legacy',
body: user,
handoverRecorded: true,
providerHandle: { kind: 'codex', threadId: 'thread-1' },
payloadFingerprint: 'legacy'
}
])
rig = projectionWriter(database, state)
@@ -68,6 +90,12 @@ it('adopts nothing on an outcome, coverage, receipt, outer-abort or COMMIT failu
kind: 'submission',
clientMessageId: id,
body: user,
handoverRecorded: true,
inputOrderContext: {
source: { epoch, sequence: seq },
basis: 'observed',
cut: journalInputOrder(rig!.state()).lineage.at(seq)
},
providerHandle: { kind: 'codex', threadId: 'thread-1' },
payloadFingerprint: agentSessionSendBodyFingerprint(sessionId, user)
})
@@ -78,13 +106,57 @@ it('adopts nothing on an outcome, coverage, receipt, outer-abort or COMMIT failu
},
committed: vi.fn()
}
await expect(writer.enqueue(send('preserved'), receipt)).resolves.toMatchObject({ seq: 3 })
await expect(writer.enqueue(send('preserved'), receipt)).resolves.toMatchObject({ seq: 6 })
expect(receipt.committed).toHaveBeenCalledOnce()
expect(rig.state().submissions.has('preserved')).toBe(true)
expect(stops.latest()?.settle).toBe(settle)
expectProjectionMatches(database, rig.state())
database.db
.prepare(`INSERT INTO queued_messages
(session_id,message_id,position,body_json,fingerprint,created_at,host_instance,state)
VALUES (?,?,?,?,?,0,'legacy','waiting')`)
.run(sessionId, 'overlay', 2, JSON.stringify(user), 'overlay')
const beforeRead = encodeJournalState(rig.state())
journalInputOrder(rig.state()).overlay(
rig.state(),
readJournalLegacyCards(database.db, rig.state())
)
expect(encodeJournalState(rig.state())).toEqual(beforeRead)
const saved = encodeJournalState(rig.state())
const live = rig.state()
const version = journalInputOrder(live).version()
const clock = journalInputOrder(live).lineage.clock
const pending = journalInputOrder(live)
.unsettledSubmissions()
.map((entry) => entry.clientMessageId)
const attempt = (id: string) =>
writer.enqueueRows(
() => [
(seq, ts): JournalRow => ({
...base(seq),
ts,
kind: 'dispatch',
clientMessageId: 'legacy',
state: 'accepted',
providerItemId: null,
reason: null
}),
send(id)
],
receipt
)
const unchanged = () => {
expect(live.inputCards.has('overlay')).toBe(false)
expect(journalInputOrder(live).version()).toBe(version)
expect(journalInputOrder(live).lineage.clock).toBe(clock)
expect(
journalInputOrder(live)
.unsettledSubmissions()
.map((entry) => entry.clientMessageId)
).toEqual(pending)
expect(live.inputEntries.has('legacy')).toBe(true)
expect(live.inputCards.get('D')?.baseline).toBeDefined()
}
const scenarios = [
[
'outcome',
@@ -105,8 +177,9 @@ it('adopts nothing on an outcome, coverage, receipt, outer-abort or COMMIT failu
] as const
for (const [name, sql] of scenarios) {
database.db.exec(sql)
await expect(writer.enqueue(send(name), receipt)).rejects.toThrow('failure')
await expect(attempt(name)).rejects.toThrow('failure')
database.db.exec('DROP TRIGGER failure')
unchanged()
expect(rig.state()).toBe(live)
expect(encodeJournalState(live)).toEqual(saved)
expect(receipt.committed).toHaveBeenCalledOnce()
@@ -120,14 +193,32 @@ it('adopts nothing on an outcome, coverage, receipt, outer-abort or COMMIT failu
}
exec(sql)
})
await expect(writer.enqueue(send('commit'), receipt)).rejects.toThrow('injected boundary failure')
await expect(attempt('commit')).rejects.toThrow('injected boundary failure')
spy.mockRestore()
unchanged()
expect(rig.state()).toBe(live)
expect(encodeJournalState(live)).toEqual(saved)
expect(stops.latest()?.settle).toBe(settle)
expect(receipt.committed).toHaveBeenCalledOnce()
expect(database.db.prepare('SELECT COUNT(*) AS n FROM test_receipts').get()?.n).toBe(1)
expectProjectionMatches(database, live)
await attempt('retry')
expect(live.inputEntries.has('legacy')).toBe(false)
expect(live.inputEntries.has('retry')).toBe(true)
expect(live.inputCards.has('overlay')).toBe(true)
const overlayWrites = database.db
.prepare(
"SELECT seq,value_json FROM journal_outcomes WHERE field='inputCards' AND entry_key=? AND deleted=0 ORDER BY seq"
)
.all(JSON.stringify('overlay'))
// Both rows touch the introduced object after the legacy-baseline cleanup in the first row.
expect(overlayWrites.map((row) => row.seq)).toEqual([live.lastSequence - 1, live.lastSequence])
expect(overlayWrites.map((row) => row.value_json)).toEqual(
Array(2).fill(JSON.stringify(['overlay', live.inputCards.get('overlay')]))
)
expect(receipt.committed).toHaveBeenCalledTimes(2)
expectProjectionMatches(database, live)
const first = rig.state().lastSequence + 1
await writer.enqueueRows(() => [
(seq, ts) => ({
@@ -1,3 +1,4 @@
import { journalInputOrder } from './journal-input-order'
// Latch handoff outcomes before rewind can retire their rows; requeue only proven-undelivered sends.
import type {
@@ -210,8 +211,10 @@ function spendReportedCommandCards(state: JournalReducerState): void {
function setCard(state: JournalReducerState, card: QueuedMessageRow): void {
if (state.queue.cards.get(card.messageId) !== card) {
const prior = state.queue.cards.get(card.messageId)
state.queue.cards.set(card.messageId, card)
state.queue.revision += 1
journalInputOrder(state).queueChanged(state, prior, card)
}
}
@@ -1,3 +1,5 @@
import type Database from '../../sqlite/sync-database'
import { journalInputOrder } from './journal-input-order'
// Adopt legacy cards and delete their rows atomically, using the same fold before and after import.
// Retry failed bulk import on next open; do not loop while the host is running.
@@ -61,9 +63,9 @@ function unconfirmedHandOff(
| { attempt: string; holdReason?: undefined }
| { attempt?: undefined; holdReason: string }
| null {
const attempts = [...state.submissions.values()]
.filter((submission) => submission.queuedMessageId === row.messageId)
.sort((a, b) => (a.submittedSequence ?? 0) - (b.submittedSequence ?? 0))
const attempts = [...journalInputOrder(state).linkedTo(row.messageId)].sort(
(a, b) => (a.submittedSequence ?? 0) - (b.submittedSequence ?? 0)
)
if (!attempts.some(handOffInDoubt)) {
return null
}
@@ -95,7 +97,10 @@ export class LegacyQueuedCards {
owner: JournalReducerState
key: string
cards: readonly QueuedMessageRow[]
byId: ReadonlyMap<string, QueuedMessageRow>
display?: JournalReducerState
displayThrough?: number
handedOff: readonly AgentJournalSubmission[]
} | null = null
constructor(private readonly deps: LegacyDeps) {}
@@ -105,8 +110,8 @@ export class LegacyQueuedCards {
if (this.rows().length === 0) {
return ''
}
const { epoch, lastSequence } = this.deps.state()
return `${this.revision}:${epoch}:${lastSequence}`
const state = this.deps.state()
return `${this.revision}:${state.epoch}:${journalInputOrder(state).linkedVersion()}`
}
cards(): readonly QueuedMessageRow[] {
@@ -121,23 +126,38 @@ export class LegacyQueuedCards {
const change = legacyQueuedChange(state, row)
return change ? [queuedCardFromChange(state, row.messageId, change)] : []
})
this.viewed = { owner: state, key, cards }
const handedOff = cards.flatMap((card) => {
const submission =
card.consumedAs === null ? undefined : state.submissions.get(card.consumedAs)
return submission && handOffInDoubt(submission) ? [submission] : []
})
this.viewed = {
owner: state,
key,
cards,
handedOff,
byId: new Map(cards.map((card) => [card.messageId, card]))
}
}
return this.viewed.cards
}
get(messageId: string): QueuedMessageRow | undefined {
this.cards()
return this.viewed?.byId.get(messageId)
}
// Failed adoption cannot hide spent sends; derive display changes only on copied state.
displayState(): JournalReducerState {
const state = this.deps.state()
const cards = this.cards()
if (this.viewed?.display) {
this.cards()
if (this.viewed?.display && this.viewed.displayThrough === state.lastSequence) {
return this.viewed.display
}
const handedOff = cards.flatMap((card) => {
const submission =
card.consumedAs === null ? undefined : state.submissions.get(card.consumedAs)
return submission && handOffInDoubt(submission) ? [{ ...submission }] : []
})
const handedOff = this.viewed?.handedOff ?? []
if (this.viewed) {
this.viewed.displayThrough = state.lastSequence
}
if (handedOff.length === 0) {
if (this.viewed) {
this.viewed.display = state
@@ -150,7 +170,8 @@ export class LegacyQueuedCards {
items: new Map(state.items),
submissions: new Map(state.submissions)
}
for (const submission of handedOff) {
for (const entry of handedOff) {
const submission = { ...entry }
display.submissions.set(submission.clientMessageId, submission)
foldQueuedHandOffInDoubt(display, submission)
}
@@ -238,12 +259,25 @@ export class LegacyQueuedCards {
}
private read(): QueuedMessageRow[] {
return this.deps
.database()
.db.prepare(
`SELECT ${LEGACY_QUEUED_MESSAGE_COLUMNS} FROM queued_messages WHERE session_id = ? ORDER BY position ASC`
)
.all(this.deps.sessionId)
.flatMap((row) => readStoredQueuedMessageRow(row) ?? [])
return readLegacyRows(this.deps.database().db, this.deps.sessionId)
}
}
export function readJournalLegacyCards(
db: Database.Database,
state: JournalReducerState
): QueuedMessageRow[] {
return readLegacyRows(db, state.sessionId).flatMap((row) => {
const change = legacyQueuedChange(state, row)
return change ? [queuedCardFromChange(state, row.messageId, change)] : []
})
}
function readLegacyRows(db: Database.Database, sessionId: string): QueuedMessageRow[] {
return db
.prepare(
`SELECT ${LEGACY_QUEUED_MESSAGE_COLUMNS} FROM queued_messages WHERE session_id = ? ORDER BY position ASC`
)
.all(sessionId)
.flatMap((row) => readStoredQueuedMessageRow(row) ?? [])
}
@@ -1,3 +1,4 @@
import { journalInputOrder } from './journal-input-order'
import { journalQueueAdoptionPlan } from './journal-queue-adoption-plan'
import {
journalAttachmentClaimReceipt,
@@ -20,7 +21,6 @@ import type { JournalQueuedMessagesDeps, JournalSubmissionConsume } from './jour
import {
deriveQueuePauses,
journalUserStopInForce,
nextSendableQueuedCard,
type DerivedQueuePause,
type JournalQueuePauseMarks
} from './queued-message-pause'
@@ -46,7 +46,11 @@ export type JournalQueueRestatement = {
}
export class JournalQueuedMessages {
readonly editLeases = new QueuedMessageEditLeases(() => this.list())
readonly editLeases = new QueuedMessageEditLeases(
() => this.list(),
undefined,
(id) => this.get(id)
)
private readonly legacy: LegacyQueuedCards
private viewed: { key: string; cards: readonly QueuedMessageRow[] } | null = null
@@ -58,7 +62,13 @@ export class JournalQueuedMessages {
return this.deps.sessionId
}
inputState(): JournalReducerState {
const state = this.deps.state()
journalInputOrder(state).overlay(state, this.legacy.cards())
return state
}
displayState(): JournalReducerState {
this.inputState()
return this.legacy.displayState()
}
@@ -77,11 +87,20 @@ export class JournalQueuedMessages {
}
get(messageId: string): QueuedMessageRow | null {
return (
this.deps.state().queue.cards.get(messageId) ??
this.legacy.cards().find((card) => card.messageId === messageId) ??
null
)
return this.deps.state().queue.cards.get(messageId) ?? this.legacy.get(messageId) ?? null
}
/** Selection reads identity summaries; current values are resolved after edits. */
nextInputCard(automatic = true): QueuedMessageRow | null {
const state = this.inputState()
const identity = journalInputOrder(state).head(state, this.deps.reopenFloor())
const card = identity ? this.get(identity.messageId) : null
return !card ||
card.state === 'returned' ||
journalInputOrder(state).paused(state, card, this.deps.reopenFloor()) ||
(automatic && this.editLeases.isHeld(card.messageId))
? null
: card
}
// Card insertion, attachment claims and send receipt must commit together.
@@ -210,9 +229,7 @@ export class JournalQueuedMessages {
card.fingerprint === input.fingerprint &&
input.consumedAs !== input.messageId
if (consumable && input.yieldsToPause) {
const cards = this.list()
const held = this.editLeases.heldIds(cards)
const next = nextSendableQueuedCard(this.derivePauses(cards), cards, held)
const next = this.nextInputCard()
if (next?.messageId === input.messageId) {
return
}
@@ -1,3 +1,4 @@
import { journalInputOrder, setJournalInputOrder } from './journal-input-order'
// Track writes through objects, iterators and collection entries; finalize only after COMMIT.
import type { JournalReducerState } from './journal-reducer'
@@ -9,6 +10,7 @@ import {
} from './journal-transition-collections'
type DraftNode = {
original: object
value: object
/** What the draft holds now: the copy once written, else the object itself. */
current: () => object
@@ -16,16 +18,24 @@ type DraftNode = {
listen: (changed: () => void) => void
}
function draftProperty<Value extends object>(value: Value, key: PropertyKey): unknown {
const descriptor = Reflect.getOwnPropertyDescriptor(value, key)
return descriptor && ('value' in descriptor ? descriptor.value : descriptor.get?.call(value))
}
class JournalDraft {
private readonly originals = new WeakMap<object, DraftNode>()
private readonly proxies = new WeakMap<object, DraftNode>()
private readonly collections: (JournalDraftMap<unknown, unknown> | JournalDraftSet<unknown>)[] =
[]
private readonly derived: object[] = []
private readonly assigned = new WeakSet<object>()
private readonly pending: (() => void)[] = []
private readonly resolved = new WeakMap<object, object>()
/** Objects and collections the fold reached; scaling tests pin how this grows with chat size. */
nodes = 0
derivedNodes = 0
private notifications: Set<() => void> | undefined
readonly access: JournalDraftAccess = {
wrap: (value, changed, path) => this.wrap(value, changed, path),
@@ -43,29 +53,37 @@ class JournalDraft {
return value
}
const node = this.proxies.get(value) ?? this.originals.get(value) ?? this.track(value, path)
node.listen(changed)
if (!path.startsWith('@inputOrder')) {
node.listen(changed)
}
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a node's draft (overlay collection or copy-on-write proxy) has the interface of the object it isolates.
return node.value as T
}
private track<T extends object>(value: T, path: string): DraftNode {
this.nodes += 1
if (path.startsWith('@inputOrder')) {
this.derivedNodes++
}
const listeners = new Set<() => void>()
let dirty = false
let notifying = false
const mark = (): void => {
dirty = true
if (notifying) {
const outer = this.notifications === undefined
const wave = (this.notifications ??= new Set())
// Rotated/shared index nodes can reach the same ancestor by several paths.
if (wave.has(mark)) {
return
}
// A batch consumes changes after each row; later mutations must notify again.
notifying = true
wave.add(mark)
try {
for (const notify of listeners) {
notify()
}
} finally {
notifying = false
if (outer) {
this.notifications = undefined
}
}
}
const listen = (notify: () => void): void => {
@@ -82,6 +100,7 @@ class JournalDraft {
: new JournalDraftSet(value, this.access, mark, path)
this.collections.push(collection)
node = {
original: value,
value: collection,
current: () => collection,
finish: () => collection.finish(),
@@ -143,6 +162,7 @@ class JournalDraft {
}
})
node = {
original: value,
value: proxy,
current,
finish: () => {
@@ -152,7 +172,7 @@ class JournalDraft {
const result = writable()
const keys = Reflect.ownKeys(result)
for (const key of keys) {
const child: unknown = Reflect.getOwnPropertyDescriptor(result, key)?.value
const child: unknown = draftProperty(result, key)
if (assignedKeys.has(key) || this.assigned.has(value) || this.tracks(child)) {
Reflect.set(result, key, this.finish(child))
}
@@ -168,13 +188,16 @@ class JournalDraft {
}
}
for (const key of keys) {
Reflect.set(value, key, Reflect.getOwnPropertyDescriptor(result, key)?.value)
Reflect.set(value, key, draftProperty(result, key))
}
return value
},
listen
}
}
if (path.startsWith('@inputOrder')) {
this.derived.push(node.value)
}
this.originals.set(value, node)
this.proxies.set(node.value, node)
return node
@@ -195,6 +218,10 @@ class JournalDraft {
adopt<T>(value: T): T {
const result = this.resolve(value)
// Derived nodes adopt their own point changes; rotated ancestors do not own those writes.
for (const node of this.derived) {
this.resolve(node)
}
for (const apply of this.pending) {
apply()
}
@@ -224,6 +251,10 @@ class JournalDraft {
}
const node = this.proxies.get(value) ?? this.originals.get(value)
if (node) {
// The derived owner points back to canonical state and shares index entries.
// Resolve that identity before children so a committed graph is finalized once.
this.resolved.set(value, node.original)
this.resolved.set(node.value, node.original)
const result = node.finish()
this.resolved.set(value, result)
this.resolved.set(node.value, result)
@@ -235,7 +266,7 @@ class JournalDraft {
}
// Values created during the fold may hold drafts introduced by spread or destructuring.
for (const key of Reflect.ownKeys(value)) {
const child: unknown = Reflect.getOwnPropertyDescriptor(value, key)?.value
const child: unknown = draftProperty(value, key)
const finished = this.finish(child)
if (finished !== child) {
Reflect.set(value, key, finished)
@@ -257,16 +288,28 @@ export function isolateJournalTransition<T>(
untracked: (key: string, value: unknown) => unknown
adopt: () => JournalReducerState
tracked: () => number
trackedDerived: () => number
} {
const draft = new JournalDraft()
const candidate = draft.wrap(state, () => undefined, '')
// Register canonical maps before the facade can reach them through an isolated query.
void candidate.inputEntries
void candidate.inputCards
const order = draft.wrap(journalInputOrder(state), () => undefined, '@inputOrder')
setJournalInputOrder(candidate, order)
const result = fold(candidate)
return {
state: candidate,
result,
changes: (reset) => draft.changes(reset),
changes: (reset) =>
draft.changes(reset).filter((entry) => !entry.path.startsWith('@inputOrder')),
untracked: (_key, value) => draft.untracked(value),
adopt: () => draft.adopt(candidate),
adopt: () => {
const adopted = draft.adopt({ state: candidate, order })
setJournalInputOrder(adopted.state, adopted.order)
return adopted.state
},
trackedDerived: () => draft.derivedNodes,
tracked: () => draft.nodes
}
}
@@ -1,3 +1,4 @@
import { journalInputOrder } from './journal-input-order'
import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types'
import { describe, expect, it } from 'vitest'
import {
@@ -48,6 +49,23 @@ function fold(rows: JournalRow[]): JournalReducerState {
const state = createJournalReducerState('session-1', EPOCH)
for (const row of rows) {
applyJournalRow(state, row)
expect(
journalInputOrder(state)
.unsettledSubmissions()
.map((entry) => entry.clientMessageId)
.sort()
).toEqual(
[...state.submissions.values()]
.filter((entry) => entry.dispatchState === 'pending' || entry.dispatchState === 'unknown')
.map((entry) => entry.clientMessageId)
.sort()
)
for (const entry of state.submissions.values()) {
if (entry.dispatchState !== 'pending') {
expect(journalInputOrder(state).forced(entry.clientMessageId)).toBe(false)
expect(journalInputOrder(state).place(entry.clientMessageId)).toBeUndefined()
}
}
}
return state
}
@@ -735,3 +753,34 @@ describe('re-adding a tombstoned row', () => {
expect(renderJournalState(state).items).toEqual([])
})
})
it('retires unknown execution debt with the submission tombstone', () => {
const state = fold([
{
kind: 'submission',
clientMessageId: 'retired',
providerHandle: { kind: 'codex', threadId: 'thread-1' },
payloadFingerprint: 'retired',
body: userText('retired'),
handoverRecorded: true,
...base(1)
},
{
kind: 'dispatch',
clientMessageId: 'retired',
state: 'unknown',
providerItemId: null,
reason: null,
...base(2)
},
{
kind: 'tombstone',
itemId: agentJournalSubmissionKey('retired'),
revision: 1,
retireSubmission: true,
...base(3)
}
])
expect(state.submissions.has('retired')).toBe(false)
expect(journalInputOrder(state).unsettledSubmissions()).toEqual([])
})
@@ -1,4 +1,10 @@
import { journalItemDeclaresEcho } from './journal-item-echo'
import {
journalInputOrder,
initializeJournalInputOrder,
type JournalInputOrder
} from './journal-input-order'
import type { JournalInputEntry, JournalInputCard } from './journal-input-lineage'
// Live append and replay share this fold. Revisions change content, preserving creation position
// and unspecified producer linkage; queued/rejected submissions use their handoff/settlement positions.
@@ -79,11 +85,17 @@ export type JournalReducerState = {
normalizedHandOffs: Set<string>
/** The queued messages, folded from their own rows and their hand-offs (`journal-queue-fold.ts`). */
queue: JournalQueueState
inputEntries: Map<string, JournalInputEntry>
inputCards: Map<string, JournalInputCard>
historyOrigin: JournalHistoryOrigin | null
}
export function createJournalReducerState(sessionId: string, epoch: string): JournalReducerState {
return {
export function createJournalReducerState(
sessionId: string,
epoch: string,
originalAcceptance?: ConstructorParameters<typeof JournalInputOrder>[1]
): JournalReducerState {
const state: JournalReducerState = {
sessionId,
epoch,
lastSequence: 0,
@@ -108,8 +120,12 @@ export function createJournalReducerState(sessionId: string, epoch: string): Jou
queuedHandOffPlaces: new Map(),
normalizedHandOffs: new Set(),
queue: createJournalQueueState(),
inputEntries: new Map(),
inputCards: new Map(),
historyOrigin: null
}
initializeJournalInputOrder(state, originalAcceptance)
return state
}
export function applyJournalRow(
@@ -117,8 +133,10 @@ export function applyJournalRow(
row: JournalRow,
savedAt = row.ts
): void {
journalInputOrder(state).beforeRow(state, row)
applyJournalRowFacts(state, row, savedAt)
foldJournalQueueRow(state, row)
journalInputOrder(state).afterRow(state, row)
}
function applyJournalRowFacts(state: JournalReducerState, row: JournalRow, savedAt: number): void {
@@ -1,16 +1,16 @@
import { journalInputOrder } from './journal-input-order'
import type { JournalInputContext } from './journal-input-lineage'
import { journalItemEchoFields, type JournalItemEcho } from './journal-item-echo'
import type {
AgentJournalSubmission,
AgentJournalItemBody,
AgentJournalItemIdentity,
AgentJournalMessageItem,
AgentJournalProducerLinkage,
AgentJournalRowAttribution,
AgentJournalTurnScope,
AgentSessionJournalIdentity,
AgentSessionJournalProviderHandle
} from '../../../shared/agent-session-journal-types'
import type { AgentSessionExecutionOptions } from '../../../shared/agent-session-execution-options'
import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding'
import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types'
import { agentJournalLinkageFields } from '../../../shared/agent-session-journal-producer'
@@ -37,7 +37,7 @@ import {
import { assertSubmissionIdUnused } from './journal-write-guards'
import { turnEndAfterStop } from './journal-stop-turn-end'
import { nextJournalItemRevision } from './journal-item-revision'
import type { ResolveDispatchInput } from './journal-store-contracts'
import type { ResolveDispatchInput, JournalSubmissionInput } from './journal-store-contracts'
import {
journalLifecycleMutationItemId,
journalLifecycleMutationRow,
@@ -99,16 +99,7 @@ export function journalTombstoneRowBuilder(
export function journalSubmissionRowBuilder(
state: () => JournalReducerState,
identity: Pick<AgentSessionJournalIdentity, 'providerHandle' | 'agent'>,
input: {
clientMessageId: string
payloadFingerprint: string
body: AgentJournalMessageItem
fence: number
handoverRecorded?: true
queuedMessageId?: string
origin?: 'client' | 'host'
options?: AgentSessionExecutionOptions
},
input: JournalSubmissionInput,
/** Stamp queue links from consumption here so every handoff preserves them. */
consume?: { messageId: string }
): RowBuilder<JournalSubmissionRow> {
@@ -120,6 +111,11 @@ export function journalSubmissionRowBuilder(
const queuedMessageId = consume?.messageId ?? input.queuedMessageId
return buildJournalSubmissionRow({
state: state(),
inputOrderContext: {
source: { epoch: state().epoch, sequence: seq },
basis: 'observed',
cut: journalInputOrder(state()).lineage.at(seq)
},
providerHandle: agentSessionJournalProviderHandle(identity),
...input,
...(queuedMessageId !== undefined ? { queuedMessageId } : {}),
@@ -286,23 +282,19 @@ export function buildJournalTombstoneRow(input: {
}
}
export function buildJournalSubmissionRow(input: {
state: JournalReducerState
clientMessageId: string
payloadFingerprint: string
providerHandle: AgentSessionJournalProviderHandle
body: AgentJournalMessageItem
seq: number
fence: number
ts: number
handoverRecorded?: true
queuedMessageId?: string
origin?: 'client' | 'host'
source?: AgentJournalSubmission['source']
options?: AgentSessionExecutionOptions
}): JournalSubmissionRow {
export function buildJournalSubmissionRow(
input: Omit<JournalSubmissionInput, 'source'> &
Pick<AgentJournalSubmission, 'source'> & {
state: JournalReducerState
providerHandle: AgentSessionJournalProviderHandle
seq: number
ts: number
inputOrderContext?: JournalInputContext
}
): JournalSubmissionRow {
return {
kind: 'submission',
...(input.inputOrderContext ? { inputOrderContext: input.inputOrderContext } : {}),
clientMessageId: input.clientMessageId,
payloadFingerprint: input.payloadFingerprint,
providerHandle: input.providerHandle,
@@ -0,0 +1,9 @@
import type { JournalRow } from './journal-row-schema'
export function serializeJournalRow(row: JournalRow): string {
return JSON.stringify(row)
}
export function journalRowByteLength(row: JournalRow): number {
return Buffer.byteLength(serializeJournalRow(row), 'utf8') + 1
}
@@ -559,3 +559,66 @@ it.each([{ dispatchId: '' }, { dispatchId: null }, { task: null }, { task: 17 }]
).toBe(false)
}
)
it('keeps ordering context additive and treats malformed optional facts as legacy annotations', () => {
const row = {
...BASE,
kind: 'submission',
clientMessageId: 'context',
payloadFingerprint: 'context',
providerHandle: { kind: 'codex', threadId: 't' },
body: { kind: 'message', role: 'user', blocks: [] },
inputOrderContext: { source: { epoch: 'source', sequence: 2 }, basis: 'observed', cut: 2 }
}
expect(parseJournalRow(JSON.stringify(row))).toMatchObject({
ok: true,
row: { inputOrderContext: row.inputOrderContext, body: row.body }
})
expect(
parseJournalRow(
JSON.stringify({
...row,
inputOrderContext: { ...row.inputOrderContext, basis: 'unreadable' }
})
)
).toEqual({
ok: true,
row: expect.not.objectContaining({ inputOrderContext: expect.anything() })
})
const epoch = {
...BASE,
kind: 'epoch',
reason: 'unreconcilable_prefix',
providerHandle: row.providerHandle,
inputOrderLineage: {
clock: 10,
cards: [{ messageId: 'C', introduced: 1, baseline: { coordinate: null } }]
}
}
expect(parseJournalRow(JSON.stringify(epoch))).toMatchObject({
ok: true,
row: { inputOrderLineage: epoch.inputOrderLineage }
})
const overlapping = {
...epoch,
inputOrderLineage: {
clock: 10,
cards: [
{
messageId: 'C',
introduced: 1,
baseline: { coordinate: { epoch: 'old', sequence: -1 } }
}
]
}
}
expect(parseJournalRow(JSON.stringify(overlapping))).toEqual({
ok: true,
row: expect.not.objectContaining({ inputOrderLineage: expect.anything() })
})
const invalid = parseJournalRow(JSON.stringify({ ...epoch, inputOrderLineage: { clock: 'bad' } }))
expect(invalid).toEqual({
ok: true,
row: expect.not.objectContaining({ inputOrderLineage: expect.anything() })
})
})
@@ -1,3 +1,5 @@
export { serializeJournalRow, journalRowByteLength } from './journal-row-encoding'
import type { JournalInputContext, JournalInputLineage } from './journal-input-lineage'
import type { JournalItemEcho } from './journal-item-echo'
// Upcast on read; preserve unknown versions/kinds and fail load instead of skipping history.
// New kinds need reader-first rollout or a version bump because older builds may delete them.
@@ -42,6 +44,7 @@ export type JournalEpochRow = JournalRowBase & {
kind: 'epoch'
reason: AgentJournalEpochReason
providerHandle: AgentSessionJournalProviderHandle
inputOrderLineage?: JournalInputLineage
/** Preserve original adoption through same-conversation epoch replacement. */
historyOrigin?: unknown
}
@@ -119,6 +122,7 @@ export function isJournalStopOrResumeRow(row: JournalRow): row is JournalStopOrR
/** Persist before dispatch; the optimistic bubble gives provider echoes a slot without duplication. */
export type JournalSubmissionRow = JournalRowBase & {
kind: 'submission'
inputOrderContext?: JournalInputContext
clientMessageId: string
payloadFingerprint: string
providerHandle: AgentSessionJournalProviderHandle
@@ -190,10 +194,6 @@ export type JournalRowParse =
/** Future content forbids writing/compacting this journal. */
| { ok: false; unreadable: true }
export function serializeJournalRow(row: JournalRow): string {
return JSON.stringify(row)
}
export function parseJournalRow(line: string): JournalRowParse {
let parsed: unknown
try {
@@ -357,7 +357,3 @@ function lifecycleMutationContent(value: unknown): AgentJournalContentVerdict {
}
return isJournalTag(kind) ? 'unreadable' : 'malformed'
}
export function journalRowByteLength(row: JournalRow): number {
return Buffer.byteLength(serializeJournalRow(row), 'utf8') + 1
}
@@ -1,6 +1,13 @@
import { isInputContext, isInputLineage } from './journal-input-lineage'
import { isAdmissibleAgentSessionContextUsage } from '../../../shared/agent-session-context-usage-schema'
export function dropUnusableRowAnnotations(record: Record<string, unknown>): void {
if (record.inputOrderContext !== undefined && !isInputContext(record.inputOrderContext)) {
delete record.inputOrderContext
}
if (record.inputOrderLineage !== undefined && !isInputLineage(record.inputOrderLineage)) {
delete record.inputOrderLineage
}
dropUnusableProducerLinkage(record)
dropUnusableTurnScope(record)
dropUnusableOwnerFence(record)
@@ -19,6 +19,8 @@ export function encodeJournalState(state: JournalReducerState): {
tombstones: [...state.tombstones],
removedTurnIds: [...state.removedTurnIds],
submissions: [...state.submissions],
inputEntries: [...state.inputEntries],
inputCards: [...state.inputCards],
receipts: [...state.receipts],
aliases: [...state.aliases],
itemEchoes: [...state.itemEchoes].map(([key, value]) =>
@@ -15,6 +15,8 @@ export const journalStateLayout = {
tombstones: 'map',
removedTurnIds: 'set',
submissions: 'map',
inputEntries: 'map',
inputCards: 'map',
receipts: 'map',
aliases: 'map',
itemEchoes: 'map',
@@ -1,6 +1,11 @@
import type { z } from 'zod'
import { z } from 'zod'
import type { JournalReducerState } from './journal-reducer'
import { outcomeFormat1Fields } from './journal-outcome-format-1'
import { inputEntrySchema, inputCardSchema } from './journal-outcome-format-2'
// Every reducer key needs an explicit stored representation in the format this build writes.
export const fields = outcomeFormat1Fields satisfies Record<keyof JournalReducerState, z.ZodType>
export const fields = {
...outcomeFormat1Fields,
inputEntries: z.array(z.tuple([z.string().min(1), inputEntrySchema])),
inputCards: z.array(z.tuple([z.string().min(1), inputCardSchema]))
} satisfies Record<keyof JournalReducerState, z.ZodType>
@@ -1,3 +1,4 @@
import { admitOutcomeFormat2 } from './journal-outcome-format-2'
import { readFileSync } from 'node:fs'
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
@@ -193,7 +194,10 @@ const prefix = z
)
function plain(state: JournalReducerState): unknown {
const fields: Record<string, unknown> = { ...state }
const { inputEntries, inputCards, ...ordinary } = state
expect([...inputEntries]).toEqual([])
expect([...inputCards]).toEqual([['card-2', { introduced: 11 }]])
const fields: Record<string, unknown> = ordinary
return JSON.parse(
JSON.stringify({
...Object.fromEntries(
@@ -223,7 +227,9 @@ it('folds the checked-in input rows into the same chat, every value of which sto
}
expect(plain(loaded.state)).toEqual(prefix.expected)
const collections = journalStateCollections(loaded.state)
expect(collections.filter(({ entries }) => [...entries].length === 0)).toEqual([])
expect(
collections.filter(({ entries }) => [...entries].length === 0).map(({ path }) => path)
).toEqual(['inputEntries'])
expect(collections.length).toBeGreaterThanOrEqual(15)
// What restating this chat on a new epoch would store, admitted as this build stores it.
const stored = [
@@ -247,7 +253,13 @@ it('folds the checked-in input rows into the same chat, every value of which sto
]
expect(stored.length).toBeGreaterThan(20)
for (const value of stored) {
expect(admitOutcomeFormat1({ ...value, deleted: false })).toEqual({
const native = value.field === 'inputCards' || value.field === 'inputEntries'
if (native) {
expect(admitOutcomeFormat1({ ...value, deleted: false })).toBe('unknown')
}
expect(
(native ? admitOutcomeFormat2 : admitOutcomeFormat1)({ ...value, deleted: false })
).toEqual({
entryKey: JSON.parse(value.key),
kept: JSON.parse(value.json)
})
@@ -275,3 +287,15 @@ it('reads an open string such as an agent type as a value, and still refuses a k
// @ts-expect-error `host` is a key the stored provider handle does not declare
expect(declared<UndeclaredStorageKeys<Later, Stored>>()).toBeUndefined()
})
it('keeps explicitly opaque annotation values whole without hiding sibling or nested structured keys', () => {
type Actual = { annotation: { future: { keys: number[] } }; nested: { kept: string } }
type Stored = { annotation?: unknown; nested: { kept: string } }
expect(declared<UndeclaredStorageKeys<Actual, Stored>>()).toBeUndefined()
// @ts-expect-error an opaque annotation does not declare a sibling key
expect(declared<UndeclaredStorageKeys<Actual & { extra: string }, Stored>>()).toBeUndefined()
expect(
// @ts-expect-error structured siblings still recursively declare their nested keys
declared<UndeclaredStorageKeys<Actual & { nested: { kept: string; extra: number } }, Stored>>()
).toBeUndefined()
})
@@ -69,6 +69,7 @@ const storedRow = z.discriminatedUnion('kind', [
z.object({
...envelope,
kind: z.literal('epoch'),
inputOrderLineage: value,
reason: value,
providerHandle: storedProviderHandle,
historyOrigin: value
@@ -98,6 +99,7 @@ const storedRow = z.discriminatedUnion('kind', [
z.object({
...envelope,
kind: z.literal('submission'),
inputOrderContext: value,
clientMessageId: value,
payloadFingerprint: value,
providerHandle: storedProviderHandle,
@@ -115,17 +115,19 @@ type ArmKeys<T, A> = [A] extends [never]
* output `S` leaves undeclared; `never` when it declares them all. An unknown-kind value never
* reaches storage: admission refuses it. A primitive holds no keys; it is settled before `object`
* because an open or branded string (`AgentType`'s `string & {}`) is assignable to `object`. */
export type UndeclaredStorageKeys<T, S> = T extends { readonly kind: AgentJournalUnknownKind }
export type UndeclaredStorageKeys<T, S> = unknown extends S
? never
: T extends string | number | boolean | bigint | symbol | null | undefined
: T extends { readonly kind: AgentJournalUnknownKind }
? never
: T extends readonly (infer E)[]
? [Extract<S, readonly unknown[]>] extends [never]
? 'an array storage does not declare'
: UndeclaredStorageKeys<E, Extract<S, readonly unknown[]>[number]>
: T extends object
? ArmKeys<T, Arm<T, S>>
: never
: T extends string | number | boolean | bigint | symbol | null | undefined
? never
: T extends readonly (infer E)[]
? [Extract<S, readonly unknown[]>] extends [never]
? 'an array storage does not declare'
: UndeclaredStorageKeys<E, Extract<S, readonly unknown[]>[number]>
: T extends object
? ArmKeys<T, Arm<T, S>>
: never
/** Compiles only when every persisted key is declared. */
export type StorageDeclaresAll<Undeclared extends never> = Undeclared
@@ -139,6 +139,7 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal
rowWriter,
submissionWriter: new JournalSubmissionWriter({
state: host.state,
reopenFloor: () => host.journal().reopenFloor(),
identity: host.identity,
rowWriter,
queuedMessages,
@@ -1,8 +1,10 @@
import { journalInputOrder } from './journal-input-order'
// Each session owns its fold and queue; all statements use the host's shared database connection.
import { JournalWriteProjection } from './journal-write-projection'
import type { AgentJournalDispatchRejection } from '../../../shared/agent-session-failure-words'
import { randomUUID } from 'node:crypto'
import { retireCoveredJournalFloor } from './journal-commit'
import { createJournalPublicReader } from './journal-public-read'
import type {
AgentJournalAcceptanceReceipt,
@@ -12,7 +14,6 @@ import type {
AgentJournalRenderItem,
AgentJournalSnapshot,
AgentJournalSubmission,
AgentJournalThreadGoal,
AgentJournalTurnLifecycle,
AgentSessionJournalIdentity
} from '../../../shared/agent-session-journal-types'
@@ -42,7 +43,6 @@ import type {
JournalItemLinkageVisitor,
JournalLifecycleBatchInput,
JournalPlannedLifecycleBatchInput,
JournalReadSince,
JournalTombstoneInput
} from './journal-store-contracts'
import {
@@ -96,6 +96,7 @@ export class AgentSessionJournal {
state: () => this.state,
adopt: (state) => {
this.state = state
this.reopenUnmarked = retireCoveredJournalFloor(state, this.reopenUnmarked)
}
})
const collaborators = createJournalStoreCollaborators({
@@ -108,10 +109,14 @@ export class AgentSessionJournal {
readOnly: () => this.database.readOnly,
cursor: this.cursor,
adopt: (loaded) => {
journalInputOrder(loaded.state).carryRecovery(journalInputOrder(this.state))
this.state = loaded.state
this.onCommitted?.()
},
prepare: (db, rows, savedAt) => this.projection.prepare(db, rows, savedAt),
prepare: (db, rows, savedAt) => {
this.queuedMessages.inputState()
return this.projection.prepare(db, rows, savedAt)
},
commit: (rows) => {
this.onCommitted?.(rows)
this.projection.verify()
@@ -142,6 +147,10 @@ export class AgentSessionJournal {
return this.identity.agent
}
get inputOrder() {
return journalInputOrder(this.state)
}
/** Only rows from this handle's opening epoch can predate the open. */
wroteBeforeOpen(sequence: number | undefined): boolean {
return journalWroteBeforeOpen(this.state, this.openedThrough, sequence)
@@ -155,6 +164,7 @@ export class AgentSessionJournal {
async open(): Promise<void> {
await this.restore()
this.openedThrough = this.cursor()
journalInputOrder(this.state).markOpeningRecovery()
}
/** Refuse future writes and drain admitted ones; the host retains database ownership. */
@@ -224,7 +234,7 @@ export class AgentSessionJournal {
journalThreadGoalSource(this.state.items.values())
/** Read across the journal, beyond any client's loaded page. */
threadGoal = (): AgentJournalThreadGoal | null => this.threadGoalSource().goal
threadGoal = () => this.threadGoalSource().goal
/** Read across the journal, beyond any client's loaded page. */
contextUsage = (): AgentSessionContextUsage =>
@@ -261,7 +271,7 @@ export class AgentSessionJournal {
return this.queue.readInOrder(read)
}
readSince(cursor: AgentJournalCursor, limit?: number): JournalReadSince {
readSince(cursor: AgentJournalCursor, limit?: number) {
const { sessionId } = this.identity
const rowsAfter = journalRowsAfterReader(this.database.db, sessionId, this.state.epoch, limit)
return readJournalSince({ state: this.state, rowsAfter }, cursor, () => this.cursor())
@@ -325,7 +335,6 @@ export class AgentSessionJournal {
const sequence = since ?? this.state.lastSequence + 1
this.reopenUnmarked = { epoch: this.state.epoch, sequence }
await this.appendQueueReopen(fence, since)
this.reopenUnmarked = null
}
}
@@ -354,10 +363,7 @@ export class AgentSessionJournal {
}
/** Failed initialization proves handed-over but unanswered sends never ran. */
async rejectPendingSubmissions(
fence: number,
rejection: AgentJournalDispatchRejection
): Promise<string[]> {
async rejectPendingSubmissions(fence: number, rejection: AgentJournalDispatchRejection) {
return rejectJournalPendingSubmissions(this, fence, rejection)
}
@@ -372,9 +378,7 @@ export class AgentSessionJournal {
return this.epochController.roll(reason, fence)
}
replaceEpochItems(
...args: Parameters<typeof this.epochController.replace>
): Promise<AgentJournalCursor> {
replaceEpochItems(...args: Parameters<typeof this.epochController.replace>) {
return this.epochController.replace(...args)
}
}
@@ -0,0 +1,37 @@
import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types'
/** Reverse links derive from submission rows and share the reducer lifetime. */
export class JournalSubmissionLinks {
private readonly linked = new Map<string, Map<string, AgentJournalSubmission>>()
private readonly ids = new Map<string, string>()
private revision = 0
version(): number {
return this.revision
}
to(id: string): Iterable<AgentJournalSubmission> {
return this.linked.get(id)?.values() ?? []
}
refresh(id: string, entry: AgentJournalSubmission | undefined): void {
const priorLink = this.ids.get(id)
const link = entry?.queuedMessageId
if (priorLink !== link && priorLink) {
const prior = this.linked.get(priorLink)!
prior.delete(id)
if (prior.size === 0) {
this.linked.delete(priorLink)
}
this.ids.delete(id)
this.revision++
}
if (link && entry) {
const entries = this.linked.get(link) ?? new Map<string, AgentJournalSubmission>()
if (entries.get(id) !== entry) {
entries.set(id, entry)
this.linked.set(link, entries)
this.ids.set(id, link)
}
this.revision++
}
}
}
@@ -3,6 +3,7 @@ import type {
AgentSessionJournalIdentity
} from '../../../shared/agent-session-journal-types'
import { journalDispatchRowApplies } from './journal-dispatch-settlement'
import { journalQueueReopenRowBuilder } from './journal-stop-and-resume-rows'
import type { JournalQueuedMessages, JournalQueuePlan } from './journal-queued-messages'
import type { JournalAttachmentClaim } from './journal-submission-hook'
import type { JournalReducerState } from './journal-reducer'
@@ -17,6 +18,7 @@ import type {
export type JournalSubmissionWriterDeps = {
state: () => JournalReducerState
reopenFloor: () => AgentJournalCursor | null
identity: AgentSessionJournalIdentity
rowWriter: JournalRowWriter
queuedMessages: JournalQueuedMessages
@@ -41,8 +43,13 @@ export class JournalSubmissionWriter {
return rowWriter
.enqueueRows(
() => {
const floor = this.deps.reopenFloor()
const reopen =
floor?.epoch === state().epoch
? [journalQueueReopenRowBuilder(state, input.fence, floor.sequence)]
: []
if (!consume) {
return [submission]
return [...reopen, submission]
}
queuedMessages.assertConsumable({
...consume,
@@ -51,7 +58,7 @@ export class JournalSubmissionWriter {
})
// Adopt legacy cards in the handoff transaction.
card = queuedMessages.plan([{ messageId: consume.messageId }])
return [...card.rows, submission]
return [...reopen, ...card.rows, submission]
},
{
write: (db, row) => {
@@ -1,5 +1,5 @@
import type { createFixture } from './journal-unsent-send-hold-contract-test-fixture'
import { describe, expect, it } from 'vitest'
import { describe, expect, it, vi } from 'vitest'
import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types'
import { USER_MESSAGE_SOURCE } from '../../../shared/agent-session/agent-session-message-source'
@@ -51,32 +51,28 @@ export function registerJournalUnsentSendHoldQueueTests(
await fixture.accept(earlier, 'A', { origin: 'client', source: USER_MESSAGE_SOURCE })
await fixture.accept(earlier, 'B', { origin: 'client', source: USER_MESSAGE_SOURCE })
})
// Simulate a crash after keeping A at a stale position but before settling B.
await interrupted.resolveDispatch(
{
clientMessageId: 'A',
state: 'rejected',
...fixture.HOST_RESTARTED,
fence: 0,
recovered: true,
const resolve = interrupted.resolveDispatch.bind(interrupted)
const fault = vi
.spyOn(interrupted, 'resolveDispatch')
.mockImplementation((...args) =>
args[0].clientMessageId === 'B'
? Promise.reject(new Error('keep and fallback unavailable'))
: resolve(...args)
)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
try {
await expect(fixture.hold(interrupted)).rejects.toThrow('settling unsent sends failed')
expect(interrupted.submission('A')).toMatchObject({
dispatchState: 'rejected',
keptAsQueuedMessageId: 'A'
},
() =>
interrupted.queuedMessages.plan([
{
messageId: 'A',
change: {
op: 'queued',
body: fixture.message('text of A'),
queuedAt: {
epoch: 'epoch-1',
sequence: interrupted.submission('A')!.acceptedSequence!
},
position: 5
}
}
])
)
})
expect(interrupted.submission('B')?.dispatchState).toBe('pending')
expect(interrupted.queuedMessages.get('A')?.position).toBe(0)
expect(interrupted.queuedMessages.get('C')?.position).toBe(2)
} finally {
fault.mockRestore()
warn.mockRestore()
}
await interrupted.close()
const journal = await fixture.open()
await fixture.hold(journal)
@@ -76,7 +76,7 @@ export async function holdUnsentSends(
? isUnansweredHandedOverSubmission(entry) && (input.unrun === true || input.unrun(entry))
: isQueuedAgentJournalSubmission(entry) &&
(hold.cause === 'hostRestarted'
? journal.wroteBeforeOpen(entry.acceptedSequence)
? journal.inputOrder.recoveryPending(entry.clientMessageId)
: (hold.which?.(entry) ?? true))
)
.sort((a, b) => (a.acceptedSequence ?? 0) - (b.acceptedSequence ?? 0))
@@ -91,18 +91,16 @@ export async function holdUnsentSends(
journal.itemBody(agentJournalSubmissionKey(submission.clientMessageId))
)
}))
const positions = headOfQueuePositions(journal, kept)
const positions = headOfQueuePositions(journal, kept, Boolean(input.unrun))
const rejection = agentSessionFailureWords(agentSessionFailureFact(hold.cause), {
surface: 'rejection'
})
const failures: unknown[] = []
for (const [index, { submission, body }] of kept.entries()) {
let ranksCommitted = false
for (const { submission, body } of kept) {
const { clientMessageId } = submission
const moves = [
...(index === 0 ? positions.earlier : []),
...positions.placed.filter((entry) => entry.messageId === submission.queuedMessageId)
]
const position = positions.placed.find((entry) => entry.messageId === clientMessageId)?.position
const moves = ranksCommitted ? [] : positions.moves
const position = positions.placed.get(clientMessageId)
const card = body && position !== undefined ? { body, position } : null
const keep: (() => JournalQueuePlan) | undefined =
card || moves.length > 0
@@ -119,7 +117,7 @@ export async function holdUnsentSends(
]
)
if (card && !queuedMessages.get(clientMessageId)) {
const queuedAt = { epoch, sequence: submission.acceptedSequence ?? 0 }
const queuedAt = { epoch, sequence: submission.submittedSequence! }
const change = {
op: 'queued' as const,
body: card.body,
@@ -145,6 +143,9 @@ export async function holdUnsentSends(
card ? { ...reject, keptAsQueuedMessageId: clientMessageId } : reject,
keep
)
if (keep) {
ranksCommitted = true
}
} catch (error) {
if (!keep) {
failures.push(error)
@@ -166,45 +167,105 @@ export async function holdUnsentSends(
return unsent.at(-1)?.acceptedSequence ?? 0
}
/** Order earlier kept cards first, then admissions; queuedAt cannot compare across epochs or handoff origins. */
/** Merge logical places once, then materialize finite ranks with the first successful keep. */
function headOfQueuePositions(
journal: AgentSessionJournal,
batch: readonly { submission: AgentJournalSubmission; body: AgentJournalMessageItem | null }[]
): { placed: QueuedMessagePositionMove[]; earlier: QueuedMessagePositionMove[] } {
batch: readonly { submission: AgentJournalSubmission; body: AgentJournalMessageItem | null }[],
unrun: boolean
): { placed: Map<string, number>; moves: QueuedMessagePositionMove[] } {
const cards = journal.queuedMessages.list()
const keptIds = new Set(
journal.submissions().flatMap((entry) => entry.keptAsQueuedMessageId ?? [])
)
const { epoch } = journal.cursor()
// Earlier settlements survive either through explicit links or cards from earlier epochs.
const earlier = cards
.filter(
(card) =>
card.state === 'waiting' &&
(keptIds.has(card.messageId) || (card.queuedAt !== null && card.queuedAt.epoch !== epoch))
)
.map((card) => card.messageId)
const placed: QueuedMessagePositionMove[] = []
for (const { submission, body } of batch) {
const { clientMessageId } = submission
if (body && !earlier.includes(clientMessageId)) {
placed.push({ messageId: clientMessageId, position: 0 })
} else if (
// Explicit Send handoffs return to the head; automatic queue handoffs stay put.
submission.origin === 'client' &&
submission.queuedMessageId !== undefined &&
cards.some((card) => card.consumedAs === clientMessageId)
const state = journal.queuedMessages.displayState()
const forcedBatch = batch.some(({ submission }) =>
journal.inputOrder.forced(submission.clientMessageId)
)
const earlier: string[] = []
const rest: (typeof cards)[number][] = []
const cardsById = new Map(cards.map((card) => [card.messageId, card]))
const explicitHeads = new Map(
batch.flatMap(({ submission }) => {
const id = submission.origin === 'client' ? submission.queuedMessageId : undefined
return id && cardsById.get(id)?.consumedAs === submission.clientMessageId
? [[submission.clientMessageId, id] as const]
: []
})
)
const explicit = new Set(explicitHeads.values())
for (const card of cards) {
if (explicit.has(card.messageId)) {
continue
}
if (
card.state === 'waiting' &&
((unrun && keptIds.has(card.messageId)) ||
(!forcedBatch && card.queuedAt && card.queuedAt.epoch !== epoch))
) {
placed.push({ messageId: submission.queuedMessageId, position: 0 })
earlier.push(card.messageId)
} else {
rest.push(card)
}
}
const inHead = (card: (typeof cards)[number]): boolean =>
earlier.includes(card.messageId) || placed.some((move) => move.messageId === card.messageId)
const others = cards.filter((card) => !inHead(card)).map((card) => card.position)
const anchor = others.length > 0 ? Math.min(...others) : 1
const first = anchor - earlier.length - placed.length
const at = new Map<number, string[]>()
const head: string[] = []
const firstCut = batch
.flatMap(
({ submission }) => journal.inputOrder.place(submission.clientMessageId)?.context.cut ?? []
)
.reduce((a, b) => Math.min(a, b), Infinity)
let previousBoundary = 0
for (let i = 0; i < rest.length; i++) {
if (
rest[i].state === 'waiting' &&
keptIds.has(rest[i].messageId) &&
(journal.inputOrder.lineage.cards.get(rest[i].messageId)?.introduced ?? -Infinity) < firstCut
) {
previousBoundary = i + 1
}
}
for (const { submission, body } of batch) {
const id = body ? submission.clientMessageId : explicitHeads.get(submission.clientMessageId)
if (!id || keptIds.has(id)) {
continue
}
if (!body && previousBoundary > 0) {
head.push(id)
continue
}
const after = journal.inputOrder.forced(submission.clientMessageId)
? journal.inputOrder.predecessors(state, submission.clientMessageId).position
: -Infinity
let low = 0,
high = rest.length
while (low < high) {
const mid = (low + high) >>> 1
if (rest[mid].position <= after) {
low = mid + 1
} else {
high = mid
}
}
const boundary = Math.max(previousBoundary, low)
previousBoundary = boundary
const bucket = at.get(boundary) ?? []
bucket.push(id)
at.set(boundary, bucket)
}
const order = [...earlier, ...head]
for (let i = 0; i <= rest.length; i++) {
order.push(...(at.get(i) ?? []))
if (i < rest.length) {
order.push(rest[i].messageId)
}
}
const placed = new Map(order.map((id, index) => [id, index]))
return {
earlier: earlier.map((messageId, index) => ({ messageId, position: first + index })),
placed: placed.map((move, index) => ({ ...move, position: first + earlier.length + index }))
placed,
moves: cards.flatMap((card) => {
const position = placed.get(card.messageId)!
return position === card.position ? [] : [{ messageId: card.messageId, position }]
})
}
}
@@ -61,6 +61,17 @@ describe('queued edit leases', () => {
expect(leases.heldIds().size).toBe(0)
})
it('reuses an edit key only after its previous target becomes invalid', () => {
rows.push(card('second'))
expect(leases.acquire(desktop, base)).toMatchObject({ status: 'held' })
expect(leases.acquire(desktop, { ...base, messageId: 'second' })).toEqual({ status: 'changed' })
rows[0]!.fingerprint = 'newer'
expect(leases.acquire(desktop, { ...base, messageId: 'second' })).toMatchObject({
status: 'held'
})
expect([...leases.heldIds()]).toEqual(['second'])
})
it('expires on its own and wakes the queue with no other activity', () => {
leases.acquire(desktop, base)
advance(QUEUED_MESSAGE_EDIT_LEASE_MS - 1)
@@ -14,14 +14,18 @@ type EditLease = Pick<QueuedMessageRow, 'messageId' | 'fingerprint' | 'state' |
/** Closing the journal handle drops its leases. */
export class QueuedMessageEditLeases {
private readonly leases = new Map<string, EditLease>()
private readonly byId = new Map<string, Set<string>>()
private timer: ReturnType<typeof setTimeout> | undefined
private timerAt: number | null = null
private wake: (() => void) | undefined
private disposed = false
private revision = 0
constructor(
private readonly rows: () => readonly QueuedMessageRow[],
private readonly now: () => number = () => performance.now()
private readonly now: () => number = () => performance.now(),
private readonly get: (id: string) => QueuedMessageRow | null | undefined = (id) =>
this.rows().find((row) => row.messageId === id)
) {}
/** Wake the queue even when no journal activity follows expiry. */
@@ -29,27 +33,70 @@ export class QueuedMessageEditLeases {
this.wake = wake
}
/** Transaction callers supply their own card snapshot. */
heldIds(rows: readonly QueuedMessageRow[] = this.rows()): ReadonlySet<string> {
if (this.leases.size === 0) {
return new Set()
version(): number {
this.expire()
return this.revision
}
isHeld(messageId: string): boolean {
this.expire()
if (!this.byId.has(messageId)) {
return false
}
const byId = new Map(rows.map((row) => [row.messageId, row]))
const now = this.now()
let lapsed = false
const revision = this.revision
const row = this.get(messageId)
for (const key of this.byId.get(messageId) ?? []) {
const lease = this.leases.get(key)!
if (!row || !leaseMatchesRow(lease, row)) {
this.remove(key)
}
}
if (this.revision !== revision) {
this.schedule()
}
return this.byId.has(messageId)
}
/** Output enumeration revalidates only leased identities; selection uses exact-ID reads. */
heldIds(rows?: readonly QueuedMessageRow[]): ReadonlySet<string> {
this.expire()
const snapshot = rows && new Map(rows.map((row) => [row.messageId, row]))
for (const [key, lease] of this.leases) {
const row = byId.get(lease.messageId)
lapsed ||= lease.until <= now
if (lease.until <= now || !row || !leaseMatchesRow(lease, row)) {
this.leases.delete(key)
const row = snapshot ? snapshot.get(lease.messageId) : this.get(lease.messageId)
if (!row || !leaseMatchesRow(lease, row)) {
this.remove(key)
}
}
this.schedule()
// A pruned deadline cancels its timer; wake in that timer's place.
if (lapsed) {
this.wake?.()
return new Set(this.byId.keys())
}
private remove(key: string): void {
const lease = this.leases.get(key)
if (!lease) {
return
}
return new Set([...this.leases.values()].map((lease) => lease.messageId))
this.leases.delete(key)
const keys = this.byId.get(lease.messageId)!
keys.delete(key)
if (keys.size === 0) {
this.byId.delete(lease.messageId)
}
this.revision++
}
private expire(): void {
if (this.timerAt === null || this.timerAt > this.now()) {
return
}
const now = this.now()
for (const [key, lease] of this.leases) {
if (lease.until <= now) {
this.remove(key)
}
}
this.schedule()
this.wake?.()
}
/** Repeated acquisition returns the same lease without extending it. */
@@ -57,14 +104,15 @@ export class QueuedMessageEditLeases {
key: EditLeaseKey,
input: { messageId: string; fingerprint: string }
): AgentSessionQueuedMessageEditHoldResult {
this.heldIds()
this.expire()
this.isHeld(this.leases.get(leaseKey(key))?.messageId ?? input.messageId)
const existing = this.leases.get(leaseKey(key))
if (existing) {
return existing.messageId === input.messageId && existing.fingerprint === input.fingerprint
? this.answer(existing)
: { status: 'changed' }
}
const row = this.rows().find((entry) => entry.messageId === input.messageId)
const row = this.get(input.messageId)
if (this.disposed || !row || !isUnsettledQueuedMessage(row)) {
return { status: 'gone' }
}
@@ -79,13 +127,17 @@ export class QueuedMessageEditLeases {
until: this.now() + QUEUED_MESSAGE_EDIT_LEASE_MS
}
this.leases.set(leaseKey(key), lease)
const keys = this.byId.get(lease.messageId) ?? new Set<string>()
keys.add(leaseKey(key))
this.byId.set(lease.messageId, keys)
this.revision++
this.schedule()
return this.answer(lease)
}
/** Expired/invalidated editors must acquire a new lease. */
renew(key: EditLeaseKey, messageId: string): AgentSessionQueuedMessageEditHoldResult {
this.heldIds()
this.isHeld(messageId)
const lease = this.leases.get(leaseKey(key))
if (!lease || lease.messageId !== messageId) {
return { status: 'expired' }
@@ -98,16 +150,14 @@ export class QueuedMessageEditLeases {
/** Release only the exact caller/edit pair. */
release(key: EditLeaseKey, messageId: string): void {
if (this.leases.get(leaseKey(key))?.messageId === messageId) {
this.leases.delete(leaseKey(key))
this.remove(leaseKey(key))
this.schedule()
}
}
retire(messageId: string): void {
for (const [key, lease] of this.leases) {
if (lease.messageId === messageId) {
this.leases.delete(key)
}
for (const key of this.byId.get(messageId) ?? []) {
this.remove(key)
}
this.schedule()
}
@@ -115,6 +165,8 @@ export class QueuedMessageEditLeases {
dispose(): void {
this.disposed = true
this.leases.clear()
this.byId.clear()
this.revision++
this.schedule()
}
@@ -146,7 +198,7 @@ export class QueuedMessageEditLeases {
const now = this.now()
for (const [key, lease] of this.leases) {
if (lease.until <= now) {
this.leases.delete(key)
this.remove(key)
}
}
this.schedule()
@@ -319,7 +319,8 @@ export class StructuredAgentSessionClientDelivery {
submissions: () =>
journal.submissions().map(this.currentWork(sessionId, journal).publishedSubmission),
cursor: () => journal.cursor(),
activeTurnId: () => this.currentWork(sessionId, journal).activeTurnId()
activeTurnId: () => this.currentWork(sessionId, journal).activeTurnId(),
behindCommand: (id: string) => journal.inputOrder.forced(id)
}
}
@@ -1,3 +1,5 @@
import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope'
import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key'
import * as autoSandbox from '../../codex/codex-structured-auto-sandbox'
import { expect, it, vi } from 'vitest'
import { codexInputFinalizationFixture } from './structured-agent-session-codex-input-host.test'
@@ -8,7 +10,8 @@ import {
import {
CODEX_INPUT_FINALIZATION_ADMISSION_CASES,
HOST_TEST_SESSION as SESSION,
hostTestMessage
hostTestMessage,
hostTestOperationId
} from './structured-agent-session-host-test-data'
import { agentSessionFailureFact } from '../../../shared/agent-session-failure'
import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words'
@@ -235,12 +238,12 @@ it.each(CODEX_INPUT_FINALIZATION_ADMISSION_CASES)(
}
)
it.each(['stop', 'exit', 'exit-after-close-record'] as const)(
it.each(['stop', 'stop-save-refusal', 'exit', 'exit-after-close-record'] as const)(
'unwinds actual Auto preparation on %s without a late write',
async (ending) => {
const { host, adapter, codex, config, pick, permissionSend, stop, starts } =
codexInputFinalizationFixture()
const exiting = ending !== 'stop'
const exiting = ending.startsWith('exit')
Object.defineProperty(process, 'platform', { configurable: true, value: 'linux' })
await pick('auto')
const paused = Promise.withResolvers<unknown>()
@@ -320,7 +323,33 @@ it.each(['stop', 'exit', 'exit-after-close-record'] as const)(
expect(child.close).toMatchObject({ cause: 'evict' })
expect(signal?.aborted).toBe(false)
}
if (ending === 'stop') {
if (ending === 'stop-save-refusal') {
const db = codexInputFinalizationFixture().database()
const receipt = journal.receiptFor(id)
db.exec(`CREATE TRIGGER fail_stop_receipt BEFORE INSERT ON agent_session_command_receipts
BEGIN SELECT RAISE(ABORT, 'receipt unavailable'); END`)
try {
expect(await stop()).toMatchObject({
ok: false,
refusal: { details: { reason: 'stopFailed', agent: 'codex' } }
})
} finally {
db.exec('DROP TRIGGER fail_stop_receipt')
}
expect(journal.receiptFor(id)).toEqual(receipt)
expect(journal.queuedMessages.pauses()).toEqual([])
expect(
journal
.snapshot()
.items.some(
(item) =>
item.body.kind === 'message' &&
item.body.blocks.some(
(block) => block.type === 'text' && block.text === 'Stopped manually'
)
)
).toBe(false)
} else if (ending === 'stop') {
await stop()
} else {
codex.connections[0]?.handlers.onExit?.(
@@ -348,6 +377,23 @@ it.each(['stop', 'exit', 'exit-after-close-record'] as const)(
expect(starts()).toHaveLength(0)
const writes = await Promise.allSettled(written.mock.results.map((result) => result.value))
expect(writes.filter((result) => result.status === 'rejected')).toEqual([])
if (ending === 'stop-save-refusal') {
const next = await permissionSend('next after refused Stop')
await vi.waitFor(() => expect(starts()).toHaveLength(1))
expect(starts()[0]?.params).toMatchObject({
clientUserMessageId: next,
input: [{ type: 'text', text: 'next after refused Stop' }]
})
const fixture = codexInputFinalizationFixture()
fixture.turns.start()
fixture.turns.echo(next)
expect(
fixture.submission((await host.waitForSendSettlement(SESSION, next))?.value)
).toMatchObject({ clientMessageId: next, dispatchState: 'accepted' })
await fixture.waitForInputFinalization(journal, next)
expect(journal.queuedMessages.list()).toEqual([])
expect(starts()).toHaveLength(1)
}
} finally {
paused.resolve(config)
await Promise.allSettled(lateReads)
@@ -355,3 +401,147 @@ it.each(['stop', 'exit', 'exit-after-close-record'] as const)(
}
}
)
it('keeps rewind outside a committed preparation until the input settles', async () => {
const { host, adapter, permissionSend, turns, starts, rewindTo } = codexInputFinalizationFixture()
const { sessions, serialize, conversationDelivery } = host.collaboratorsForTests()
const journal = sessions.get(SESSION)!.journal
const prepared = Promise.withResolvers<void>()
const release = Promise.withResolvers<void>()
const dispatch = adapter.dispatch.bind(adapter)
vi.spyOn(adapter, 'dispatch').mockImplementation(async (input) => {
prepared.resolve()
await release.promise
return dispatch(input)
})
const providerRewind = vi.spyOn(adapter, 'rewind')
let rewinding: ReturnType<typeof rewindTo> | undefined
try {
const id = await permissionSend('committed preparation')
await prepared.promise
expect(journal.submission(id)?.handedOverAt).toBeDefined()
const epoch = journal.epoch
const reserved = Promise.withResolvers<void>()
const order = conversationDelivery.loop['deps'].deliveryOrder
const reserve = order.reserve.bind(order)
vi.spyOn(order, 'reserve').mockImplementation((sessionId) => {
reserved.resolve()
return reserve(sessionId)
})
rewinding = rewindTo(agentJournalSubmissionKey(id), epoch)
await reserved.promise
await serialize(SESSION, async () => {})
expect(providerRewind).not.toHaveBeenCalled()
expect(journal.epoch).toBe(epoch)
expect(starts()).toHaveLength(0)
release.resolve()
await vi.waitFor(() => expect(starts()).toHaveLength(1))
turns.start()
turns.echo(id)
await codexInputFinalizationFixture().waitForInputFinalization(journal, id)
expect(await rewinding).toMatchObject({
ok: false,
refusal: { details: { rewindReason: 'busy' } }
})
expect(providerRewind).not.toHaveBeenCalled()
expect(journal.epoch).toBe(epoch)
} finally {
release.resolve()
await rewinding
}
})
it('keeps earlier B before historically forced proven-unrun A through close and Resume delivery', async () => {
const {
host,
adapter,
codex,
config,
permissionSend,
began,
turns,
starts,
pick,
waitForInputFinalization
} = codexInputFinalizationFixture()
const { sessions, serialize } = host.collaboratorsForTests()
const journal = sessions.get(SESSION)!.journal
const w = await permissionSend('working W')
await began(w)
await serialize(SESSION, () =>
journal.queuedMessages.insert({
messageId: 'force-C',
body: { ...hostTestMessage('/compact'), command: { name: 'compact' } }
})
)
const a = await permissionSend('forced A', { personality: 'friendly' })
expect(journal.inputOrder.forced(a)).toBe(true)
await serialize(SESSION, () => journal.queuedMessages.withdraw(['force-C']))
await pick('auto')
const preparation = Promise.withResolvers<unknown>()
const read = vi.fn(() => preparation.promise)
codex.routes['config/read'] = read
try {
turns.end('completed')
await vi.waitFor(() => expect(read).toHaveBeenCalledOnce())
expect(journal.submission(a)).toMatchObject({
dispatchState: 'pending',
handedOverAt: expect.any(Number)
})
const b = await permissionSend('ordinary B', { personality: 'serious' })
expect(journal.inputOrder.forced(b)).toBe(false)
await host.close(SESSION, 'user-close')
await waitForInputFinalization(journal, a)
expect(starts()).toHaveLength(1)
const kept = journal.queuedMessages.list().filter((c) => c.state === 'waiting')
expect.soft(kept.map((c) => c.messageId)).toEqual([b, a])
for (const [id, personality] of [
[a, 'friendly'],
[b, 'serious']
] as const) {
expect(kept.find((c) => c.messageId === id)?.options).toEqual(journal.submission(id)?.options)
expect(kept.find((c) => c.messageId === id)?.options).toMatchObject({ personality })
expect(journal.submission(id)).toMatchObject({
dispatchState: 'rejected',
keptAsQueuedMessageId: id
})
}
preparation.resolve(config)
codex.routes['config/read'] = () => config
const caller = { callerKey: 'client-1' }
await host.revealSession(SESSION)
const method = 'agentSession.queuedMessagesResume'
expect(
await host.queuedMessagesResume(caller, {
envelope: {
sessionId: SESSION,
clientOperationId: hostTestOperationId(),
expectedRuntimeFence: null,
payloadFingerprint: computeAgentSessionPayloadFingerprint({
method,
sessionId: SESSION,
fields: {}
})
}
})
).toMatchObject({ ok: true })
const reopened = sessions.get(SESSION)!.journal
const delivered: string[] = []
for (const { messageId: id } of kept) {
await vi.waitFor(() => expect(reopened.queuedMessages.get(id)?.consumedAs).toBeTruthy())
const h = reopened.queuedMessages.get(id)!.consumedAs!
await began(h)
expect(reopened.submission(h)?.dispatchState).toBe('accepted')
delivered.push(reopened.submission(h)!.queuedMessageId!)
turns.end('completed')
await host.flushStreamedEvents(SESSION)
}
expect.soft(delivered).toEqual([b, a])
expect(starts()).toHaveLength(3)
expect(reopened.queuedMessages.list().filter((c) => c.state === 'waiting')).toEqual([])
expect(reopened.queuedMessages.displayState().inputEntries.size).toBe(0)
} finally {
preparation.resolve(config)
await adapter.closeAll()
}
})
@@ -619,24 +619,46 @@ export function registerCodexInputFinalization(current: typeof codexInputFinaliz
results.map((result) => (result.type === 'return' ? result.value : undefined))
)
}
const stoppedX = 'stopped-before-carrier'
if (boundary === 'retire') {
await host.collaboratorsForTests().serialize(SESSION, async () => {
await journal.appendSubmission({
clientMessageId: stoppedX,
payloadFingerprint: stoppedX,
body: hostTestMessage('earlier stopped X'),
source: { kind: 'user' },
origin: 'client',
handoverRecorded: true,
fence: original.fence
})
await journal.stops.accept({
event: { reason: 'user-stop' },
fence: original.fence,
liveTurnId: () => null
})
})
expect(journal.submission(stoppedX)?.rejection).toMatchObject({ neverReachedAgent: true })
}
const id = await permissionSend('same original input', { personality: 'friendly' })
const waiting = host.waitForSendSettlement(SESSION, id)
const originalBody = hostTestMessage('same original input')
// Ready replacement-child exit proof settles before lease release; terminal rows keep that fence.
const settlesAtChildFence = ['readiness', 'rebind'].includes(boundary)
let boundFence = journal.submission(id)!.fence
let stopping: Promise<unknown> = Promise.resolve()
try {
await entered.promise
expect(continuingStructuredAgentSessionInput(journal, id)).toBe(true)
expect(signal?.aborted).toBe(false)
if (boundary === 'retire') {
expect(journal.stoppedHistoryCarried().get(id)).toEqual([stoppedX])
}
const child = sessions.get(SESSION)?.child
expect(journal.submission(id)).toMatchObject({
clientMessageId: id,
options: { personality: 'friendly' }
})
expect(journal.itemBody(agentJournalSubmissionKey(id))).toEqual(originalBody)
boundFence = child?.fence ?? journal.submission(id)!.fence
const boundFence = child?.fence ?? journal.submission(id)!.fence
if (boundary === 'rebind') {
expect(reboundFence).toBe(child!.fence)
}
@@ -755,6 +777,19 @@ export function registerCodexInputFinalization(current: typeof codexInputFinaliz
expect(journal.submission(id)?.rejection).not.toHaveProperty('neverReachedAgent')
expect(starts()).toHaveLength(0)
expect(codex.connections).toHaveLength(boundary === 'retire' ? 1 : 2)
const next = await permissionSend('new question after Stop')
await vi.waitFor(() => expect(starts()).toHaveLength(1))
expect(starts()[0]?.params).toMatchObject({
clientUserMessageId: next,
input: [{ type: 'text', text: 'new question after Stop' }]
})
turns.start()
turns.echo(next)
expect(
current().submission((await host.waitForSendSettlement(SESSION, next))?.value)
).toMatchObject({ clientMessageId: next, dispatchState: 'accepted' })
expect(journal.stoppedHistoryCarried().get(next) ?? []).toEqual([])
await current().waitForInputFinalization(journal, next)
} finally {
held.resolve()
retryReady.resolve()
@@ -783,6 +783,13 @@ export function codexInputFinalizationFixture() {
adapter,
codex,
config,
database: () => openTestJournalHostDatabase(root).db,
rewindTo: (itemId: string, expectedEpoch: string) =>
host.rewind(CALLER, {
envelope: envelope('agentSession.rewind', { itemId, expectedEpoch }),
itemId,
expectedEpoch
}),
turns,
permissionSend,
began,
@@ -124,10 +124,10 @@ export function measureCodexInputWork(
})
expect(work.reportSubmissions).toBeGreaterThanOrEqual(population + 2)
} else {
expect(plans.mock.calls.reduce((sum, [changes]) => sum + changes.length, 0)).toBe(
2 * population + 4
)
expect(comparisons).toBe(population * population + 6 * population + 3)
expect(
plans.mock.calls.reduce((sum, [changes]) => sum + changes.length, 0)
).toBeLessThanOrEqual(2 * population + 4)
expect(comparisons).toBe(0)
expect(work.cards).toBeGreaterThanOrEqual(population)
}
if (population === 100) {
@@ -21,6 +21,7 @@ import { structuredAgentSessionEndedChildFailure } from './structured-agent-sess
import { waitsForCodexTurnOptions } from './structured-agent-session-message-options'
import { sameProviderChild } from './structured-agent-session-provider-child'
import { DeliveryProgress } from './structured-agent-session-delivery-progress'
import { selectableStructuredSubmission } from './structured-agent-session-input-selection'
export type { StructuredAgentSessionDeliveryLoopDeps } from './structured-agent-session-delivery-loop-deps'
@@ -61,10 +62,7 @@ export class StructuredAgentSessionDeliveryLoop {
return Promise.resolve()
}
this.running.get(sessionId)?.woke()
this.deps.deliveryOrder.reconcile(
sessionId,
this.deps.sessions.get(sessionId)?.journal.submissions() ?? []
)
this.deps.deliveryOrder.reconcile(sessionId, this.deps.sessions.get(sessionId)?.journal)
if (this.running.has(sessionId)) {
return this.runs.get(sessionId) ?? Promise.resolve()
}
@@ -187,9 +185,9 @@ export class StructuredAgentSessionDeliveryLoop {
if (!(await this.deps.closeWhatTheUserClosed(sessionId))) {
return this.stop(sessionId)
}
const oldest = oldestQueuedSubmission(session)
// Commands block while their child lives; ended command commits wake delivery again.
const work = this.deps.currentWork(sessionId)
const oldest = selectableStructuredSubmission(session.journal, work)
if (!oldest || (session.child && work && structuredAgentSessionCommandRunning(work))) {
return this.stop(sessionId)
}
@@ -1,5 +1,5 @@
import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types'
import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission'
import type { AgentSessionJournal } from '../agent-session-journal/journal-store'
import { selectedStructuredSubmission } from './structured-agent-session-input-selection'
import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types'
export type StructuredAgentSessionDeliveryPlace = {
@@ -26,8 +26,9 @@ type Lane = {
export class StructuredAgentSessionDeliveryOrder {
private readonly lanes = new Map<string, Lane>()
reconcile(sessionId: string, submissions: readonly AgentJournalSubmission[]): void {
const queued = submissions.filter(isQueuedAgentJournalSubmission)
reconcile(sessionId: string, journal: AgentSessionJournal | undefined): void {
const selected = journal && selectedStructuredSubmission(journal)
const queued = selected ? [selected] : []
const lane = this.lanes.get(sessionId)
if (lane?.options && (queued.length > 0 || lane.active.size > 0)) {
lane.options.demand = 'sealed'
@@ -39,9 +40,7 @@ export class StructuredAgentSessionDeliveryOrder {
place.release()
}
}
for (const submission of queued.sort(
(a, b) => (a.acceptedSequence ?? 0) - (b.acceptedSequence ?? 0)
)) {
for (const submission of queued) {
const current = this.lanes.get(sessionId)
if (!current?.sends.has(submission.clientMessageId)) {
const place = this.reserve(sessionId)
@@ -1,3 +1,4 @@
import { selectableStructuredSubmission } from './structured-agent-session-input-selection'
import { structuredAgentSessionReceivingAccount } from './structured-agent-session-receiving-account'
// Open conversations with queued sends always need delivery; opening also wakes leftover recovery.
@@ -234,9 +235,8 @@ export function createStructuredAgentSessionConversationDelivery(input: {
})
const adoptOpened = async (
sessionId: string,
opened: OpenedStructuredAgentSessionConversation
{ session }: OpenedStructuredAgentSessionConversation
): Promise<void> => {
const { session } = opened
sessions.set(sessionId, session)
input.clientDelivery.publishRestored(sessionId)
await settleInterruptedStructuredAgentSessionCommands(deps, sessionId, session)
@@ -247,13 +247,13 @@ export function createStructuredAgentSessionConversationDelivery(input: {
}
const wakesQueued = new Set<string>()
const afterCommit = (sessionId: string, journal: AgentSessionJournal): void => {
input.deliveryOrder.reconcile(sessionId, journal.submissions())
input.deliveryOrder.reconcile(sessionId, journal)
const work = hostStructuredAgentSessionCurrentWork({ store: deps.store, sessions }, sessionId)
if (
wakesQueued.has(sessionId) ||
!work ||
structuredAgentSessionCommandRunning(work) ||
!journal.submissions().some(isQueuedAgentJournalSubmission)
!selectableStructuredSubmission(journal, work)
) {
return
}
@@ -0,0 +1,633 @@
import * as transitions from '../agent-session-journal/journal-reducer-transition'
import * as outcomes from '../agent-session-journal/journal-outcome-codec'
import { journalInputOrder, JournalInputOrder } from '../agent-session-journal/journal-input-order'
import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages'
import { QueuedMessageEditLeases } from '../agent-session-journal/queued-message-edit-leases'
import * as pause from '../agent-session-journal/queued-message-pause'
import { expect, it, vi } from 'vitest'
import { JournalInputCardIndex } from '../agent-session-journal/journal-input-card-index'
import { JournalInputPlaceIndex } from '../agent-session-journal/journal-input-place-index'
import { JournalInputDomainIndex } from '../agent-session-journal/journal-input-domain-index'
import { JournalInputLineageOwner } from '../agent-session-journal/journal-input-lineage'
import * as receiptTable from '../agent-session-journal/command-receipt-table'
import { loadTestJournal } from '../agent-session-journal/journal-host-database-test-support'
import { selectStructuredAgentSessionInput } from './structured-agent-session-input-selection'
import {
createQueuedMessageTestRig,
eventually
} from './structured-agent-session-queued-message-rig.test-fixture'
import { openRigTurnFor } from './structured-agent-session-queued-rig-turn.test-fixture'
import {
HOST_TEST_SESSION as SESSION,
hostTestMessage
} from './structured-agent-session-host-test-data'
import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key'
import { QUEUED_MESSAGE_PAUSED_SEND_FAILED } from '../../../shared/agent-session-queued-message-wire'
it.each([10, 100])(
'bounds actual edit/growth, replay, replacement and complete recovery N=%i',
async (n) => {
type Counts = {
rows: number
queueVisits: number
inputVisits: number
sets: number
queries: number
domains: number
lineage: number
receipts: number
nodes: number
nodeReads: number
draftNodes: number
derivedDraftNodes: number
nativeEntries: number
nativeCards: number
nativeBytes: number
}
const phases: Record<string, Counts> = {}
let phase = 'setup'
const metric = () =>
(phases[phase] ??= {
rows: 0,
queueVisits: 0,
inputVisits: 0,
sets: 0,
queries: 0,
domains: 0,
lineage: 0,
receipts: 0,
nodes: 0,
nodeReads: 0,
draftNodes: 0,
derivedDraftNodes: 0,
nativeEntries: 0,
nativeCards: 0,
nativeBytes: 0
})
const consumers: Record<
string,
{
heads: number
lists: number
pauses: number
leases: number
gets: number
waitingRows: number
leaseRows: number
headVisits: number
predicates: number
indexedCards: number
}
> = {}
const upstream: Record<
string,
Record<string, { calls: number; returned: number; materialized: number }>
> = {}
const source = (): string => {
const stack = new Error('Attribute production queue reads').stack ?? ''
if (selecting || stack.includes('selectStructuredAgentSessionInput')) {
return 'selection'
}
if (stack.includes('insert')) {
return 'insertion'
}
if (stack.includes('snapshot') || stack.includes('Snapshot')) {
return 'publication'
}
return 'other'
}
let selecting = 0
const seam = () =>
(consumers[phase] ??= {
heads: 0,
lists: 0,
pauses: 0,
leases: 0,
gets: 0,
waitingRows: 0,
leaseRows: 0,
headVisits: 0,
predicates: 0,
indexedCards: 0
})
const stackLimit = Error.stackTraceLimit
Error.stackTraceLimit = 40
const indexCard = JournalInputCardIndex.prototype.indexCard
const indexed = vi
.spyOn(JournalInputCardIndex.prototype, 'indexCard')
.mockImplementation(function (this: JournalInputCardIndex, ...args) {
if (selecting) {
seam().indexedCards++
}
return indexCard.apply(this, args)
})
const next = JournalQueuedMessages.prototype.nextInputCard
const head = vi
.spyOn(JournalQueuedMessages.prototype, 'nextInputCard')
.mockImplementation(function (this: JournalQueuedMessages, ...args) {
selecting++
seam().heads++
try {
return next.apply(this, args)
} finally {
selecting--
}
})
const list = JournalQueuedMessages.prototype.list
const lists = vi.spyOn(JournalQueuedMessages.prototype, 'list')
lists.mockImplementation(function (this: JournalQueuedMessages) {
const caller = source(),
before = this['viewed'],
result = list.call(this)
const callers = (upstream[phase] ??= {})
const row = (callers[caller] ??= { calls: 0, returned: 0, materialized: 0 })
row.calls++
row.returned += result.length
if (before !== this['viewed']) {
row.materialized += result.length
}
if (caller === 'selection') {
seam().lists++
}
return result
})
const derive = pause.deriveQueuePauses
const pauses = vi.spyOn(pause, 'deriveQueuePauses')
pauses.mockImplementation((...args) => {
if (source() === 'selection') {
seam().pauses++
seam().waitingRows += args[0].cards.length
}
return derive(...args)
})
const held = QueuedMessageEditLeases.prototype.heldIds
const leases = vi.spyOn(QueuedMessageEditLeases.prototype, 'heldIds')
leases.mockImplementation(function (this: QueuedMessageEditLeases, ...args) {
if (source() === 'selection') {
seam().leases++
seam().leaseRows += args[0]?.length ?? 0
}
return held.apply(this, args)
})
const pick = pause.nextSendableQueuedCard
const picks = vi
.spyOn(pause, 'nextSendableQueuedCard')
.mockImplementation((pauses, rows, held) => {
const counting = source() === 'selection'
const counted = rows.slice()
counted[Symbol.iterator] = function* () {
for (const row of rows) {
if (counting) {
seam().headVisits++
}
yield row
}
return undefined
}
return pick(pauses, counted, held)
})
const predicate = pause.queuePauseHolding
const predicates = vi.spyOn(pause, 'queuePauseHolding').mockImplementation((...args) => {
if (source() === 'selection') {
seam().predicates++
}
return predicate(...args)
})
const get = JournalQueuedMessages.prototype.get
const gets = vi.spyOn(JournalQueuedMessages.prototype, 'get')
gets.mockImplementation(function (this: JournalQueuedMessages, ...args) {
if (selecting) {
seam().gets++
}
return get.apply(this, args)
})
const isolate = transitions.isolateJournalTransition
const prepared = vi
.spyOn(transitions, 'isolateJournalTransition')
.mockImplementation((...args) => {
const draft = isolate(...args)
return {
...draft,
adopt: () => {
const state = draft.adopt()
metric().draftNodes += draft.tracked() - draft.trackedDerived()
metric().derivedDraftNodes += draft.trackedDerived()
return state
}
}
})
const encode = outcomes.encodeJournalOutcomes
const encoded = vi.spyOn(outcomes, 'encodeJournalOutcomes').mockImplementation((...args) => {
const result = encode(...args)
for (const value of result.values) {
if (value.field === 'inputEntries') {
metric().nativeEntries++
}
if (value.field === 'inputCards') {
metric().nativeCards++
}
if (value.field === 'inputEntries' || value.field === 'inputCards') {
metric().nativeBytes += Buffer.byteLength(value.json)
}
}
return result
})
const reflected = Reflect.get
const nodes = new WeakSet<object>()
const nodeReads = vi.spyOn(Reflect, 'get').mockImplementation((...args) => {
const [target] = args
// The prepared draft reads current objects through this seam. Observe rather than
// installing accessors on transaction nodes, which would mutate the measured graph.
if (Object.hasOwn(target, 'height') && Object.hasOwn(target, 'entry')) {
metric().nodeReads++
if (!nodes.has(target)) {
nodes.add(target)
metric().nodes++
}
}
return reflected(...args)
})
const boundaryCalls: Record<string, number> = {}
const predecessor = JournalInputOrder.prototype.predecessors
const predecessors = vi
.spyOn(JournalInputOrder.prototype, 'predecessors')
.mockImplementation(function (this: JournalInputOrder, ...args) {
boundaryCalls[phase] = (boundaryCalls[phase] ?? 0) + 1
return predecessor.apply(this, args)
})
const set = JournalInputPlaceIndex.prototype.set
const indexSet = vi.spyOn(JournalInputPlaceIndex.prototype, 'set').mockImplementation(function (
this: JournalInputPlaceIndex,
...args
) {
metric().sets++
set.apply(this, args)
})
const query = JournalInputPlaceIndex.prototype.query
const indexQuery = vi
.spyOn(JournalInputPlaceIndex.prototype, 'query')
.mockImplementation(function (this: JournalInputPlaceIndex, ...args) {
metric().queries++
return query.apply(this, args)
})
const domainSet = JournalInputDomainIndex.prototype.set
const domain = vi.spyOn(JournalInputDomainIndex.prototype, 'set').mockImplementation(function (
this: JournalInputDomainIndex,
...args
) {
metric().domains++
return domainSet.apply(this, args)
})
const change = JournalInputLineageOwner.prototype.change
const line = vi
.spyOn(JournalInputLineageOwner.prototype, 'change')
.mockImplementation(function (this: JournalInputLineageOwner, ...args) {
metric().lineage++
return change.apply(this, args)
})
const receipt = receiptTable.readCommandReceipt
const receipts = vi.spyOn(receiptTable, 'readCommandReceipt').mockImplementation((...args) => {
metric().receipts++
return receipt(...args)
})
const before = JournalInputOrder.prototype.beforeRow
const rows = vi.spyOn(JournalInputOrder.prototype, 'beforeRow').mockImplementation(function (
this: JournalInputOrder,
state,
row
) {
metric().rows++
const values = state.queue.cards.values.bind(state.queue.cards)
const visits = vi.spyOn(state.queue.cards, 'values').mockImplementation(function* () {
for (const card of values()) {
metric().queueVisits++
yield card
}
return undefined
})
try {
return before.call(this, state, row)
} finally {
visits.mockRestore()
}
})
const after = JournalInputOrder.prototype.afterRow
const folded = vi.spyOn(JournalInputOrder.prototype, 'afterRow').mockImplementation(function (
this: JournalInputOrder,
state,
row
) {
const values = state.submissions.values.bind(state.submissions)
const visits = vi.spyOn(state.submissions, 'values').mockImplementation(function* () {
for (const input of values()) {
metric().inputVisits++
yield input
}
return undefined
})
try {
return after.call(this, state, row)
} finally {
visits.mockRestore()
}
})
const rig = await createQueuedMessageTestRig({
restartable: true,
probeOwner: async () => ({ outcome: 'pid-absent' })
})
try {
const w = await rig.workingSend()
await openRigTurnFor(rig, w)
const { serialize, sessions } = rig.host.collaboratorsForTests()
const journal = sessions.get(SESSION)!.journal
const q = Array.from({ length: n }, (_, i) => `Q${i}`)
await serialize(SESSION, async () => {
for (const [i, id] of q.entries()) {
await journal.queuedMessages.insert({
messageId: id,
body:
i === 0
? { ...hostTestMessage('/compact'), command: { name: 'compact' } }
: hostTestMessage(id)
})
}
for (let i = 1; i < n; i++) {
if (i % 3 === 0) {
continue
}
const card = journal.queuedMessages.get(q[i])!
await journal.queuedMessages['write'](() => ({
changes: [
{
messageId: card.messageId,
change: {
op: 'queued',
body: card.body,
position: card.position,
queuedAt: i % 3 === 1 ? null : { epoch: 'imported-domain', sequence: 1 }
}
}
],
result: undefined
}))
}
await journal.queuedMessages.insert({
messageId: 'E',
body: hostTestMessage('held'),
holdReason: QUEUED_MESSAGE_PAUSED_SEND_FAILED
})
})
phase = 'edits-growth'
const ids: string[] = []
for (let i = 0; i < n; i++) {
await serialize(SESSION, async () => {
for (const id of ['E', 'Q1']) {
expect(
await journal.queuedMessages.update({
messageId: id,
expectedBodyFingerprint: journal.queuedMessages.get(id)!.fingerprint,
text: `${id}-edit-${i}`,
images: []
})
).toMatchObject({ status: 'updated' })
}
await journal.queuedMessages.insert({
messageId: `D${i}`,
body: hostTestMessage(`D${i}`)
})
})
const m = rig.send(`M${i}`)
ids.push(m.id)
expect(await m.result).toMatchObject({ ok: true })
expect(journal.inputOrder.forced(m.id)).toBe(true)
expect(
journal.inputOrder.precedes(
journal.queuedMessages.get(`D${i}`)!,
journal.submission(m.id)!
)
).toBe(true)
if (i) {
expect(
journal.inputOrder.precedes(
journal.queuedMessages.get(`D${i}`)!,
journal.submission(ids[i - 1])!
)
).toBe(false)
}
}
await serialize(SESSION, () =>
journal.queuedMessages
.insert({ messageId: 'L', body: hostTestMessage('later sentinel') })
.then(() => {})
)
expect(metric().queueVisits).toBe(0)
expect(metric().inputVisits).toBe(0)
expect(metric().receipts).toBe(n) // Existing direct acceptance receipt read, no ordering probe.
// Two coordinate indexes, one point per new card; body-only edits never update an order index.
expect(metric().sets).toBe(2 * (n + 1))
expect(metric().nativeEntries).toBe(n)
expect(metric().nativeCards).toBe(n + 1)
// Four AVL point paths (all/domain in two coordinate indexes), at most
// twice log height, reaching at most eight draft objects per level.
expect(metric().derivedDraftNodes).toBeLessThanOrEqual(
64 * (n + 1) * Math.ceil(Math.log2(2 * n + 3))
)
expect(metric().nodeReads).toBeLessThanOrEqual(
200 * (n + 1) * Math.ceil(Math.log2(2 * n + 3))
)
expect(journal.queuedMessages.displayState().inputEntries.size).toBe(n + 1)
expect(journal.inputOrder.lineage.cards.size).toBe(2 * n + 2)
expect(domain).not.toHaveBeenCalled()
for (let i = 0; i < n; i++) {
const key = { callerKey: 'projection', editId: `lease-${i}` }
expect(
journal.queuedMessages.editLeases.acquire(key, {
messageId: 'Q0',
fingerprint: journal.queuedMessages.get('Q0')!.fingerprint
}).status
).toBe('held')
expect(selectStructuredAgentSessionInput(journal).kind).toBe('none')
journal.queuedMessages.editLeases.release(key, 'Q0')
expect(selectStructuredAgentSessionInput(journal)).toMatchObject({
kind: 'card',
card: { messageId: 'Q0' }
})
}
rig.host.stopDelivery()
phase = 'original-replay'
expect(loadTestJournal(rig.root, SESSION)?.damage).toBeNull()
expect(metric().receipts).toBe(0)
for (let pass = 0; pass < 2; pass++) {
phase = `replacement-${pass}`
await serialize(SESSION, () =>
journal.replaceEpochItems('unreconcilable_prefix', 1, []).then(() => {})
)
expect(metric().rows).toBe(2 * (3 * n + 3)) // Builder and isolated publication each fold the same actual cohort.
expect(metric().receipts).toBe(n)
expect(
[...journal.queuedMessages.displayState().inputCards.values()].filter(
(card) => card.baseline !== undefined
).length
).toBeLessThanOrEqual(2 * n + 4)
phase = `output-replay-${pass}`
const replay = loadTestJournal(rig.root, SESSION)!
expect(replay.damage).toBeNull()
expect(replay.newer).toBeNull()
expect(metric().rows).toBe(0) // Keyed restore assigns canonical facts and rebuilds derived indexes once.
expect(ids.every((id) => journalInputOrder(replay.state).forced(id))).toBe(true)
}
phase = 'recovery'
await rig.restartHostProcess()
for (const id of ids) {
expect(await rig.submission(id)).toMatchObject({
dispatchState: 'rejected',
keptAsQueuedMessageId: id
})
}
const reopened = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal
const expected = [...q, 'E', ...ids.flatMap((id, i) => [`D${i}`, id]), 'L']
const cards = reopened.queuedMessages.list().filter((card) => card.state === 'waiting')
expect(cards.map((card) => card.messageId)).toEqual(expected)
expect(
cards.every(
(card, i) =>
Number.isFinite(card.position) && (i === 0 || card.position > cards[i - 1].position)
)
).toBe(true)
expect(reopened.queuedMessages.get('E')?.holdReason).toBe(QUEUED_MESSAGE_PAUSED_SEND_FAILED)
expect(reopened.queuedMessages.get('Q1')?.body).toEqual(hostTestMessage(`Q1-edit-${n - 1}`))
expect(reopened.queuedMessages.displayState().inputEntries.size).toBe(0)
// A real held-card retry releases E at its existing selected Queue place.
await rig.host.collaboratorsForTests().serialize(SESSION, () =>
reopened.queuedMessages['write'](() => ({
changes: [
{
messageId: 'E',
change: {
op: 'queued',
body: reopened.queuedMessages.get('E')!.body,
position: reopened.queuedMessages.get('E')!.position,
queuedAt: reopened.queuedMessages.get('E')!.queuedAt
}
}
],
result: undefined
}))
)
phase = 'delivery'
const base = rig.dispatch.mock.calls.length
expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } })
await eventually(() => expect(rig.compact).toHaveBeenCalledOnce())
const command = rig.compact.mock.calls[0]![0].command
rig.finishCompact()
await eventually(() =>
expect(reopened.item(agentJournalItemKey(command.identity))?.body).toMatchObject({
kind: 'turn',
state: 'completed'
})
)
const actual = ['Q0']
for (let i = 0; i < expected.length - 1; i++) {
await eventually(() => expect(rig.dispatch.mock.calls).toHaveLength(base + i + 1))
const input = rig.dispatch.mock.calls[base + i]![0]
const submission = await rig.submission(input.clientMessageId)
actual.push(submission!.queuedMessageId!)
if (submission!.queuedMessageId === 'Q1') {
expect(input.body).toEqual(hostTestMessage(`Q1-edit-${n - 1}`))
}
await rig.settleAccepted(input.clientMessageId, `actual-${i}`)
expect(await rig.submission(input.clientMessageId)).toMatchObject({
dispatchState: 'accepted'
})
}
expect(actual).toEqual(expected)
expect(new Set(actual).size).toBe(expected.length)
expect(reopened.queuedMessages.list().filter((card) => card.state === 'waiting')).toEqual([])
expect(reopened.inputOrder.unsettledSubmissions()).toEqual([])
expect(
rig.dispatch.mock.calls.slice(base).some(([input]) => input.clientMessageId === w)
).toBe(false)
expect(reopened.queuedMessages.displayState().inputEntries.size).toBe(0)
expect(reopened.queuedMessages.editLeases.heldIds().size).toBe(0)
// Each folded card transition updates two current trees; recovery/pause adds a bounded bulk pass.
// A set removes/inserts in two AVLs; rotations and summary refreshes cost a fixed number of node reads per level.
for (const [name, count] of Object.entries(phases)) {
expect(count.nodeReads, name).toBeLessThanOrEqual(
200 * (count.sets + count.queries + 1) * Math.ceil(Math.log2(4 * n + 8))
)
expect(count.nodes, name).toBeLessThanOrEqual(count.nodeReads)
}
expect(phases['original-replay'].sets).toBeLessThanOrEqual(6 * n + 16)
for (let pass = 0; pass < 2; pass++) {
for (const name of [`replacement-${pass}`, `output-replay-${pass}`]) {
// The builder and isolated publisher each initialize two trees from K cards;
// their first card also takes the ordinary affected-ID update. Keyed replay builds once.
expect(phases[name].sets, name).toBe(
name.startsWith('replacement') ? 4 * (2 * n + 3) : 2 * (2 * n + 2)
)
// One fold query per admitted input; each actual consumer boundary call adds one point query.
expect(phases[name].queries, name).toBeLessThanOrEqual(n + (boundaryCalls[name] ?? 0))
expect(phases[name].lineage, name).toBe(
name.startsWith('replacement') ? 2 * (2 * n + 2) : 0
)
}
}
// Per row, at most two affected-card refreshes of two indexes; two effective-pause passes may rebuild K.
expect(phases.recovery.rows).toBeLessThanOrEqual(8 * n + 12)
expect(phases.recovery.sets).toBeLessThanOrEqual(4 * phases.recovery.rows + 4 * (2 * n + 2))
expect(phases.recovery.queries).toBe(n) // One predecessor query per pending input; keyed restore does not readmit it.
expect(phases.delivery.sets).toBeLessThanOrEqual(32 * n + 8)
expect(phases.delivery.queries).toBe(0)
console.info('current-consumer-evidence', JSON.stringify({ n, phases, consumers, upstream }))
for (const [name, count] of Object.entries(consumers)) {
expect(count.lists, name).toBe(0)
expect(count.pauses, name).toBe(0)
expect(count.leases, name).toBe(0)
expect(count.waitingRows, name).toBe(0)
expect(count.leaseRows, name).toBe(0)
expect(count.headVisits, name).toBe(0)
// Each head or effective-pause reindex evaluates at most one predicate per affected card.
expect(count.predicates, name).toBeLessThanOrEqual(count.heads + count.indexedCards)
// One latest-value get and one lease fingerprint get at most per head.
expect(count.gets, name).toBeLessThanOrEqual(2 * count.heads)
}
expect(consumers['edits-growth'].indexedCards).toBe(0)
expect(consumers['edits-growth'].heads).toBeGreaterThan(n)
console.info('current-consumer-counts', JSON.stringify({ n, consumers }))
console.info(
'ordered-place-counts',
JSON.stringify({
n,
phases,
domainCalls: domain.mock.calls.length,
lineageCalls: line.mock.calls.length,
complete: true
})
)
} finally {
await rig.dispose()
Error.stackTraceLimit = stackLimit
indexed.mockRestore()
picks.mockRestore()
predicates.mockRestore()
predecessors.mockRestore()
head.mockRestore()
lists.mockRestore()
pauses.mockRestore()
leases.mockRestore()
gets.mockRestore()
encoded.mockRestore()
prepared.mockRestore()
nodeReads.mockRestore()
indexSet.mockRestore()
indexQuery.mockRestore()
domain.mockRestore()
line.mockRestore()
receipts.mockRestore()
rows.mockRestore()
folded.mockRestore()
}
},
60_000
)
@@ -0,0 +1,344 @@
import { journalInputOrder } from '../agent-session-journal/journal-input-order'
import { AgentSessionJournal } from '../agent-session-journal/journal-store'
import { expect, it, vi } from 'vitest'
import {
createQueuedMessageTestRig,
eventually
} from './structured-agent-session-queued-message-rig.test-fixture'
import { openRigTurnFor } from './structured-agent-session-queued-rig-turn.test-fixture'
import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key'
import {
HOST_TEST_SESSION as SESSION,
hostTestMessage
} from './structured-agent-session-host-test-data'
import { loadTestJournal } from '../agent-session-journal/journal-host-database-test-support'
import { queuePauseHolding } from '../agent-session-journal/queued-message-pause'
it.each([false, true])(
'failed reopen=%s: unentered M retains ordinary head recovery through graceful restart',
async (fail) => {
const rig = await createQueuedMessageTestRig({
restartable: true,
probeOwner: async () => ({ outcome: 'pid-absent' })
})
let release = () => {}
try {
const warm = await rig.workingSend()
await rig.settleAccepted(warm, 'warm')
const { sessions, serialize, conversationDelivery } = rig.host.collaboratorsForTests()
const journal = sessions.get(SESSION)!.journal
await serialize(SESSION, async () => {
await journal.queuedMessages.insert({
messageId: 'C',
body: { ...hostTestMessage('/compact'), command: { name: 'compact' } }
})
const mark = fail
? vi
.spyOn(journal, 'appendQueueReopen')
.mockRejectedValueOnce(new Error('injected reopen write failure'))
: null
await journal.markQueueReopen(1).catch(() => {})
mark?.mockRestore()
})
const n = rig.send('N opens work without an accepted echo')
expect(await n.result).toMatchObject({ ok: true })
await eventually(async () => expect((await rig.submission(n.id))?.handedOverAt).toBeDefined())
await openRigTurnFor(rig, n.id)
expect(journal.queuedMessages.pauses().map((p) => p.reason)).toEqual(['restarted'])
const order = conversationDelivery.loop['deps'].deliveryOrder
const reserved = order.reserve(SESSION)
release = reserved.release
const before = order.beforeSend.bind(order)
const waiting = vi.spyOn(order, 'beforeSend').mockImplementation((...args) => before(...args))
const effectiveEligible = journal.queuedMessages
.list()
.filter(
(card) =>
card.state === 'waiting' &&
card.holdReason === null &&
!queuePauseHolding(journal.queuedMessages.pauses(), card)
)
.map((card) => card.messageId)
expect(effectiveEligible).toEqual([])
const m = rig.send('M ordinary steer while C is held, never entered', undefined, {
carried: { personality: 'friendly', effort: 'high' }
})
expect(await m.result).toMatchObject({ ok: true })
await serialize(SESSION, async () => {})
await eventually(() => expect(waiting.mock.calls.some(([, id]) => id === m.id)).toBe(true))
expect(await rig.submission(m.id)).toMatchObject({ dispatchState: 'pending' })
expect((await rig.submission(m.id))?.handedOverAt).toBeUndefined()
expect(rig.dispatch.mock.calls.some(([input]) => input.clientMessageId === m.id)).toBe(false)
const original = journal.submission(m.id)!
const receipt = rig.store.readCommandReceipt({ kind: 'global' }, m.id)
expect(receipt).toMatchObject({
verdict: 'readable',
receipt: {
status: 'accepted',
method: 'agentSession.send',
result: { kind: 'journal-row', epoch: journal.epoch, sequence: original.acceptedSequence }
}
})
expect(original.options).toEqual({ personality: 'friendly', effort: 'high' })
const replayBefore = loadTestJournal(rig.root, SESSION)!
expect(replayBefore.damage).toBeNull()
expect(replayBefore.newer).toBeNull()
expect(journal.inputOrder.forced(m.id)).toBe(false)
expect(journalInputOrder(replayBefore.state).forced(m.id)).toBe(false)
waiting.mockRestore()
await rig.restartHostProcess()
const replayAfter = loadTestJournal(rig.root, SESSION)!
expect(replayAfter.damage).toBeNull()
expect(replayAfter.newer).toBeNull()
const shutdownM = replayAfter.state.submissions.get(m.id)
expect(shutdownM?.handedOverAt).toBeUndefined()
const recoveredM = await rig.submission(m.id)
expect(recoveredM).toMatchObject({ dispatchState: 'rejected', keptAsQueuedMessageId: m.id })
expect(recoveredM?.handedOverAt).toBeUndefined()
expect(recoveredM?.options).toEqual(original.options)
expect(recoveredM?.payloadFingerprint).toBe(original.payloadFingerprint)
expect(recoveredM?.acceptedSequence).toBe(original.acceptedSequence)
const reopened = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal
expect(rig.store.readCommandReceipt({ kind: 'global' }, m.id)).toEqual(receipt)
expect(reopened.queuedMessages.get(m.id)).toMatchObject({
body: hostTestMessage('M ordinary steer while C is held, never entered'),
options: original.options
})
const cards = reopened.queuedMessages.list().filter((c) => c.state === 'waiting')
const observedOrder = cards.map((c) => c.messageId)
expect(observedOrder).toEqual([m.id, 'C'])
expect(rig.compact).not.toHaveBeenCalled()
const deliveries: string[] = []
expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } })
for (const id of observedOrder) {
if (id === 'C') {
await eventually(() => expect(rig.compact).toHaveBeenCalledOnce())
deliveries.push('C')
const command = rig.compact.mock.calls[0]![0].command
rig.finishCompact()
await eventually(() =>
expect(reopened.item(agentJournalItemKey(command.identity))?.body).toMatchObject({
kind: 'turn',
state: 'completed'
})
)
} else {
await eventually(async () =>
expect((await rig.handoff(m.id))?.handedOverAt).toBeDefined()
)
deliveries.push('M')
const handoff = await rig.handoffId(m.id)
expect(rig.compact).not.toHaveBeenCalled()
await rig.settleAccepted(handoff, 'M-after-restart')
expect(await rig.submission(handoff)).toMatchObject({ dispatchState: 'accepted' })
}
}
expect(deliveries).toEqual(['M', 'C'])
} finally {
release()
await rig.dispose()
}
}
)
it.each([false, true])(
'reopen mark failure=%s keeps a paused command from forcing a later steer',
async (fail) => {
const rig = await createQueuedMessageTestRig()
try {
const warm = await rig.workingSend()
await rig.settleAccepted(warm, 'warm')
const { sessions, serialize } = rig.host.collaboratorsForTests()
const journal = sessions.get(SESSION)!.journal
await serialize(SESSION, async () => {
await journal.queuedMessages.insert({
messageId: 'C',
body: { ...hostTestMessage('/compact'), command: { name: 'compact' } }
})
const mark = fail
? vi.spyOn(journal, 'appendQueueReopen').mockRejectedValueOnce(new Error('disk full'))
: null
await journal.markQueueReopen(1).catch(() => {})
mark?.mockRestore()
})
expect(journal.queuedMessages.pauses().map((p) => p.reason)).toEqual(['restarted'])
const n = rig.send('N opens next work')
expect(await n.result).toMatchObject({ ok: true })
await eventually(async () => expect((await rig.submission(n.id))?.handedOverAt).toBeDefined())
await openRigTurnFor(rig, n.id)
expect(journal.queuedMessages.pauses().map((p) => p.reason)).toEqual(['restarted'])
const m = rig.send('M steer while C is held')
expect(await m.result).toMatchObject({
ok: true,
value: { submission: { clientMessageId: m.id } }
})
await serialize(SESSION, async () => {})
expect(journal.inputOrder.forced(m.id)).toBe(false)
expect(rig.compact).not.toHaveBeenCalled()
await eventually(async () => expect((await rig.submission(m.id))?.handedOverAt).toBeDefined())
await rig.settleAccepted(n.id, 'N')
expect(journal.queuedMessages.pauses()).toEqual([])
const later = rig.send('later admission after actual accepted N')
expect(await later.result).toMatchObject({ ok: true })
expect(journal.inputOrder.forced(later.id)).toBe(true)
expect(
journal.inputOrder.precedes(journal.queuedMessages.get('C')!, journal.submission(later.id)!)
).toBe(true)
await rig.settleAccepted(m.id, 'M')
await openRigTurnFor(rig, n.id, 'interrupted')
await eventually(() => expect(rig.compact).toHaveBeenCalledOnce())
const command = rig.compact.mock.calls[0]![0].command
expect(journal.submission(later.id)?.handedOverAt).toBeUndefined()
rig.finishCompact()
await eventually(() =>
expect(journal.item(agentJournalItemKey(command.identity))?.body).toMatchObject({
kind: 'turn',
state: 'completed'
})
)
await eventually(async () =>
expect((await rig.submission(later.id))?.handedOverAt).toBeDefined()
)
await rig.settleAccepted(later.id, 'later')
for (const id of [n.id, m.id, later.id]) {
expect(journal.submission(id)?.dispatchState).toBe('accepted')
expect(
rig.dispatch.mock.calls.filter(([input]) => input.clientMessageId === id)
).toHaveLength(1)
}
} finally {
await rig.dispose()
}
}
)
// Two host restarts/replacements and four ordered handoffs need the extended budget.
it('keeps an earlier-process recovery identity through two adoptions after both writes fail, then delivers the observed C,N,M place', async () => {
const rig = await createQueuedMessageTestRig({
restartable: true,
probeOwner: async () => ({ outcome: 'pid-absent' })
})
const resolve = AgentSessionJournal.prototype.resolveDispatch
let blocked = true
let older = ''
const warned = vi.spyOn(console, 'warn').mockImplementation(() => {})
const failure = vi
.spyOn(AgentSessionJournal.prototype, 'resolveDispatch')
.mockImplementation(function (this: AgentSessionJournal, ...args) {
return blocked && args[0].clientMessageId === older
? Promise.reject(new Error('both recovery writes unavailable'))
: resolve.apply(this, args)
})
let release = () => {}
try {
const w = await rig.workingSend()
await openRigTurnFor(rig, w)
const initial = rig.host.collaboratorsForTests()
const reservation = initial.conversationDelivery.loop['deps'].deliveryOrder.reserve(SESSION)
release = reservation.release
const o = rig.send('older O')
older = o.id
expect(await o.result).toMatchObject({ ok: true })
expect((await rig.submission(older))?.handedOverAt).toBeUndefined()
await initial.serialize(SESSION, async () => {
const journal = initial.sessions.get(SESSION)!.journal
await journal.queuedMessages.insert({
messageId: 'C',
body: { ...hostTestMessage('/compact'), command: { name: 'compact' } }
})
await journal.queuedMessages['write'](() => ({
changes: [
{
messageId: 'N',
change: { op: 'queued', body: hostTestMessage('null N'), position: 2, queuedAt: null }
}
],
result: undefined
}))
})
await rig.restartHostProcess()
expect(await rig.submission(older)).toMatchObject({ dispatchState: 'pending' })
rig.host.stopDelivery()
const { serialize, sessions } = rig.host.collaboratorsForTests()
const journal = sessions.get(SESSION)!.journal
expect(journal.inputOrder.recoveryPending(older)).toBe(true)
const fence = journal.queuedMessages.displayState().highestFence
await serialize(SESSION, () => journal.appendQueueResume(fence).then(() => {}))
const m = rig.send('fresh M')
expect(await m.result).toMatchObject({ ok: true })
expect(journal.inputOrder.forced(m.id)).toBe(true)
expect(
journal.inputOrder.precedes(journal.queuedMessages.get('N')!, journal.submission(m.id)!)
).toBe(true)
for (let pass = 0; pass < 2; pass++) {
await serialize(SESSION, () =>
journal.replaceEpochItems('unreconcilable_prefix', fence, []).then(() => {})
)
expect(journal.inputOrder.recoveryPending(older)).toBe(true)
expect(journal.inputOrder.recoveryPending(m.id)).toBe(false)
expect(
journal.inputOrder.precedes(journal.queuedMessages.get('N')!, journal.submission(m.id)!)
).toBe(true)
const replay = loadTestJournal(rig.root, SESSION)!
expect(replay.damage).toBeNull()
expect(
journalInputOrder(replay.state).precedes(
replay.state.queue.cards.get('N')!,
replay.state.submissions.get(m.id)!
)
).toBe(true)
}
blocked = false
release()
await rig.restartHostProcess()
for (const id of [older, m.id]) {
expect(await rig.submission(id)).toMatchObject({
dispatchState: 'rejected',
keptAsQueuedMessageId: id
})
}
const reopened = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal
expect(
reopened.queuedMessages
.list()
.filter((card) => card.state === 'waiting')
.map((card) => card.messageId)
).toEqual([older, 'C', 'N', m.id])
expect(reopened.inputOrder.recoveryPending(older)).toBe(false)
const base = rig.dispatch.mock.calls.length
expect(await rig.resume()).toMatchObject({ ok: true })
const actual: string[] = []
await eventually(() => expect(rig.dispatch.mock.calls).toHaveLength(base + 1))
actual.push(
(await rig.submission(rig.dispatch.mock.calls[base]![0].clientMessageId))!.queuedMessageId!
)
await rig.settleAccepted(rig.dispatch.mock.calls[base]![0].clientMessageId, 'O')
await eventually(() => expect(rig.compact).toHaveBeenCalledOnce())
actual.push('C')
const command = rig.compact.mock.calls[0]![0].command
rig.finishCompact()
await eventually(() =>
expect(reopened.item(agentJournalItemKey(command.identity))?.body).toMatchObject({
kind: 'turn',
state: 'completed'
})
)
for (let i = 0; i < 2; i++) {
await eventually(() => expect(rig.dispatch.mock.calls).toHaveLength(base + i + 2))
const input = rig.dispatch.mock.calls[base + i + 1]![0]
actual.push((await rig.submission(input.clientMessageId))!.queuedMessageId!)
await rig.settleAccepted(input.clientMessageId, `after-C-${i}`)
}
expect(actual).toEqual([older, 'C', 'N', m.id])
expect(new Set(actual).size).toBe(4)
expect(reopened.queuedMessages.list().filter((card) => card.state === 'waiting')).toEqual([])
} finally {
blocked = false
release()
failure.mockRestore()
warned.mockRestore()
await rig.dispose()
}
}, 60_000)
@@ -0,0 +1,119 @@
import { it, expect, vi } from 'vitest'
import {
createQueuedMessageTestRig,
eventually,
QUEUED_RIG_CALLER as CALLER
} from './structured-agent-session-queued-message-rig.test-fixture'
import {
HOST_TEST_SESSION as SESSION,
hostTestOperationId
} from './structured-agent-session-host-test-data'
import {
agentJournalItemKey,
agentJournalSubmissionKey
} from '../../../shared/agent-session-journal-item-key'
it('preserves existing queuedAt across two live replacements despite an older original receipt', async () => {
const rewind = vi.fn(async () => ({ ok: true as const }))
const rig = await createQueuedMessageTestRig({ restartable: true, rewind })
try {
const first = await rig.workingSend()
await rig.settleAccepted(first, 'first')
const second = rig.send('second rewind target')
await second.result
await eventually(async () =>
expect((await rig.submission(second.id))?.handedOverAt).toBeDefined()
)
await rig.settleAccepted(second.id, 'second')
const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal
const epoch1 = journal.epoch
const original = journal.replaceEpochItems.bind(journal)
const snapshots: unknown[] = []
let command: Promise<unknown> | undefined, rewindAgain: Promise<unknown> | undefined
const c = hostTestOperationId()
const fields = { command: 'compact' as const, delivery: 'queue-if-active' as const }
const capture = () => ({
epoch: journal.epoch,
submissions: journal.submissions().map((s) => ({
id: s.clientMessageId,
seq: s.acceptedSequence,
handed: s.handedOverAt,
state: s.dispatchState
})),
cards: journal.queuedMessages.list(),
pauses: journal.queuedMessages.pauses()
})
vi.spyOn(journal, 'replaceEpochItems').mockImplementation(async (...args) => {
snapshots.push(capture())
const result = await original(...args)
snapshots.push(capture())
if (snapshots.length === 2) {
// Real admissions queue before delivery resumes after the first replacement.
command = rig.host.conversationCommand(CALLER, {
...fields,
envelope: rig.envelope(fields, 'agentSession.conversationCommand', c),
userSend: true
})
const target = journal
.snapshot()
.items.find(
(item) =>
item.body.kind === 'message' &&
item.body.blocks.some(
(block) => block.type === 'text' && block.text === 'work on this'
)
)!
const next = { itemId: target.itemId, expectedEpoch: journal.epoch }
rewindAgain = rig.host.rewind(CALLER, {
...next,
envelope: rig.envelope(next, 'agentSession.rewind', hostTestOperationId())
})
}
return result
})
const m = rig.send('M from E1')
const cut = { itemId: agentJournalSubmissionKey(second.id), expectedEpoch: epoch1 }
const rewound = rig.host.rewind(CALLER, {
...cut,
envelope: rig.envelope(cut, 'agentSession.rewind', hostTestOperationId())
})
expect(await m.result).toMatchObject({ ok: true })
expect(await rewound).toMatchObject({ ok: true })
expect(await command).toMatchObject({ ok: true, value: { queued: { state: 'waiting' } } })
const secondVerdict = await rewindAgain
expect(secondVerdict).toMatchObject({ ok: true })
expect(snapshots).toHaveLength(4)
const receipt = rig.store.readCommandReceipt({ kind: 'global' }, m.id)
const card = journal.queuedMessages.get(c)!
expect(receipt).toMatchObject({
verdict: 'readable',
receipt: {
status: 'accepted',
method: 'agentSession.send',
sessionId: SESSION,
result: { kind: 'journal-row', epoch: epoch1 }
}
})
expect(card.queuedAt!.epoch).not.toBe(epoch1)
expect(card.queuedAt!.epoch).toBe(journal.epoch)
expect(card.queuedAt!.sequence).toBeGreaterThanOrEqual(
journal.submission(m.id)!.acceptedSequence!
)
expect(journal.queuedMessages.pauses()).toEqual([])
await eventually(async () => expect((await rig.submission(m.id))?.handedOverAt).toBeDefined())
await rig.settleAccepted(m.id, 'M')
await eventually(() => expect(rig.compact).toHaveBeenCalledTimes(1))
const compact = rig.compact.mock.calls[0]![0].command
rig.finishCompact()
await eventually(() =>
expect(journal.item(agentJournalItemKey(compact.identity))?.body).toMatchObject({
kind: 'turn',
state: 'completed'
})
)
expect(await rig.submission(m.id)).toMatchObject({ dispatchState: 'accepted' })
} finally {
vi.restoreAllMocks()
await rig.dispose()
}
})
@@ -0,0 +1,384 @@
import { expect, it, vi } from 'vitest'
import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key'
import { structuredAgentSessionQueueRequest } from '../../../renderer/src/components/native-chat/structured-agent-session-queue-request'
import {
createQueuedMessageTestRig,
eventually,
QUEUED_RIG_CALLER
} from './structured-agent-session-queued-message-rig.test-fixture'
import { openRigTurnFor } from './structured-agent-session-queued-rig-turn.test-fixture'
import {
HOST_TEST_SESSION as SESSION,
hostTestOperationId
} from './structured-agent-session-host-test-data'
async function queueCompact(rig: Awaited<ReturnType<typeof createQueuedMessageTestRig>>) {
const id = hostTestOperationId()
const fields = { command: 'compact' as const, delivery: 'queue-if-active' as const }
expect(
await rig.host.conversationCommand(QUEUED_RIG_CALLER, {
envelope: rig.envelope(fields, 'agentSession.conversationCommand', id),
...fields,
userSend: true
})
).toMatchObject({ ok: true, value: { queued: { messageId: id } } })
return id
}
it.each([
{ followUpsQueue: false, alternate: undefined },
{ followUpsQueue: true, alternate: true }
])(
'fresh and stale publication both run C,D,M before later cards (Queue=$followUpsQueue)',
async ({ followUpsQueue, alternate }) => {
vi.useFakeTimers()
try {
for (const fresh of [true, false]) {
const rig = await createQueuedMessageTestRig()
try {
const working = await rig.workingSend()
const compactId = await queueCompact(rig)
const earlier = rig.send('earlier card', 'queue-if-active')
await earlier.result
const delivery = structuredAgentSessionQueueRequest(
{
capability: 'supported',
followUpsQueue,
isWorking: true,
stopping: false,
session: {
queuedMessages: fresh
? [
{
messageId: compactId,
state: 'waiting',
body: {
kind: 'message',
role: 'user',
blocks: [],
command: { name: 'compact' }
},
position: 1
}
]
: []
},
prompts: []
},
alternate
)
const later = rig.send('after compaction and the earlier card', delivery)
expect(await later.result).toMatchObject({
ok: true,
value: fresh ? { queued: {} } : { submission: { clientMessageId: later.id } }
})
const tail = rig.send('newer card', 'queue-if-active')
await tail.result
await vi.advanceTimersByTimeAsync(100)
expect(rig.dispatch).toHaveBeenCalledOnce()
expect(rig.compact).not.toHaveBeenCalled()
await rig.settleAccepted(working, 'first')
await eventually(() => expect(rig.compact).toHaveBeenCalledOnce())
expect(rig.dispatch).toHaveBeenCalledOnce()
const command = rig.compact.mock.calls[0]![0].command
const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal
rig.finishCompact()
await eventually(() =>
expect(journal.item(agentJournalItemKey(command.identity))?.body).toMatchObject({
kind: 'turn',
state: 'completed'
})
)
await eventually(async () =>
expect((await rig.handoff(earlier.id))?.handedOverAt).toBeDefined()
)
const earlierId = await rig.handoffId(earlier.id)
expect(rig.dispatch.mock.calls.at(-1)?.[0].clientMessageId).toBe(earlierId)
await rig.settleAccepted(earlierId, 'earlier')
await eventually(async () =>
expect(
fresh
? (await rig.handoff(later.id))?.handedOverAt
: (await rig.submission(later.id))?.handedOverAt
).toBeDefined()
)
const laterId = fresh ? await rig.handoffId(later.id) : later.id
expect(rig.dispatch.mock.calls.at(-1)?.[0].clientMessageId).toBe(laterId)
expect(await rig.handoff(tail.id)).toBeUndefined()
await rig.settleAccepted(laterId, 'later')
expect(await rig.submission(laterId)).toMatchObject({ dispatchState: 'accepted' })
await eventually(async () =>
expect((await rig.handoff(tail.id))?.handedOverAt).toBeDefined()
)
expect(rig.dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([
working,
earlierId,
laterId,
await rig.handoffId(tail.id)
])
} finally {
await rig.dispose()
}
}
} finally {
vi.useRealTimers()
}
}
)
it('explicit card Send after handover entry preserves the committed input then card, once each', async () => {
vi.useFakeTimers()
const rig = await createQueuedMessageTestRig()
try {
const working = await rig.workingSend()
await openRigTurnFor(rig, working)
const card = rig.send('explicit card', 'queue-if-active')
await card.result
const prepared = Promise.withResolvers<void>()
const release = Promise.withResolvers<void>()
const writes: string[] = []
let held = false
rig.dispatch.mockImplementation(async (input) => {
if (
!held &&
input.body.blocks.some((block) => block.type === 'text' && block.text === 'preparing input')
) {
held = true
prepared.resolve()
await release.promise
}
writes.push(input.clientMessageId)
return { state: 'admitted' }
})
const message = rig.send('preparing input')
await message.result
await prepared.promise
expect((await rig.submission(message.id))?.handedOverAt).toBeDefined()
const sent = await rig.sendNow(card.id)
expect(sent).toMatchObject({ ok: true, value: { submission: { queuedMessageId: card.id } } })
release.resolve()
await eventually(async () => expect((await rig.handoff(card.id))?.handedOverAt).toBeDefined())
const cardId = await rig.handoffId(card.id)
await eventually(() => expect(writes).toEqual([message.id, cardId]))
expect(rig.dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([
working,
message.id,
cardId
])
await rig.settleAccepted(cardId, 'card')
await rig.settleAccepted(message.id, 'message')
await rig.settleAccepted(working, 'working')
expect(await rig.submission(message.id)).toMatchObject({ dispatchState: 'accepted' })
} finally {
await rig.dispose()
vi.useRealTimers()
}
})
it('removing C and explicitly sending D keeps forced M queued until W actually ends', async () => {
vi.useFakeTimers()
const rig = await createQueuedMessageTestRig()
try {
const working = await rig.workingSend()
await openRigTurnFor(rig, working)
const compactId = await queueCompact(rig)
const earlier = rig.send('D', 'queue-if-active')
await earlier.result
const message = rig.send('forced M without delivery')
await message.result
await rig.deleteQueued(compactId)
await rig.sendNow(earlier.id)
await eventually(async () =>
expect((await rig.handoff(earlier.id))?.handedOverAt).toBeDefined()
)
await rig.settleAccepted(await rig.handoffId(earlier.id), 'explicit-D')
await vi.advanceTimersByTimeAsync(100)
expect((await rig.submission(message.id))?.handedOverAt).toBeUndefined()
expect(await rig.submission(message.id)).toMatchObject({ dispatchState: 'pending' })
expect(rig.compact).not.toHaveBeenCalled()
await rig.settleAccepted(working, 'W')
await openRigTurnFor(rig, working, 'interrupted')
await eventually(async () =>
expect((await rig.submission(message.id))?.handedOverAt).toBeDefined()
)
await rig.settleAccepted(message.id, 'M')
expect(await rig.submission(message.id)).toMatchObject({ dispatchState: 'accepted' })
} finally {
await rig.dispose()
vi.useRealTimers()
}
})
it('reselects after the external reservation wait without rejecting the original pending input', async () => {
const rig = await createQueuedMessageTestRig()
let release = () => {}
try {
const working = await rig.workingSend()
await openRigTurnFor(rig, working)
const card = rig.send('explicit before entry', 'queue-if-active')
await card.result
const { conversationDelivery, serialize } = rig.host.collaboratorsForTests()
const order = conversationDelivery.loop['deps'].deliveryOrder
const held = order.reserve(SESSION)
release = held.release
const entered = Promise.withResolvers<void>()
const before = order.beforeSend.bind(order)
const waiting = vi.spyOn(order, 'beforeSend').mockImplementation((...args) => {
entered.resolve()
return before(...args)
})
const message = rig.send('original M still pending')
await message.result
await entered.promise
expect(await rig.submission(message.id)).toMatchObject({ dispatchState: 'pending' })
expect((await rig.submission(message.id))?.handedOverAt).toBeUndefined()
await rig.sendNow(card.id)
await serialize(SESSION, async () => {})
release()
await eventually(async () => expect((await rig.handoff(card.id))?.handedOverAt).toBeDefined())
const cardId = await rig.handoffId(card.id)
await eventually(async () =>
expect((await rig.submission(message.id))?.handedOverAt).toBeDefined()
)
expect(rig.dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([
working,
cardId,
message.id
])
await rig.settleAccepted(cardId, 'card')
await rig.settleAccepted(message.id, 'M')
expect(await rig.submission(message.id)).toMatchObject({ dispatchState: 'accepted' })
waiting.mockRestore()
} finally {
release()
await rig.dispose()
}
})
it('streams N and 10N frames without rescanning queue/history or waking a forced input', async () => {
const rig = await createQueuedMessageTestRig()
try {
const working = await rig.workingSend()
await openRigTurnFor(rig, working)
const compactId = await queueCompact(rig)
const message = rig.send('forced M')
await message.result
const { sessions, conversationDelivery, serialize } = rig.host.collaboratorsForTests()
const journal = sessions.get(SESSION)!.journal
const scope = { kind: 'thread' as const }
for (let index = 0; index < 100; index++) {
await journal.appendItem(
{ provider: 'orca', clientMessageId: `old-${index}` },
{ kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'history' }] },
{ fence: 1, turnScope: scope }
)
}
await serialize(SESSION, async () => {})
const pauses = vi.spyOn(journal.queuedMessages, 'pauses')
const visits = vi.spyOn(journal, 'visitItems')
const wake = vi.spyOn(conversationDelivery.loop, 'wake')
let revision = 0
for (const count of [10, 100]) {
pauses.mockClear()
visits.mockClear()
wake.mockClear()
for (let index = 0; index < count; index++) {
rig.providerEvents().appendItem(
{ provider: 'orca', clientMessageId: 'stream' },
{
kind: 'message',
role: 'assistant',
blocks: [{ type: 'text', text: `token-${revision++}` }]
},
{ turnScope: scope }
)
await rig.host.flushStreamedEvents(SESSION)
await serialize(SESSION, async () => {})
}
expect(pauses).not.toHaveBeenCalled()
expect(visits).not.toHaveBeenCalled()
expect(wake).not.toHaveBeenCalled()
expect(journal.submission(message.id)?.handedOverAt).toBeUndefined()
}
pauses.mockRestore()
visits.mockRestore()
wake.mockRestore()
await rig.settleAccepted(working, 'W')
await openRigTurnFor(rig, working, 'interrupted')
await eventually(() => expect(rig.compact).toHaveBeenCalledOnce())
rig.finishCompact()
await eventually(async () =>
expect((await rig.submission(message.id))?.handedOverAt).toBeDefined()
)
await rig.settleAccepted(message.id, 'M')
expect(await rig.handoff(compactId)).toMatchObject({ dispatchState: 'accepted' })
expect(await rig.submission(message.id)).toMatchObject({ dispatchState: 'accepted' })
} finally {
await rig.dispose()
}
})
it.each(['resume', 'new-send'] as const)(
'restart holds C,D,M in their place until %s releases the queue',
async (release) => {
const rig = await createQueuedMessageTestRig({
restartable: true,
probeOwner: async () => ({ outcome: 'pid-absent' })
})
try {
await rig.workingSend()
const compactId = await queueCompact(rig)
const earlier = rig.send('D before held M', 'queue-if-active')
await earlier.result
const message = rig.send('M kept after restart')
await message.result
const tail = rig.send('later card', 'queue-if-active')
await tail.result
await rig.restartHostProcess()
expect(await rig.submission(message.id)).toMatchObject({
dispatchState: 'rejected',
keptAsQueuedMessageId: message.id
})
expect((await rig.drafts()).map((card) => card.messageId)).toEqual([
compactId,
earlier.id,
message.id,
tail.id
])
expect(rig.compact).not.toHaveBeenCalled()
expect(await rig.handoff(message.id)).toBeUndefined()
if (release === 'resume') {
expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } })
} else {
const next = rig.send('N releases reopen only when accepted')
expect(await next.result).toMatchObject({ ok: true })
await eventually(async () =>
expect((await rig.submission(next.id))?.handedOverAt).toBeDefined()
)
expect(rig.compact).not.toHaveBeenCalled()
await rig.settleAccepted(next.id, 'N')
}
await eventually(() => expect(rig.compact).toHaveBeenCalledOnce())
const command = rig.compact.mock.calls[0]![0].command
rig.finishCompact()
const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal
await eventually(() =>
expect(journal.item(agentJournalItemKey(command.identity))?.body).toMatchObject({
kind: 'turn',
state: 'completed'
})
)
for (const id of [earlier.id, message.id, tail.id]) {
await eventually(async () => expect((await rig.handoff(id))?.handedOverAt).toBeDefined())
const handed = await rig.handoffId(id)
await rig.settleAccepted(handed, id)
expect(await rig.submission(handed)).toMatchObject({ dispatchState: 'accepted' })
}
expect(
rig.dispatch.mock.calls.filter(([input]) => input.clientMessageId === message.id)
).toHaveLength(0)
} finally {
await rig.dispose()
}
}
)
@@ -0,0 +1,297 @@
import { journalInputOrder } from '../agent-session-journal/journal-input-order'
import { writeLegacyGapFixture } from './structured-agent-session-legacy-gap.test-support'
import type { JournalReducerState } from '../agent-session-journal/journal-reducer'
import { expect, it, vi } from 'vitest'
import { AgentSessionJournal } from '../agent-session-journal/journal-store'
import {
loadTestJournal,
openTestJournalHostDatabase
} from '../agent-session-journal/journal-host-database-test-support'
import { selectStructuredAgentSessionInput } from './structured-agent-session-input-selection'
import {
createQueuedMessageTestRig,
eventually
} from './structured-agent-session-queued-message-rig.test-fixture'
import { openRigTurnFor } from './structured-agent-session-queued-rig-turn.test-fixture'
import {
HOST_TEST_SESSION as SESSION,
hostTestMessage
} from './structured-agent-session-host-test-data'
it.each([
{ mode: 'mixed', partial: false },
{ mode: 'mixed', partial: true },
{ mode: 'ordinary', partial: true },
{ mode: 'moved', partial: true },
{ mode: 'ordinary-first', partial: false }
])(
'preserves personal recovery and committed prefix $mode partial=$partial',
async ({ mode, partial }) => {
const rig = await createQueuedMessageTestRig({
restartable: true,
probeOwner: async () => ({ outcome: 'pid-absent' })
})
let release = () => {}
let blocked = ''
const resolve = AgentSessionJournal.prototype.resolveDispatch
const fault = vi
.spyOn(AgentSessionJournal.prototype, 'resolveDispatch')
.mockImplementation(function (this: AgentSessionJournal, ...args) {
return args[0].clientMessageId === blocked
? Promise.reject(new Error('keep and fallback fail'))
: resolve.apply(this, args)
})
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
try {
const w = await rig.workingSend()
await openRigTurnFor(rig, w)
const { sessions, serialize, conversationDelivery } = rig.host.collaboratorsForTests()
release = conversationDelivery.loop['deps'].deliveryOrder.reserve(SESSION).release
const journal = sessions.get(SESSION)!.journal
const mixed = mode === 'mixed'
const first = mode === 'ordinary-first' ? rig.send('M1') : undefined
await first?.result
await serialize(SESSION, async () => {
await journal.queuedMessages.insert({
messageId: 'C',
body:
mixed || first
? { ...hostTestMessage('/compact'), command: { name: 'compact' } }
: hostTestMessage('C')
})
if (mixed || first) {
await journal.queuedMessages.insert({ messageId: 'D', body: hostTestMessage('D') })
}
})
const m1 = first ?? rig.send('M1')
await m1.result
if (mixed) {
await rig.deleteQueued('C')
}
const m2 = rig.send('M2')
await m2.result
expect(journal.inputOrder.forced(m1.id)).toBe(mixed)
expect(journal.inputOrder.forced(m2.id)).toBe(Boolean(first))
expect(journal.submission(m1.id)?.handedOverAt).toBeUndefined()
expect(journal.submission(m2.id)?.handedOverAt).toBeUndefined()
if (partial) {
blocked = m2.id
}
await rig.restartHostProcess()
release()
expect(await rig.submission(m1.id)).toMatchObject({ keptAsQueuedMessageId: m1.id })
if (partial) {
expect(await rig.submission(m2.id)).toMatchObject({ dispatchState: 'pending' })
const reopened = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal
expect(
reopened.queuedMessages
.list()
.filter((c) => c.state === 'waiting')
.map((c) => c.messageId)
).toEqual(mixed ? ['D', m1.id] : [m1.id, 'C'])
if (mode === 'moved') {
expect(
await rig.host
.collaboratorsForTests()
.serialize(SESSION, () =>
reopened.queuedMessages.move({ messageId: 'C', beforeMessageId: m1.id })
)
).toMatchObject({ status: 'moved' })
}
blocked = ''
await rig.restartHostProcess()
}
expect(await rig.submission(m2.id)).toMatchObject({ keptAsQueuedMessageId: m2.id })
const reopened = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal
const expected = mixed
? ['D', m1.id, m2.id]
: first
? [m1.id, 'C', 'D', m2.id]
: mode === 'moved'
? ['C', m1.id, m2.id]
: [m1.id, m2.id, 'C']
const cards = reopened.queuedMessages.list().filter((c) => c.state === 'waiting')
expect(cards.map((c) => c.messageId)).toEqual(expected)
expect(
cards.every(
(c, i) => Number.isFinite(c.position) && (i === 0 || c.position > cards[i - 1].position)
)
).toBe(true)
expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } })
const actual: string[] = []
for (const id of expected) {
if (first && id === 'C') {
await eventually(() => expect(rig.compact).toHaveBeenCalledOnce())
rig.finishCompact()
actual.push(id)
continue
}
await eventually(async () => expect((await rig.handoff(id))?.handedOverAt).toBeDefined())
const handoff = await rig.handoffId(id)
actual.push((await rig.submission(handoff))!.queuedMessageId!)
await rig.settleAccepted(handoff, `frontier-${actual.length}`)
expect(await rig.submission(handoff)).toMatchObject({ dispatchState: 'accepted' })
}
expect(actual).toEqual(expected)
expect(reopened.queuedMessages.list().filter((c) => c.state === 'waiting')).toEqual([])
expect(reopened.queuedMessages.displayState().inputEntries.size).toBe(0)
expect(rig.dispatch.mock.calls.filter(([i]) => i.clientMessageId === w)).toHaveLength(1)
} finally {
blocked = ''
release()
fault.mockRestore()
warn.mockRestore()
await rig.dispose()
}
}
)
it('retains an automatic unentered link through two production replacements and its real reservation release', async () => {
const rig = await createQueuedMessageTestRig()
let release = () => {}
try {
const w = await rig.workingSend()
await openRigTurnFor(rig, w)
const { sessions, serialize, conversationDelivery } = rig.host.collaboratorsForTests()
const journal = sessions.get(SESSION)!.journal
await serialize(SESSION, async () => {
await journal.queuedMessages.insert({
messageId: 'C',
body: { ...hostTestMessage('/compact'), command: { name: 'compact' } }
})
await journal.queuedMessages.insert({
messageId: 'D',
body: hostTestMessage('latest D'),
options: { effort: 'high' }
})
})
const m = rig.send('M')
await m.result
const receipt = rig.store.readCommandReceipt({ kind: 'global' }, m.id)
const intro = journal.inputOrder.lineage.cards.get('D')!.introduced
await rig.deleteQueued('C')
const reservation = conversationDelivery.loop['deps'].deliveryOrder.reserve(SESSION)
release = reservation.release
await rig.settleAccepted(w, 'W')
await openRigTurnFor(rig, w, 'interrupted')
await eventually(async () => expect((await rig.handoff('D'))?.dispatchState).toBe('pending'))
const h = await rig.handoffId('D')
expect((await rig.submission(h))?.handedOverAt).toBeUndefined()
for (let i = 0; i < 2; i++) {
await serialize(SESSION, () =>
journal.replaceEpochItems('unreconcilable_prefix', 1, []).then(() => {})
)
expect(journal.inputOrder.lineage.cards.get('D')!.introduced).toBe(intro)
expect(journal.queuedMessages.get('D')).toMatchObject({
state: 'dispatched',
consumedAs: h,
body: hostTestMessage('latest D'),
options: { effort: 'high' }
})
expect(selectStructuredAgentSessionInput(journal)).toMatchObject({
kind: 'submission',
submission: { clientMessageId: h }
})
const replay = loadTestJournal(rig.root, SESSION)!
expect(replay.damage).toBeNull()
expect(journalInputOrder(replay.state).lineage.cards.get('D')!.introduced).toBe(intro)
expect(
journalInputOrder(replay.state).precedes(
replay.state.queue.cards.get('D')!,
replay.state.submissions.get(m.id)!
)
).toBe(true)
expect(rig.store.readCommandReceipt({ kind: 'global' }, m.id)).toEqual(receipt)
}
release()
await eventually(async () => expect((await rig.submission(h))?.handedOverAt).toBeDefined())
await rig.settleAccepted(h, 'H')
await eventually(async () => expect((await rig.submission(m.id))?.handedOverAt).toBeDefined())
await rig.settleAccepted(m.id, 'M')
expect(rig.dispatch.mock.calls.map(([i]) => i.clientMessageId)).toEqual([w, h, m.id])
expect(await rig.submission(h)).toMatchObject({ dispatchState: 'accepted' })
expect(await rig.submission(m.id)).toMatchObject({ dispatchState: 'accepted' })
expect(journal.queuedMessages.displayState().inputEntries.size).toBe(0)
expect(loadTestJournal(rig.root, SESSION)!.state.inputEntries.size).toBe(0)
} finally {
release()
await rig.dispose()
}
})
it.each([10, 100])(
'preserves initial legacy terminal gaps through replacement and actual recovery N=%i',
async (n) => {
const rig = await createQueuedMessageTestRig({
restartable: true,
legacyInputs: true,
probeOwner: async () => ({ outcome: 'pid-absent' })
})
try {
const original = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal
await rig.host.flushAllStreamedEvents()
writeLegacyGapFixture(rig.root, original, n)
const check = (state: JournalReducerState) => {
const m = state.submissions.get('legacy-M')!
expect.soft(journalInputOrder(state).forced('legacy-M')).toBe(true)
expect
.soft(
['C', 'D'].map((id) => journalInputOrder(state).precedes(state.queue.cards.get(id)!, m))
)
.toEqual([true, true])
}
check(loadTestJournal(rig.root, SESSION)!.state)
const journal = new AgentSessionJournal({
identity: original['identity'],
database: openTestJournalHostDatabase(rig.root)
})
await journal.open()
for (let pass = 0; pass < 2; pass++) {
await journal.replaceEpochItems('unreconcilable_prefix', 1, [])
check(journal.queuedMessages.displayState())
check(loadTestJournal(rig.root, SESSION)!.state)
}
await journal.close()
rig.crashRestartHostProcess()
await rig.submission('legacy-M')
const reopened = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal
const saved = reopened.queuedMessages
.list()
.filter((c) => c.state === 'waiting')
.map((c) => c.messageId)
expect.soft(saved).toEqual(['C', 'D', 'legacy-M'])
expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } })
const delivered: string[] = []
for (const id of saved) {
if (id === 'C') {
await eventually(() => expect(rig.compact).toHaveBeenCalledOnce())
rig.finishCompact()
delivered.push(id)
continue
}
await eventually(async () => expect((await rig.handoff(id))?.handedOverAt).toBeDefined())
const h = await rig.handoffId(id)
await rig.settleAccepted(h, id)
delivered.push(id)
expect(await rig.submission(h)).toMatchObject({
dispatchState: 'accepted',
queuedMessageId: id
})
}
expect.soft(delivered).toEqual(['C', 'D', 'legacy-M'])
expect
.soft(
rig.dispatch.mock.calls.map(
([input]) => reopened.submission(input.clientMessageId)?.queuedMessageId
)
)
.toEqual(['D', 'legacy-M'])
expect(reopened.queuedMessages.list().filter((c) => c.state === 'waiting')).toEqual([])
expect(reopened.queuedMessages.displayState().inputEntries.size).toBe(0)
expect(loadTestJournal(rig.root, SESSION)!.state.inputEntries.size).toBe(0)
expect([...reopened.inputOrder.lineage.cards.values()].every((c) => !c.baseline)).toBe(true)
} finally {
await rig.dispose()
}
}
)
@@ -0,0 +1,100 @@
import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types'
import type { AgentSessionJournal } from '../agent-session-journal/journal-store'
import type { QueuedMessageRow } from '../agent-session-journal/queued-message-table'
import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work'
import { pendingPromptsAllUnanswerableHere } from '../../../shared/agent-session-approval-subject'
export type StructuredInputSelection =
| { kind: 'card'; card: QueuedMessageRow }
| { kind: 'submission'; submission: AgentJournalSubmission }
| { kind: 'none' }
/** One live checkpoint owns automatic input order; debt alone is not execution. */
export function selectStructuredAgentSessionInput(
journal: AgentSessionJournal,
currentInput?: string,
unanswerable = false
): StructuredInputSelection {
const queued = journal.inputOrder.firstQueued()
const current = currentInput ? journal.submission(currentInput) : undefined
const selected =
current?.dispatchState === 'pending' &&
(!queued || current.submittedSequence! < queued.submittedSequence!)
? current
: queued
const nextCard = journal.queuedMessages.nextInputCard()
if (!selected) {
return nextCard ? { kind: 'card', card: nextCard } : { kind: 'none' }
}
// Explicit card Send intentionally overrides the automatic queue place.
const override = journal.inputOrder.explicitSend()
if (override) {
return { kind: 'submission', submission: override }
}
if (unanswerable) {
return { kind: 'submission', submission: selected }
}
const predecessors = journal.inputOrder.predecessors(
journal.queuedMessages.inputState(),
selected.clientMessageId
)
const handoff = predecessors.handoff?.consumedAs
if (handoff) {
const submission = journal.submission(handoff)
if (submission) {
return { kind: 'submission', submission }
}
}
if (predecessors.head) {
return nextCard ? { kind: 'card', card: nextCard } : { kind: 'none' }
}
return { kind: 'submission', submission: selected }
}
export function selectedStructuredSubmission(
journal: AgentSessionJournal
): AgentJournalSubmission | undefined {
const selected = selectStructuredAgentSessionInput(journal)
return selected.kind === 'submission' ? selected.submission : undefined
}
export function selectableStructuredSubmission(
journal: AgentSessionJournal,
work: StructuredAgentSessionCurrentWork | null,
currentInput?: string
): AgentJournalSubmission | undefined {
if (!journal.inputOrder.firstQueued() && currentInput === undefined) {
return undefined
}
const unanswerable = work !== null && pendingPromptsAllUnanswerableHere(work.actionablePrompts())
const selected = selectStructuredAgentSessionInput(journal, currentInput, unanswerable)
if (selected.kind !== 'submission') {
return undefined
}
const { submission } = selected
if (unanswerable) {
return submission
}
return journal.inputOrder.forced(submission.clientMessageId) &&
work &&
structuredInputExecutionWorking(journal, work, currentInput)
? undefined
: submission
}
/** Provider-held/uncertain work still blocks cards; deferred, proven-unhanded inputs do not. */
export function structuredInputExecutionWorking(
journal: AgentSessionJournal,
work: StructuredAgentSessionCurrentWork,
currentInput?: string
): boolean {
if (work.activeTurnId() !== null) {
return true
}
for (const entry of journal.inputOrder.enteredSubmissions()) {
if (entry.clientMessageId !== currentInput && work.owesSend(entry)) {
return true
}
}
return false
}
@@ -0,0 +1,81 @@
import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding'
import { insertCommandReceiptIfAbsent } from '../agent-session-journal/command-receipt-table'
import { writeCommandReceiptTestRecord } from '../agent-session-journal/command-receipt-test-support'
import {
insertTestJournalRow,
writeTestLegacyTerminalGaps,
openTestJournalHostDatabase
} from '../agent-session-journal/journal-host-database-test-support'
import type { AgentSessionJournal } from '../agent-session-journal/journal-store'
import type { JournalSubmissionRow } from '../agent-session-journal/journal-row-schema'
import { hostTestMessage } from './structured-agent-session-host-test-data'
/** Actual unannotated initial-source writer shape, including terminal direct inputs between consumers. */
export function writeLegacyGapFixture(root: string, journal: AgentSessionJournal, n: number): void {
const database = openTestJournalHostDatabase(root)
const { sessionId } = journal['identity']
const { epoch, sequence } = journal.cursor()
let seq = sequence
const base = () => ({ v: 3, epoch, seq: ++seq, fence: 1, ts: 1 })
for (const id of ['C', 'D']) {
const coordinate = { epoch, sequence: seq + 1 }
insertTestJournalRow(database.db, sessionId, {
...base(),
kind: 'queue',
messageId: id,
change: {
op: 'queued',
body:
id === 'C'
? { ...hostTestMessage('/compact'), command: { name: 'compact' } }
: hostTestMessage(id),
position: id === 'C' ? 0 : 1,
queuedAt: coordinate
}
})
}
const submission = (id: string) => {
const row: JournalSubmissionRow = {
...base(),
kind: 'submission' as const,
clientMessageId: id,
payloadFingerprint: id,
body: hostTestMessage(id),
providerHandle: agentSessionJournalProviderHandle(journal['identity']),
handoverRecorded: true,
origin: 'client' as const,
options: { effort: 'high' }
}
insertTestJournalRow(database.db, sessionId, row)
return row.seq
}
seq = writeTestLegacyTerminalGaps(
database.db,
sessionId,
{
epoch,
sequence: seq,
fence: 1,
providerHandle: agentSessionJournalProviderHandle(journal['identity'])
},
n
)
const acceptedSequence = submission('legacy-M')
database.transaction((db) => {
writeCommandReceiptTestRecord(db, sessionId)
insertCommandReceiptIfAbsent(
db,
{ kind: 'global' },
{
operationId: 'legacy-M',
sessionId,
callerKey: 'fixture',
method: 'agentSession.send',
fingerprint: 'legacy-M',
status: 'accepted',
acceptedAt: 1,
result: { kind: 'journal-row', epoch, sequence: acceptedSequence }
}
)
})
}
@@ -2,7 +2,8 @@ import { structuredAgentSessionSendOpeningTurn } from '../../../shared/structure
import type { AgentSessionJournal } from '../agent-session-journal/journal-store'
import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types'
import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work'
import { oldestQueuedSubmission } from './structured-agent-session-start-failure-row'
import type { oldestQueuedSubmission } from './structured-agent-session-start-failure-row'
import { selectableStructuredSubmission } from './structured-agent-session-input-selection'
/** Read opening-turn evidence from the journal fold without a snapshot. */
export function structuredAgentSessionSendOpeningTurnInJournal(
@@ -25,9 +26,9 @@ export function structuredAgentSessionSendOpeningTurnInJournal(
export function structuredAgentSessionNextHandover(
session: Pick<StructuredAgentSessionHostSession, 'journal'>,
fence: number,
work?: Pick<StructuredAgentSessionCurrentWork, 'publishedSubmission'> | null
work?: StructuredAgentSessionCurrentWork | null
): ReturnType<typeof oldestQueuedSubmission> {
return structuredAgentSessionSendOpeningTurnInJournal(session.journal, fence, work)
? undefined
: oldestQueuedSubmission(session)
: selectableStructuredSubmission(session.journal, work ?? null)
}
@@ -1,3 +1,9 @@
import { JournalWriteProjection } from '../agent-session-journal/journal-write-projection'
import { encodeJournalState } from '../agent-session-journal/journal-state-codec'
import { journalOutcomeSource } from '../agent-session-journal/journal-outcome-format'
import { LegacyQueuedCards } from '../agent-session-journal/journal-queue-import'
import { openRigTurnFor } from './structured-agent-session-queued-rig-turn.test-fixture'
import { loadTestJournal } from '../agent-session-journal/journal-host-database-test-support'
import { expect, it, vi } from 'vitest'
import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key'
import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire'
@@ -60,7 +66,8 @@ it.each([100, 1_000])('raw selection reuse: %i waiting cards', async (size) => {
const turn = { provider: 'codex' as const, threadId: 'progress', turnId: 't', ordinal: 0 }
await journal.appendItem(turn, message('progress'), threadWrite)
feed.publish(SESSION, journal)
expect(visits).toBe(2 * size)
// Provider progress cannot change a legacy card’s exact handoff classification.
expect(visits).toBe(0)
expect(journal.publicRead().normalizationKey).toBe('')
expect(journal.queuedMessages.list().filter((c) => c.state === 'waiting')).toHaveLength(size)
for (const events of received) {
@@ -205,3 +212,232 @@ it('a rewind while the import has failed moves the legacy cards first: a refused
await nothingSentTo('edited')
expect(legacyIds()).toEqual([])
})
it.each([
{ importNext: false, format1: false },
{ importNext: true, format1: false },
{ importNext: false, format1: true }
])(
'persists fallback identity through recovery (next import=$importNext, format1=$format1)',
async ({ importNext, format1 }) => {
const rig = fixture.rig
await rig.host.close(SESSION, 'evict')
const seed = journalFor(SESSION)
await seed.open()
await seed.queuedMessages.insert({ messageId: 'C', body: message('/compact') })
await seed.close()
if (format1) {
db().exec(
"DELETE FROM journal_outcomes WHERE field IN ('inputEntries','inputCards'); UPDATE journal_outcomes SET format=1; UPDATE journal_outcome_epochs SET format=1"
)
}
seedLegacy(SESSION, [['D', 'waiting', 'original D']])
db().prepare('UPDATE queued_messages SET position = 2 WHERE message_id = ?').run('D')
db().exec(`CREATE TEMP TRIGGER overlay_import_failure BEFORE DELETE ON queued_messages
BEGIN SELECT RAISE(ABORT, 'import failure'); END`)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const importAtOpen = LegacyQueuedCards.prototype.importAtOpen
const fault = vi
.spyOn(LegacyQueuedCards.prototype, 'importAtOpen')
.mockImplementation(function (this: LegacyQueuedCards) {
return importAtOpen
.call(this)
.finally(() => db().exec('DROP TRIGGER IF EXISTS overlay_import_failure'))
})
try {
const w = await rig.workingSend()
await openRigTurnFor(rig, w)
await rig.settleAccepted(w, 'W')
const live = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal
expect(journalOutcomeSource(db(), SESSION, live.epoch)?.format).toBe(2)
expect(live.queuedMessages.get('C')?.state).toBe('waiting')
expect(legacyIds()).toEqual(['D'])
live.publicRead()
const m = rig.send('original M', undefined, { carried: { effort: 'high' } })
const result = await m.result
expect(result).toMatchObject({ ok: true, value: { submission: { clientMessageId: m.id } } })
const receipt = rig.store.readCommandReceipt({ kind: 'global' }, m.id)
expect(live.inputOrder.forced(m.id)).toBe(true)
expect(live.submission(m.id)?.handedOverAt).toBeUndefined()
expect(live.inputOrder.precedes(live.queuedMessages.get('D')!, live.submission(m.id)!)).toBe(
true
)
if (!importNext) {
db().exec(`CREATE TEMP TRIGGER overlay_import_failure BEFORE DELETE ON queued_messages
BEGIN SELECT RAISE(ABORT, 'import failure'); END`)
}
await rig.host.flushAllStreamedEvents()
rig.crashRestartHostProcess()
await rig.submission(m.id)
const reopened = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal
const order = reopened.queuedMessages
.list()
.filter((c) => c.state === 'waiting')
.map((c) => c.messageId)
// Let the complete production iteration expose even the causal wrong C,M,D ordering.
expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } })
const delivered: string[] = []
for (const id of order) {
if (id === 'C') {
await eventually(() => expect(rig.compact).toHaveBeenCalledOnce())
rig.finishCompact()
} else {
await eventually(async () => expect((await rig.handoff(id))?.handedOverAt).toBeDefined())
const h = await rig.handoffId(id)
const calls = rig.dispatch.mock.calls.filter(([input]) => input.clientMessageId === h)
expect(calls).toHaveLength(1)
expect(calls[0]![0].body).toEqual(message(id === 'D' ? 'original D' : 'original M'))
if (id === m.id) {
expect(calls[0]![0].executionOptions).toMatchObject({ effort: 'high' })
}
await rig.settleAccepted(h, id)
expect(await rig.submission(h)).toMatchObject({
dispatchState: 'accepted',
queuedMessageId: id
})
}
delivered.push(id)
}
expect(rig.store.readCommandReceipt({ kind: 'global' }, m.id)).toEqual(receipt)
expect(reopened.item(agentJournalSubmissionKey(m.id))?.body).toEqual(message('original M'))
expect(reopened.submission(m.id)?.options).toEqual({ effort: 'high' })
expect(reopened.queuedMessages.list().filter((c) => c.state === 'waiting')).toEqual([])
const restored = loadTestJournal(rig.root, SESSION)!
expect(restored.damage).toBeNull()
expect(restored.state.inputEntries.size).toBe(0)
expect(restored.state.inputCards.size).toBe(0)
expect(reopened['state'].inputCards.size).toBe(0)
expect(rig.dispatch.mock.calls.filter(([input]) => input.clientMessageId === w)).toHaveLength(
1
)
expect(delivered).toEqual(['C', 'D', m.id])
} finally {
db().exec('DROP TRIGGER IF EXISTS overlay_import_failure')
fault.mockRestore()
warn.mockRestore()
}
}
)
it.each([
{ readFirst: false, legacyCommand: false },
{ readFirst: true, legacyCommand: false },
{ readFirst: false, legacyCommand: true }
])(
'keeps reads derivative and persists fallback (public=$readFirst, legacyCommand=$legacyCommand)',
async ({ readFirst, legacyCommand }) => {
const seed = journalFor('overlay-direct')
await seed.open()
if (!legacyCommand) {
await seed.queuedMessages.insert({ messageId: 'C', body: message('/compact') })
}
await seed.close()
seedLegacy(
'overlay-direct',
legacyCommand
? [
['C', 'waiting', '/compact'],
['D', 'waiting', 'D']
]
: [['D', 'waiting', 'D']]
)
db().exec(`CREATE TEMP TRIGGER overlay_direct_import_failure BEFORE DELETE ON queued_messages
BEGIN SELECT RAISE(ABORT, 'import failure'); END`)
const journal = journalFor('overlay-direct')
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
try {
await journal.open()
const state = journal['state']
const before = encodeJournalState(state)
if (readFirst) {
journal.publicRead()
journal.queuedMessages.nextInputCard()
expect(encodeJournalState(state)).toEqual(before)
JournalWriteProjection.verifyEachCommit = true
await journal.appendItem(
{ provider: 'orca', clientMessageId: 'unrelated' },
message('unrelated'),
threadWrite
)
}
const target = await journal.appendSubmission({
clientMessageId: 'original-M',
payloadFingerprint: 'original',
body: message('original M'),
handoverRecorded: true,
origin: 'client',
fence: 0,
options: { effort: 'high' }
})
expect(journal.inputOrder.forced('original-M')).toBe(true)
expect(
journal.inputOrder.precedes(
journal.queuedMessages.get('D')!,
journal.submission('original-M')!
)
).toBe(true)
const saved = loadTestJournal(fixture.rig.root, 'overlay-direct')!
expect(saved.damage).toBeNull()
expect(saved.state.inputCards.get('D')).toEqual(journal['state'].inputCards.get('D'))
expect(saved.state.inputCards.has('D')).toBe(true)
expect(
db()
.prepare(
"SELECT COUNT(*) AS n FROM journal_outcomes WHERE field='inputCards' AND entry_key=? AND deleted=0"
)
.get(JSON.stringify('D'))?.n
).toBe(1)
expect(target).toEqual({
epoch: journal.epoch,
sequence: journal.submission('original-M')!.acceptedSequence
})
const introduction = journal['state'].inputCards.get('D')!.introduced
db().exec('DROP TRIGGER overlay_direct_import_failure')
for (let pass = 0; pass < 2; pass++) {
await journal.replaceEpochItems('unreconcilable_prefix', 0, [])
const restored = loadTestJournal(fixture.rig.root, 'overlay-direct')!
expect(restored.damage).toBeNull()
expect(restored.state.inputCards.get('D')?.introduced).toBe(introduction)
expect(
journal.inputOrder.precedes(
journal.queuedMessages.get('D')!,
journal.submission('original-M')!
)
).toBe(true)
expect(restored.state.inputEntries.get('original-M')).toEqual(
journal['state'].inputEntries.get('original-M')
)
}
} finally {
JournalWriteProjection.verifyEachCommit = false
db().exec('DROP TRIGGER IF EXISTS overlay_direct_import_failure')
warn.mockRestore()
await journal.close()
}
}
)
it('discovers real overlay ownership before pruning a genuinely saved native identity', async () => {
const session = 'overlay-saved'
const seed = journalFor(session)
await seed.open()
await seed.queuedMessages.insert({ messageId: 'D', body: message('saved D') })
const saved = seed['state'].inputCards.get('D')
const native = db()
.prepare(
"SELECT value_json FROM journal_outcomes WHERE field='inputCards' AND entry_key=? ORDER BY seq DESC LIMIT 1"
)
.get(JSON.stringify('D'))
expect(native?.value_json).toBe(JSON.stringify(['D', saved]))
await seed.close()
// Independent fixture control: retain the writer's genuine native record, with only legacy body ownership.
db()
.prepare("DELETE FROM journal_outcomes WHERE field='queue.cards' AND entry_key=?")
.run(JSON.stringify('D'))
seedLegacy(session, [['D', 'waiting', 'saved D']])
const restored = loadTestJournal(fixture.rig.root, session)!
expect(restored.damage).toBeNull()
expect(restored.state.queue.cards.has('D')).toBe(false)
expect(restored.state.inputEntries.size).toBe(0)
expect(restored.state.inputCards.get('D')).toEqual(saved)
})
@@ -1,3 +1,10 @@
import * as provenance from '../agent-session-journal/journal-input-provenance'
import { JournalDraftMap } from '../agent-session-journal/journal-transition-collections'
import * as transitions from '../agent-session-journal/journal-reducer-transition'
import * as outcomes from '../agent-session-journal/journal-outcome-codec'
import { JournalInputCardIndex } from '../agent-session-journal/journal-input-card-index'
import { JournalInputLineageOwner } from '../agent-session-journal/journal-input-lineage'
import { encodeJournalState } from '../agent-session-journal/journal-state-codec'
import { expect, it, vi } from 'vitest'
import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types'
import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire'
@@ -5,6 +12,8 @@ import { AgentSessionSubscribers } from './structured-agent-session-subscribers'
import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data'
import {
fixture,
threadWrite,
type Legacy,
db,
handOff,
@@ -134,9 +143,9 @@ it.each([100, 1_000].flatMap((size) => [false, true].map((adopt) => ({ size, ado
expect(journal.queuedMessages.displayState() === display).toBe(false)
expect(journal.snapshot()).not.toBe(snapshot)
feed.publish(SESSION, journal)
// A reopen that retries the failed import writes one card; its outcome write lists that card
// once in the card map and once in the order set: a constant 2, whatever the history size.
expect(work()).toEqual([corrections + (adopt ? 0 : 2), 1, 1])
// Import normalizes the one affected submission and refreshes its derived entered index.
// Each touched map key is listed for encoding and adoption; no other history key is visited.
expect(work()).toEqual([corrections + (adopt ? 0 : 2 * 2), 1, 1])
reset()
await journal.appendItem(
{ provider: 'codex', threadId: 'progress', turnId: 't', ordinal: 0 },
@@ -165,3 +174,314 @@ it.each([100, 1_000].flatMap((size) => [false, true].map((adopt) => ({ size, ado
await journal.close()
}
)
it.each([10, 100])('bounds actual canonical fallback reads/writes N=%i', async (n) => {
const session = 'overlay-scale'
const seed = journalFor(session)
await seed.open()
await seed.queuedMessages.insert({ messageId: 'C', body: message('/compact') })
await seed.close()
seedLegacy(
session,
Array.from({ length: n }, (_, i): Legacy => [`D${i}`, 'waiting', `body${i}`])
)
db().exec(`CREATE TEMP TRIGGER overlay_scale_import_failure BEFORE DELETE ON queued_messages
BEGIN SELECT RAISE(ABORT, 'import failure'); END`)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const journal = journalFor(session)
let indexed = 0
let draftKeys = 0
let receiptQueries = 0
let cohortVisits = 0
let introduced = 0
let frontiers = 0
let mutations = 0
let entries = 0
let cards = 0
let bytes = 0
let drafts: ReturnType<typeof transitions.isolateJournalTransition>[] = []
const index = JournalInputCardIndex.prototype.indexCard
const indexSpy = vi
.spyOn(JournalInputCardIndex.prototype, 'indexCard')
.mockImplementation(function (this: JournalInputCardIndex, ...args) {
indexed++
return index.apply(this, args)
})
const introduce = JournalInputCardIndex.prototype.introduceOverlays
const introduction = vi
.spyOn(JournalInputCardIndex.prototype, 'introduceOverlays')
.mockImplementation(function (this: JournalInputCardIndex, change) {
introduce.call(this, (card) => {
introduced++
change(card)
})
})
const frontier = JournalInputLineageOwner.prototype.restoreFrontier
const frontierSpy = vi
.spyOn(JournalInputLineageOwner.prototype, 'restoreFrontier')
.mockImplementation(function (this: JournalInputLineageOwner, ...args) {
frontiers++
return frontier.apply(this, args)
})
const isolate = transitions.isolateJournalTransition
const isolateSpy = vi
.spyOn(transitions, 'isolateJournalTransition')
.mockImplementation((...args) => {
const draft = isolate(...args)
drafts.push(draft)
return draft
})
const encode = outcomes.encodeJournalOutcomes
const encodeSpy = vi.spyOn(outcomes, 'encodeJournalOutcomes').mockImplementation((...args) => {
const encoded = encode(...args)
for (const value of encoded.values) {
if (value.field === 'inputEntries') {
entries++
}
if (value.field === 'inputCards') {
cards++
}
if (value.field === 'inputEntries' || value.field === 'inputCards') {
bytes += Buffer.byteLength(value.json)
}
}
return encoded
})
const map = Array.prototype.map
const flatMap = Array.prototype.flatMap
const isCohort = (array: unknown[]) => {
const first = array[0]
return (
typeof first === 'object' &&
first !== null &&
'messageId' in first &&
typeof first.messageId === 'string' &&
first.messageId.startsWith('D')
)
}
const maps = vi.spyOn(Array.prototype, 'map').mockImplementation(function (
this: unknown[],
callback,
thisArg
) {
const cohort = isCohort(this)
return map.call(this, (...args) => {
if (cohort) {
cohortVisits++
}
return callback.apply(thisArg, args)
})
})
const flatMaps = vi.spyOn(Array.prototype, 'flatMap').mockImplementation(function (
this: unknown[],
callback,
thisArg
) {
const cohort = isCohort(this)
return flatMap.call(this, (...args) => {
if (cohort) {
cohortVisits++
}
return callback.apply(thisArg, args)
})
})
const keys = JournalDraftMap.prototype.keys
const keySpy = vi.spyOn(JournalDraftMap.prototype, 'keys').mockImplementation(function* (
this: JournalDraftMap<unknown, unknown>
) {
for (const key of keys.call(this)) {
draftKeys++
yield key
}
return undefined
})
const acceptance = provenance.journalInputAcceptance
const receiptSpy = vi
.spyOn(provenance, 'journalInputAcceptance')
.mockImplementation((...args) => {
receiptQueries++
return acceptance(...args)
})
const measurements: Record<string, unknown> = {}
const measure = () => ({
indexed,
draftKeys,
receiptQueries,
cohortVisits,
introduced,
frontiers,
mutations,
entries,
cards,
bytes,
nodes: drafts.reduce((sum, draft) => sum + draft.tracked(), 0),
derived: drafts.reduce((sum, draft) => sum + draft.trackedDerived(), 0)
})
const reset = () => {
indexed =
draftKeys =
receiptQueries =
cohortVisits =
introduced =
frontiers =
mutations =
entries =
cards =
bytes =
0
drafts = []
}
try {
await journal.open()
measurements.discovery = measure()
expect(indexed).toBeLessThanOrEqual(5 * n + 1)
expect(cohortVisits).toBeLessThanOrEqual(8 * n)
expect(frontiers).toBe(1)
expect(cards).toBe(2 * n - 1) // Attempted failed import, never adopted.
expect(introduced).toBe(n) // Failed import prepared this actual cohort, but adopted none.
expect(journal['state'].inputCards.size).toBe(1)
reset()
const set = journal['state'].inputCards.set.bind(journal['state'].inputCards)
const mutation = vi.spyOn(journal['state'].inputCards, 'set').mockImplementation((...args) => {
mutations++
return set(...args)
})
const remove = journal['state'].inputCards.delete.bind(journal['state'].inputCards)
const clear = journal['state'].inputCards.clear.bind(journal['state'].inputCards)
const deletion = vi
.spyOn(journal['state'].inputCards, 'delete')
.mockImplementation((...args) => {
mutations++
return remove(...args)
})
const clearing = vi.spyOn(journal['state'].inputCards, 'clear').mockImplementation(() => {
mutations++
return clear()
})
const before = encodeJournalState(journal['state'])
for (let i = 0; i < n; i++) {
journal.publicRead()
expect(journal.queuedMessages.nextInputCard()?.messageId).toBe('C')
}
expect(encodeJournalState(journal['state'])).toEqual(before)
expect(measure()).toEqual({
indexed: 0,
draftKeys: 0,
receiptQueries: 0,
cohortVisits: 0,
introduced: 0,
frontiers: 0,
mutations: 0,
entries: 0,
cards: 0,
bytes: 0,
nodes: 0,
derived: 0
})
mutation.mockRestore()
deletion.mockRestore()
clearing.mockRestore()
measurements.reads = measure()
await journal.appendItem(
{ provider: 'orca', clientMessageId: 'unrelated' },
message('unrelated'),
threadWrite
)
measurements.firstWrite = measure()
expect(introduced).toBe(n)
expect(cards).toBe(n)
expect(indexed).toBe(n)
expect(cohortVisits).toBe(0)
expect(journal['state'].inputCards.size).toBe(n + 1)
reset()
for (let i = 0; i < n; i++) {
await journal.appendSubmission({
clientMessageId: `M${i}`,
body: message(`M${i}`),
payloadFingerprint: `M${i}`,
origin: 'client',
handoverRecorded: true,
fence: 0
})
expect(journal.inputOrder.forced(`M${i}`)).toBe(true)
expect(
journal.inputOrder.precedes(
journal.queuedMessages.get(`D${n - 1}`)!,
journal.submission(`M${i}`)!
)
).toBe(true)
}
measurements.admissions = measure()
expect({
indexed,
draftKeys,
receiptQueries,
cohortVisits,
introduced,
frontiers,
cards,
entries
}).toEqual({
indexed: 0,
draftKeys: 0,
receiptQueries: 0,
cohortVisits: 0,
introduced: 0,
frontiers: 0,
cards: 0,
entries: n
})
// One input's fixed graph plus at most two AVL paths, each with bounded wrapper fields.
expect(measure().nodes).toBeLessThanOrEqual(160 * n * (1 + Math.ceil(Math.log2(n + 2))))
// Each native entry contains fixed IDs/cursors and one witness, never card bodies/populations.
expect(bytes).toBeLessThanOrEqual(400 * n)
reset()
db().exec('DROP TRIGGER overlay_scale_import_failure')
const old = journal.queuedMessages.get('D0')!
expect(
await journal.queuedMessages.update({
messageId: 'D0',
expectedBodyFingerprint: old.fingerprint,
text: 'latest D'
})
).toMatchObject({ status: 'updated' })
expect(journal.queuedMessages.get('D0')?.body).toEqual(message('latest D'))
expect(journal['state'].inputCards.size).toBe(n + 1)
measurements.edit = measure()
expect({ indexed, cards, cohortVisits, introduced }).toEqual({
indexed: 1,
cards: 0,
cohortVisits: 0,
introduced: 0
})
reset()
await journal.queuedMessages.withdraw(Array.from({ length: n }, (_, i) => `D${i}`))
expect(journal['state'].inputCards.size).toBe(1)
expect(legacyIds(session)).toEqual([])
measurements.retirement = measure()
// One bulk adoption, withdrawals, then current-cohort retirement; not n separate flushes.
expect(cards).toBe(n)
expect(bytes).toBe(4 * n) // Native tombstone null values.
expect(indexed).toBeLessThanOrEqual(3 * n)
expect(cohortVisits).toBeLessThanOrEqual(6 * n)
console.info('overlay-operation-counts', JSON.stringify({ n, measurements }))
expect(fixture.rig.dispatch).not.toHaveBeenCalled()
} finally {
db().exec('DROP TRIGGER IF EXISTS overlay_scale_import_failure')
for (const spy of [
indexSpy,
introduction,
frontierSpy,
isolateSpy,
encodeSpy,
maps,
flatMaps,
keySpy,
receiptSpy,
warn
]) {
spy.mockRestore()
}
await journal.close()
}
})
@@ -21,25 +21,22 @@ import type { AgentSessionTurnContext } from './structured-agent-session-turns'
import { QueuedMessageNotConsumableError } from '../agent-session-journal/queued-message-consume-error'
import type { QueuedMessageRow } from '../agent-session-journal/queued-message-table'
import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types'
import { structuredQueuePauses } from './structured-agent-session-queued-pause'
import { nextSendableQueuedCard } from '../agent-session-journal/queued-message-pause'
import type { StructuredAgentSessionLogger } from './structured-agent-session-logger'
import { agentSessionAttachmentExpiredRefusal } from './structured-agent-session-turns'
import { isAgentSessionAttachmentExpiredError } from '../agent-session-attachments/agent-session-attachment-claims'
import { captureStructuredExecutionOptions } from './structured-agent-session-execution-options'
import type { JournalOperationReceipt } from '../agent-session-journal/journal-row-writer'
import {
selectStructuredAgentSessionInput,
structuredInputExecutionWorking
} from './structured-agent-session-input-selection'
/** Preserve order and holds; only automatic drain stops at edit leases, which still count as admission backlog. */
function oldestActionableQueuedMessage(
journal: Pick<AgentSessionJournal, 'queuedMessages'>,
automatic: boolean
): QueuedMessageRow | null {
const rows = journal.queuedMessages.list()
if (!rows.some((row) => row.state === 'waiting')) {
return null
}
const edited = automatic ? journal.queuedMessages.editLeases.heldIds(rows) : undefined
return nextSendableQueuedCard(structuredQueuePauses(journal), rows, edited)
return journal.queuedMessages.nextInputCard(automatic)
}
/** Host-local gate shared by admission, drain and Send-now; only Send-now overrides working. */
@@ -66,12 +63,20 @@ export function structuredQueueHold(input: StructuredQueueGateInput): Structured
export function nextStructuredQueuedMessage(
input: StructuredQueueGateInput & { journal: AgentSessionJournal }
): QueuedMessageRow | null {
const next = oldestActionableQueuedMessage(input.journal, true)
// Check cheap working state before the prompt scan on every streamed frame.
if (next === null || input.work.working()) {
if (structuredInputExecutionWorking(input.journal, input.work)) {
return null
}
return structuredQueueHold(input) === null ? next : null
const selected = selectStructuredAgentSessionInput(input.journal)
const next = selected.kind === 'card' ? selected.card : null
// Check cheap working state before the prompt scan on every streamed frame.
if (next === null) {
return null
}
return !input.record ||
structuredAgentSessionSendBlock(input.record) ||
input.work.hasActionablePrompt()
? null
: next
}
/** Queue behind work/actionable backlog; held or returned cards cannot trap a fresh send that lifts their pause. */
@@ -215,9 +220,11 @@ export class StructuredAgentSessionQueuedMessageDrain {
}
// Skip serialized work during token streams; settlement/generation-end schedules a fresh gate check.
try {
const work = this.deps.currentWork(sessionId)
if (
oldestActionableQueuedMessage(journal, true) === null ||
this.deps.currentWork(sessionId)?.working() !== false
!work ||
nextStructuredQueuedMessage({ journal, record: this.deps.getRecord(sessionId), work }) ===
null
) {
return
}
@@ -224,7 +224,7 @@ export async function runWithReadyStructuredAgentSessionOperation<T>(
}
context.deliveryOrder.reconcile(
envelope.sessionId,
context.sessions.get(envelope.sessionId)?.journal.submissions() ?? []
context.sessions.get(envelope.sessionId)?.journal
)
const place = context.deliveryOrder.reserve(envelope.sessionId)
return {
@@ -10,6 +10,7 @@ import { isStructuredAgentSessionCommandTurnId } from './structured-agent-sessio
/** Scope active turns to the live generation so ended commands cannot hold delivery waits. */
export type SendSettlementJournal = Pick<AgentSessionJournal, 'submissions' | 'cursor'> & {
activeTurnId: () => string | null
behindCommand?: (clientMessageId: string) => boolean
}
type SettledSend = {
@@ -59,7 +60,10 @@ function settledSend(
until === 'answered'
? submission.dispatchState === 'pending'
: isQueuedAgentJournalSubmission(submission) &&
!(until === 'handed-over-or-behind-command' && runningCommand(journal))
!(
until === 'handed-over-or-behind-command' &&
(runningCommand(journal) || journal.behindCommand?.(clientMessageId))
)
return waiting ? 'pending' : { cursor: journal.cursor(), value: { clientMessageId, submission } }
}
@@ -1,3 +1,4 @@
import { selectedStructuredSubmission } from './structured-agent-session-input-selection'
import type { AgentSessionFailureWordsContext } from '../../../shared/agent-session-failure-words'
import { structuredAgentSessionStartFailure } from './structured-agent-session-failure-text'
import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key'
@@ -145,16 +146,7 @@ export function acceptedNoLaterThan(
export function oldestQueuedSubmission(
session: Pick<StructuredAgentSessionHostSession, 'journal'>
): ReturnType<StructuredAgentSessionHostSession['journal']['submissions']>[number] | undefined {
let oldest: ReturnType<typeof oldestQueuedSubmission>
for (const submission of session.journal.submissions()) {
if (
isQueuedAgentJournalSubmission(submission) &&
(oldest === undefined || (submission.acceptedSequence ?? 0) < (oldest.acceptedSequence ?? 0))
) {
oldest = submission
}
}
return oldest
return selectedStructuredSubmission(session.journal)
}
export async function recordStructuredAgentSessionDeliveryFailure(
@@ -216,13 +216,17 @@ describe('real-store status and queue publications', () => {
subscribers.publish(id, session.journal)
}
}
// Each publish derives queue pauses twice and checks current work independently.
// Three publications retain the former four-read ceiling; cached reads may disappear.
const lists = counts.lists - warm.lists
const submissions = counts.submissions - warm.submissions
expect(lists).toBeLessThanOrEqual(12 * n)
expect(submissions).toBeLessThanOrEqual(12 * n)
expect(counts).toEqual({
...warm,
lists: warm.lists + 12 * n,
listRows: warm.listRows + 12 * n * cards,
submissions: warm.submissions + 12 * n,
submissionRows: warm.submissionRows + 12 * n * sends
lists: warm.lists + lists,
listRows: warm.listRows + lists * cards,
submissions: warm.submissions + submissions,
submissionRows: warm.submissionRows + submissions * sends
})
expect(work.counts).toMatchObject({ views: 9 * n, inventories: 0, clones: 0, decodes: 0 })
expect(statusEvents).toEqual([])
@@ -18,7 +18,10 @@ import {
words,
batch
} from './chat-worker-request-failure-rig.test-fixture'
import { countLegacyFailureDisplay } from './chat-worker-legacy-operation-counts.test-support'
import {
countLegacyFailureDisplay,
emptyLegacyCounts
} from './chat-worker-legacy-operation-counts.test-support'
it.each([100, 1000])(
'bounds the added failure display with %i readable legacy cards',
@@ -91,6 +94,31 @@ it.each([100, 1000])(
db.prepare('SELECT message_id FROM queued_messages WHERE session_id = ?').all(SESSION)
).toHaveLength(n)
expect(journal.queuedMessages.list()).toHaveLength(n)
const cold = structuredClone(ops.counts)
// Three decoded populations: restore, cached source and failed import. The import builds
// N queue rows on an isolated draft before its DELETE fails; none of that work is free.
expect(cold).toEqual({
upstream: {
...emptyLegacyCounts(),
decodedRows: 3 * n,
restoredRows: n,
ownerClassifications: 2 * n,
cardConstructions: 5 * n,
handoffVisits: n,
identityEntries: n,
sourceIdEntries: n,
overlayEntries: 2 * n,
overlayVisits: 2 * n,
priorOverlayVisits: n,
indexUpdates: 4 * n,
overlayIndexUpdates: 2 * n,
pointIndexUpdates: 2 * n,
indexRebuilds: 1
},
publication: emptyLegacyCounts(),
failure: emptyLegacyCounts()
})
ops.reset()
const projection = new StructuredAgentSessionJournalProjections()
r.deliver.mockImplementation(() => ops.publication(() => projection.read(journal, null)))
r.sessions.set(SESSION, { ...r.session, journal })
@@ -109,7 +137,24 @@ it.each([100, 1000])(
})
])
await tick()
const changed = structuredClone(ops.counts)
// First successful native introduction charges N point updates, without reclassification.
expect(changed).toEqual({
upstream: { ...emptyLegacyCounts(), indexUpdates: n, pointIndexUpdates: n },
publication: {
...emptyLegacyCounts(),
displays: 2,
handoffApplications: 2 * n,
itemCopies: 2,
itemEntries: 2 * n + 3,
submissionCopies: 2,
submissionEntries: 2 * (n + 1)
},
failure: emptyLegacyCounts()
})
expect(r.deliver).toHaveBeenCalledTimes(2)
ops.reset()
r.deliver.mockClear()
await batch(journal, 'ordinary-output', [
mutation(
'ordinary-output',
@@ -124,25 +169,22 @@ it.each([100, 1000])(
await tick()
const ordinary = structuredClone(ops.counts)
expect(ordinary).toEqual({
upstream: emptyLegacyCounts(),
publication: {
...emptyLegacyCounts(),
displays: 1,
cardVisits: n,
handoffApplications: n,
itemCopies: 1,
itemEntries: n + 3,
submissionCopies: 1,
submissionEntries: n + 1
},
failure: {
displays: 0,
cardVisits: 0,
itemCopies: 0,
itemEntries: 0,
submissionCopies: 0,
submissionEntries: 0
}
failure: emptyLegacyCounts()
})
expect(r.deliver).toHaveBeenCalledTimes(1)
expect(r.mail()).toHaveLength(0)
ops.reset()
r.deliver.mockClear()
await batch(journal, 'eligible-failure', [
mutation('eligible-turn', {
kind: 'turn',
@@ -156,24 +198,27 @@ it.each([100, 1000])(
await tick()
const failure = structuredClone(ops.counts)
const oneDisplay = {
...emptyLegacyCounts(),
displays: 1,
cardVisits: n,
handoffApplications: n,
itemCopies: 1,
itemEntries: n + 4,
submissionCopies: 1,
submissionEntries: n + 1
}
expect(failure).toEqual({
upstream: emptyLegacyCounts(),
publication: oneDisplay,
failure: {
...emptyLegacyCounts(),
displays: 1,
cardVisits: 0,
itemCopies: 0,
itemEntries: 0,
submissionCopies: 0,
submissionEntries: 0
}
})
expect(r.deliver).toHaveBeenCalledTimes(1)
expect(r.mail()).toHaveLength(1)
expect(r.mail()[0]?.body).toBe(
[
@@ -185,6 +230,8 @@ it.each([100, 1000])(
expect(r.db.getDispatchContextById(r.dispatch().id)?.status).toBe('dispatched')
expect(r.db.getTask(r.dispatch().task_id)?.status).toBe('dispatched')
expect(journal.submission('legacy:0')?.dispatchState).toBe('rejected')
process.stdout.write(`LEGACY_FAILURE_OPERATIONS ${JSON.stringify({ n, ordinary, failure })}\n`)
process.stdout.write(
`LEGACY_FAILURE_OPERATIONS ${JSON.stringify({ n, cold, changed, ordinary, failure })}\n`
)
}
)
@@ -1,9 +1,28 @@
import { vi } from 'vitest'
import * as legacy from '../../native-chat/agent-session-journal/journal-queue-import'
import * as fold from '../../native-chat/agent-session-journal/journal-queue-fold'
import * as stored from '../../native-chat/agent-session-journal/queued-message-stored-row'
import { JournalInputCardIndex } from '../../native-chat/agent-session-journal/journal-input-card-index'
const emptyCounts = () => ({
export const emptyLegacyCounts = () => ({
decodedRows: 0,
restoredRows: 0,
ownerClassifications: 0,
cardConstructions: 0,
handoffVisits: 0,
identityEntries: 0,
sourceIdEntries: 0,
overlayEntries: 0,
overlayVisits: 0,
priorOverlayVisits: 0,
rebuildVisits: 0,
indexUpdates: 0,
overlayIndexUpdates: 0,
rebuildIndexUpdates: 0,
pointIndexUpdates: 0,
indexRebuilds: 0,
displays: 0,
cardVisits: 0,
handoffApplications: 0,
itemCopies: 0,
itemEntries: 0,
submissionCopies: 0,
@@ -27,38 +46,227 @@ function countMapCopies<K, V>(map: Map<K, V>, copy: () => void, visit: () => voi
})
}
/** Delegates the real legacy display, counting its cached array and source Map iterators only. */
function countIteration<T>(array: readonly T[], visit: () => void) {
const iterate = array[Symbol.iterator].bind(array)
Object.defineProperty(array, Symbol.iterator, {
configurable: true,
value: function* () {
for (const value of { [Symbol.iterator]: iterate }) {
visit()
yield value
}
}
})
}
function countValues<K, V>(map: Map<K, V>, visit: () => void) {
const values = map.values.bind(map)
Object.defineProperty(map, 'values', {
configurable: true,
value: function* () {
for (const value of { [Symbol.iterator]: values }) {
visit()
yield value
}
}
})
}
function countMap<T>(array: readonly T[], visit: () => void) {
const map = array.map.bind(array)
Object.defineProperty(array, 'map', {
configurable: true,
value: <U>(
callback: (value: T, index: number, array: readonly T[]) => U,
thisArg?: unknown
): U[] =>
map((value, index, source) => {
visit()
return callback.call(thisArg, value, index, source)
})
})
}
function countFlatMap<T>(
array: readonly T[],
visit: () => void,
result?: (values: readonly unknown[]) => void
) {
const flatMap = array.flatMap.bind(array)
Object.defineProperty(array, 'flatMap', {
configurable: true,
value: <U, This = undefined>(
callback: (this: This, value: T, index: number, array: readonly T[]) => U | readonly U[],
thisArg?: This
): U[] => {
const values = flatMap<U, This>(function (value, index, source) {
visit()
return callback.call(this, value, index, source)
}, thisArg)
result?.(values)
return values
}
})
}
/** Count delegated passes at their real boundaries, including upstream before publication. */
export function countLegacyFailureDisplay() {
const counts = { publication: emptyCounts(), failure: emptyCounts() }
let phase: keyof typeof counts = 'failure'
let displaying = false
const arrays = new WeakSet<readonly unknown[]>()
const counts = {
upstream: emptyLegacyCounts(),
publication: emptyLegacyCounts(),
failure: emptyLegacyCounts()
}
let phase: keyof typeof counts = 'upstream'
let classifying = false
let overlaying = false
let ensuring = false
const overlays = new WeakSet<readonly unknown[]>()
const handoffs = new WeakSet<readonly unknown[]>()
const decode = stored.readStoredQueuedMessageRow
vi.spyOn(stored, 'readStoredQueuedMessageRow').mockImplementation((row) => {
counts[phase].decodedRows++
return decode(row)
})
const restore = legacy.readJournalLegacyCards
vi.spyOn(legacy, 'readJournalLegacyCards').mockImplementation((...args) => {
const measured = counts[phase]
const before = measured.decodedRows
const cards = restore(...args)
measured.restoredRows += measured.decodedRows - before
return cards
})
const construct = fold.queuedCardFromChange
vi.spyOn(fold, 'queuedCardFromChange').mockImplementation((...args) => {
counts[phase].cardConstructions++
return construct(...args)
})
const indexMaps = new WeakSet<object>()
const watchIndexMap = <K, V>(map: Map<K, V>) => {
if (!indexMaps.has(map)) {
indexMaps.add(map)
countValues(map, () => {
if (ensuring) {
counts[phase].rebuildVisits++
} else if (overlaying) {
counts[phase].priorOverlayVisits++
}
})
}
}
const index = JournalInputCardIndex.prototype
const overlay = index.overlay
vi.spyOn(index, 'overlay').mockImplementation(function (
this: JournalInputCardIndex,
state,
rows
) {
watchIndexMap(this['overlays'])
if (!overlays.has(rows)) {
overlays.add(rows)
countMap(rows, () => {
if (overlaying) {
counts[phase].overlayEntries++
}
})
countIteration(rows, () => {
if (overlaying) {
counts[phase].overlayVisits++
}
})
}
overlaying = true
try {
return overlay.call(this, state, rows)
} finally {
overlaying = false
}
})
const update = index.indexCard
vi.spyOn(index, 'indexCard').mockImplementation(function (
this: JournalInputCardIndex,
state,
card
) {
counts[phase].indexUpdates++
if (ensuring) {
counts[phase].rebuildIndexUpdates++
} else if (overlaying) {
counts[phase].overlayIndexUpdates++
} else {
counts[phase].pointIndexUpdates++
}
return update.call(this, state, card)
})
const ensure = index.ensure
vi.spyOn(index, 'ensure').mockImplementation(function (
this: JournalInputCardIndex,
state,
floor
) {
watchIndexMap(this['overlays'])
watchIndexMap(state.queue.cards)
const before = this['index']
ensuring = true
try {
ensure.call(this, state, floor)
} finally {
ensuring = false
}
if (this['index'] !== before) {
counts[phase].indexRebuilds++
}
})
const Original = legacy.LegacyQueuedCards
vi.spyOn(legacy, 'LegacyQueuedCards').mockImplementation(
class extends Original {
constructor(private readonly source: ConstructorParameters<typeof Original>[0]) {
super(source)
const read = this['read'].bind(this)
this['read'] = () => {
const rows = read()
countMap(rows, () => counts[phase].sourceIdEntries++)
countFlatMap(
rows,
() => counts[phase].ownerClassifications++,
(cards) => {
if (classifying) {
countFlatMap(cards, () => {
if (classifying) {
counts[phase].handoffVisits++
}
})
countMap(cards, () => {
if (classifying) {
counts[phase].identityEntries++
}
})
}
}
)
return rows
}
}
override cards() {
const cards = super.cards()
if (!arrays.has(cards)) {
arrays.add(cards)
cards.forEach((card, index) => {
Object.defineProperty(cards, index, {
get: () => {
if (displaying) {
counts[phase].cardVisits++
}
return card
}
})
})
classifying = true
try {
const cards = super.cards()
const handedOff = this['viewed']?.handedOff
if (handedOff && !handoffs.has(handedOff)) {
handoffs.add(handedOff)
countIteration(handedOff, () => counts[phase].handoffApplications++)
}
return cards
} finally {
classifying = false
}
return cards
}
override displayState() {
const previous = phase
if (phase !== 'publication') {
phase = 'failure'
}
const measured = counts[phase]
measured.displays++
const state = this.source.state()
@@ -72,13 +280,12 @@ export function countLegacyFailureDisplay() {
() => measured.submissionCopies++,
() => measured.submissionEntries++
)
displaying = true
try {
return super.displayState()
} finally {
displaying = false
items.mockRestore()
submissions.mockRestore()
phase = previous
}
}
}
@@ -86,16 +293,18 @@ export function countLegacyFailureDisplay() {
return {
counts,
publication: <T>(publish: () => T): T => {
const previous = phase
phase = 'publication'
try {
return publish()
} finally {
phase = 'failure'
phase = previous
}
},
reset: () => {
Object.assign(counts.publication, emptyCounts())
Object.assign(counts.failure, emptyCounts())
for (const value of Object.values(counts)) {
Object.assign(value, emptyLegacyCounts())
}
}
}
}
@@ -115,3 +115,65 @@ it.each(['accepted', 'pending', 'oldest'] as const)(
}
}
)
it('a pending-result reader gets the original direct receipt behind a waiting command before W ends', async () => {
const rig = await createQueuedMessageTestRig()
setStructuredAgentSessionHost(rig.host)
try {
const working = await rig.workingSend()
const commandId = hostTestOperationId()
const fields = { command: 'compact' as const, delivery: 'queue-if-active' as const }
expect(
await rig.host.conversationCommand(QUEUED_RIG_CALLER, {
...fields,
envelope: rig.envelope(fields, 'agentSession.conversationCommand', commandId),
userSend: true
})
).toMatchObject({ ok: true, value: { queued: { messageId: commandId } } })
const body = hostTestMessage('M behind waiting compact')
const messageId = hostTestOperationId()
const params = {
body,
envelope: rig.envelope(
{ body: agentSessionMessagePayload(body) },
'agentSession.send',
messageId
)
}
const result = await sendStructuredAgentSessionForClient(params, {
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the installed-host registry path never reads runtime methods.
runtime: {} as RpcContext['runtime'],
clientId: QUEUED_RIG_CALLER.callerKey,
clientKind: 'runtime',
clientCapabilities: [
STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY,
AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY
]
})
expect(result).toMatchObject({
ok: true,
value: { clientMessageId: messageId, submission: { dispatchState: 'pending' } }
})
expect(await rig.submission(messageId)).not.toHaveProperty('handedOverAt')
const replay = await rig.host.send(QUEUED_RIG_CALLER, { ...params, userSend: true })
expect(replay).toMatchObject({
ok: true,
replayed: true,
value: { submission: { clientMessageId: messageId } }
})
expect(rig.compact).not.toHaveBeenCalled()
await rig.settleAccepted(working, 'W')
await eventually(() => expect(rig.compact).toHaveBeenCalledOnce())
rig.finishCompact()
await eventually(async () =>
expect((await rig.submission(messageId))?.handedOverAt).toBeDefined()
)
await rig.settleAccepted(messageId, 'M')
expect(await rig.submission(messageId)).toMatchObject({ dispatchState: 'accepted' })
expect(
rig.dispatch.mock.calls.filter(([input]) => input.clientMessageId === messageId)
).toHaveLength(1)
} finally {
await rig.dispose()
}
})
@@ -33,7 +33,8 @@ import {
} from './structured-agent-session-rpc.test-fixture'
import {
createQueuedMessageTestRig,
eventually
eventually,
QUEUED_RIG_CALLER
} from '../../../native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture'
import {
HOST_TEST_SESSION as HOST_SESSION,
@@ -492,6 +493,15 @@ describe('agentSession.send reply timing', () => {
let release = () => {}
try {
const working = await rig.workingSend()
const compactId = hostTestOperationId()
const compactFields = { command: 'compact' as const, delivery: 'queue-if-active' as const }
expect(
await rig.host.conversationCommand(QUEUED_RIG_CALLER, {
...compactFields,
envelope: rig.envelope(compactFields, 'agentSession.conversationCommand', compactId),
userSend: true
})
).toMatchObject({ ok: true, value: { queued: { messageId: compactId } } })
const image = imageSend()
expect(
await call('agentSession.send', image.request, STRUCTURED_MOBILE_CLIENT)
@@ -499,8 +509,12 @@ describe('agentSession.send reply timing', () => {
ok: true,
result: { value: { submission: { clientMessageId: image.id } } }
})
expect(await rig.drafts()).toEqual([])
expect((await rig.drafts()).map((card) => card.messageId)).toEqual([compactId])
expect((await rig.submission(image.id))?.handedOverAt).toBeUndefined()
await rig.settleAccepted(working, 'working')
await eventually(() => expect(rig.compact).toHaveBeenCalledOnce())
expect((await rig.submission(image.id))?.handedOverAt).toBeUndefined()
rig.finishCompact()
await eventually(async () => {
expect((await rig.submission(image.id))?.handedOverAt).toBeDefined()
expect(
@@ -1,4 +1,4 @@
import type { StructuredAgentSessionHostCapabilityState } from '@/runtime/structured-agent-session-host-capability'
import type { StructuredAgentSessionHostCapabilityState } from '../../../../shared/structured-agent-session-host-capability'
import {
structuredAgentSessionFollowUpsQueue,
structuredAgentSessionQueueEnabled
@@ -1,4 +1,5 @@
import { AGENT_SESSION_PERMISSION_OWN_RUNTIME_CAPABILITY } from '../../../shared/agent-session-permission-modes-capability'
import type { StructuredAgentSessionHostCapabilityState } from '../../../shared/structured-agent-session-host-capability'
import { useEffect, useState, useSyncExternalStore } from 'react'
import { useAppStore } from '@/store'
import {
@@ -29,8 +30,7 @@ export function structuredAgentSessionHostKey(target: RuntimeClientTarget): stri
return target.kind === 'local' ? 'local' : `environment:${target.environmentId}`
}
/** Unknown before verification or after revocation; a contact outage retains the known build. */
export type StructuredAgentSessionHostCapabilityState = 'unknown' | 'supported' | 'unsupported'
export type { StructuredAgentSessionHostCapabilityState }
/** Displays the existing host status; connection lifecycle owns probing. */
export function useStructuredAgentSessionOwnPermissionState(
@@ -0,0 +1,2 @@
/** Unknown before verification or after revocation; outages retain the known build. */
export type StructuredAgentSessionHostCapabilityState = 'unknown' | 'supported' | 'unsupported'