Preserve saved queue order and recover restart custody on SQL open

This commit is contained in:
Brennan Benson
2026-11-09 23:59:26 -08:00
parent a50f7edc33
commit d9f9f7ddd7
11 changed files with 598 additions and 122 deletions
@@ -0,0 +1,156 @@
import type {
AgentJournalMessageItem,
AgentJournalSubmission
} from '../../../shared/agent-session/journal/agent-session-journal-types'
import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key'
import { agentSessionSendBodyFingerprint } from '../../../shared/agent-session/requests/structured-agent-session-send-mutation'
import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session/requests/agent-session-queued-submission'
import type { JournalReducerState } from './projection/journal-reducer'
import { memoryJournalMap } from './journal-memory-collections'
import { journalInputOrder } from './input-order/journal-input-order'
import { compareQueuedRows } from './input-order/journal-queued-row-index'
import { unsentSendKeptAsCard } from './queued-messages/journal-unsent-send-hold'
import type { QueuedMessageRow } from './queued-messages/queued-message-table'
/** Materialize old places once while unready; ready SQL rows never own a legacy decoder. */
export function placeImportedLegacyQueue(
state: JournalReducerState,
cards: Map<string, QueuedMessageRow>,
personal: (submission: AgentJournalSubmission) => boolean
): void {
const owner = journalInputOrder(state)
const legacy = owner.legacy
if (!legacy) {
return
}
const batch = [...memoryJournalMap(state.submissions).values()]
.filter(isQueuedAgentJournalSubmission)
.sort((a, b) => (a.acceptedSequence ?? 0) - (b.acceptedSequence ?? 0))
.map((submission) => ({
submission,
body: unsentSendKeptAsCard(
submission,
state.items.get(agentJournalSubmissionKey(submission.clientMessageId))?.body ?? null
)
}))
if (batch.length === 0) {
return
}
const placed = legacyQueuePositions(state, cards, batch)
for (const [id, position] of placed) {
const card = cards.get(id)
if (card) {
cards.set(id, { ...card, position })
}
}
for (const { submission, body } of batch) {
if (!body || cards.has(submission.clientMessageId)) {
continue
}
cards.set(submission.clientMessageId, {
messageId: submission.clientMessageId,
body,
position: placed.get(submission.clientMessageId)!,
fingerprint: agentSessionSendBodyFingerprint(state.sessionId, body),
state: 'waiting',
holdReason: null,
returnedReason: null,
returnedRejection: null,
consumedAs: null,
queuedAt: { epoch: state.epoch, sequence: submission.submittedSequence ?? 0 },
deliveryMode: 'queue',
unattended: !personal(submission),
...(submission.options ? { options: submission.options } : {})
})
}
}
function legacyQueuePositions(
state: JournalReducerState,
allCards: ReadonlyMap<string, QueuedMessageRow>,
batch: readonly { submission: AgentJournalSubmission; body: AgentJournalMessageItem | null }[]
): Map<string, number> {
const owner = journalInputOrder(state)
const legacy = owner.legacy!
const cards = [...allCards.values()]
.filter((card) => card.state !== 'withdrawn')
.sort(compareQueuedRows)
const forcedBatch = batch.some(({ submission }) => legacy.forced(submission.clientMessageId))
const explicitHeads = new Map(
batch.flatMap(({ submission }) => {
const id = submission.origin === 'client' ? submission.queuedMessageId : undefined
return id && allCards.get(id)?.consumedAs === submission.clientMessageId
? [[submission.clientMessageId, id] as const]
: []
})
)
const explicit = new Set(explicitHeads.values())
const earlier: string[] = []
const rest: QueuedMessageRow[] = []
for (const card of cards) {
if (explicit.has(card.messageId)) {
continue
}
if (
card.state === 'waiting' &&
!forcedBatch &&
card.queuedAt &&
card.queuedAt.epoch !== state.epoch
) {
earlier.push(card.messageId)
} else {
rest.push(card)
}
}
const at = new Map<number, string[]>()
const head: string[] = []
const firstCut = batch
.flatMap(({ submission }) => legacy.place(submission.clientMessageId)?.context.cut ?? [])
.reduce((a, b) => Math.min(a, b), Infinity)
let previousBoundary = 0
for (let i = 0; i < rest.length; i++) {
if (
rest[i].state === 'waiting' &&
owner.keptCard(rest[i].messageId) &&
(legacy.lineage.cards.get(rest[i].messageId)?.introduced ?? -Infinity) < firstCut
) {
previousBoundary = i + 1
}
}
for (const { submission, body } of batch) {
const id = body ? submission.clientMessageId : explicitHeads.get(submission.clientMessageId)
if (!id || (body && owner.keptCard(id))) {
continue
}
if (!body && previousBoundary > 0) {
head.push(id)
continue
}
const after = legacy.forced(submission.clientMessageId)
? legacy.predecessors(state, submission.clientMessageId).position
: -Infinity
let low = 0,
high = rest.length
while (low < high) {
const mid = (low + high) >>> 1
if (rest[mid].position <= after) {
low = mid + 1
} else {
high = mid
}
}
const boundary = Math.max(previousBoundary, low)
previousBoundary = boundary
const bucket = at.get(boundary) ?? []
bucket.push(id)
at.set(boundary, bucket)
}
const order = [...earlier, ...head]
for (let i = 0; i <= rest.length; i++) {
order.push(...(at.get(i) ?? []))
if (i < rest.length) {
order.push(rest[i].messageId)
}
}
return new Map(order.map((id, index) => [id, index]))
}
@@ -2,6 +2,9 @@ import { journalInputOrder } from './input-order/journal-input-order'
import { memoryJournalMap } from './journal-memory-collections'
import { readAnsweredTurn } from './projection/journal-dispatch-reducer'
import type Database from '../../sqlite/sync-database'
import { journalOutcomeSource } from './outcomes/journal-outcome-format'
import { decodeJournalOutcome } from './outcomes/journal-outcome-codec'
import { outcomeFormat1EntryParsers } from './outcomes/journal-outcome-format-1'
import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key'
import type { AgentJournalSubmission } from '../../../shared/agent-session/journal/agent-session-journal-types'
import {
@@ -59,7 +62,8 @@ export function restoredJournalDomainSubmission(
submission,
sequence: journalInputSourceSequence(source) ?? submission.submittedSequence ?? 0,
index: source?.kind === 'outcome-admission' ? (source.item.sequenceIndex ?? 0) : 0,
ordinal,
// A retired outcome may outlive the reconstructed admission's ordinal.
ordinal: Number.isFinite(ordinal) ? ordinal : (submission.submittedSequence ?? 0),
candidate: state.inputs.candidates.has(id),
admittedPersonal,
echoObserved: state.inputs.echoes.has(agentJournalSubmissionKey(id)),
@@ -145,7 +149,8 @@ export function insertJournalDomainImportedSubmission(
/** Restore current dispatch membership from each saved submission. */
export function restoredJournalDomainSubmissions(
state: JournalReducerState,
personal: (submission: AgentJournalSubmission) => boolean
personal: (submission: AgentJournalSubmission) => boolean,
retiredAccepted: ReadonlyMap<string, AgentJournalSubmission> = new Map()
): JournalDomainImportAttempt[] {
const order = journalInputOrder(state)
const current = [...memoryJournalMap(state.submissions).values()].map((submission) =>
@@ -156,20 +161,56 @@ export function restoredJournalDomainSubmissions(
personal(submission)
)
)
const retained = [...memoryJournalMap(state.inputs.sources)].flatMap(([id, source]) =>
state.submissions.has(id)
? []
: [
{
...restoredJournalDomainSubmission(
state,
source.kind === 'submission' ? journalSubmissionFromRow(source) : source.submission,
order.inputOrdinal(id),
false
),
current: false
}
]
)
const retained = [
...new Set([...memoryJournalMap(state.inputs.sources).keys(), ...retiredAccepted.keys()])
].flatMap((id) => {
if (state.submissions.has(id)) {
return []
}
const source = state.inputs.sources.get(id)
const submission =
retiredAccepted.get(id) ??
(source?.kind === 'submission' ? journalSubmissionFromRow(source) : source?.submission)
if (!submission) {
return []
}
return [
{
...restoredJournalDomainSubmission(
state,
submission,
order.inputOrdinal(id),
retiredAccepted.has(id) && personal(submission)
),
current: false
}
]
})
return [...current, ...retained]
}
/** Preserve historical acceptance without restoring a deleted current submission key. */
export function readImportedRetiredAcceptances(
db: Database.Database,
state: JournalReducerState
): Map<string, AgentJournalSubmission> {
const source = journalOutcomeSource(db, state.sessionId, state.epoch)
const accepted = new Map<string, AgentJournalSubmission>()
if (!source) {
return accepted
}
const rows = db.prepare(`SELECT seq,field,entry_key,format,ordinal,value_json,deleted
FROM journal_outcomes WHERE generation=? AND seq<=? AND field='submissions' AND deleted=0 ORDER BY seq DESC`)
for (const row of rows.iterate(source.generation, source.through)) {
const value = decodeJournalOutcome(row)
const [id, submission] = outcomeFormat1EntryParsers.submissions.parse(JSON.parse(value.json))
if (
submission.dispatchState === 'accepted' &&
!state.submissions.has(id) &&
!accepted.has(id)
) {
accepted.set(id, submission)
}
}
return accepted
}
@@ -1,6 +1,7 @@
import type Database from '../../sqlite/sync-database'
import {
restoredJournalDomainSubmissions,
readImportedRetiredAcceptances,
type JournalDomainImportAttempt
} from './journal-domain-import-submissions'
export type { JournalDomainImportAttempt } from './journal-domain-import-submissions'
@@ -29,6 +30,7 @@ import { journalOutcomeSource } from './outcomes/journal-outcome-format'
import { memoryJournalMap, memoryJournalSet } from './journal-memory-collections'
import type { JournalReducerState } from './projection/journal-reducer'
import { journalDomainStopNote } from './journal-domain-stop-notes'
import { placeImportedLegacyQueue } from './journal-domain-import-queue'
import type { JournalQueuePauseMarks } from './queued-messages/queued-message-pause-marks'
import type { JournalHistoryOrigin } from './context-epochs/journal-history-origin'
import {
@@ -129,6 +131,7 @@ export function prepareJournalDomainImport(
journalOutcomeSource(db, sessionId, state.epoch) ?? undefined,
cards
)
placeImportedLegacyQueue(state, cards, personal)
return {
sessionId,
workspaceId: String(
@@ -158,7 +161,7 @@ export function prepareJournalDomainImport(
},
queueFailureBoundary: journalInputOrder(state).newestTurn.importBoundary()
},
pages: importPages(state, cards, personal)
pages: importPages(state, cards, personal, readImportedRetiredAcceptances(db, state))
}
}
@@ -190,7 +193,8 @@ function currentCards(
function* importPages(
state: JournalReducerState,
cards: ReadonlyMap<string, QueuedMessageRow>,
personal: (submission: AgentJournalSubmission) => boolean
personal: (submission: AgentJournalSubmission) => boolean,
retiredAccepted: ReadonlyMap<string, AgentJournalSubmission>
): Generator<JournalDomainImportPage> {
// Ties keep the restored map order, exactly as the restored timeline sorts them.
const items = [...memoryJournalMap(state.items)].toSorted(([, a], [, b]) =>
@@ -234,7 +238,7 @@ function* importPages(
...(echoes.has(providerItemId) ? { echo: { value: echoes.get(providerItemId) } } : {})
}))
)
yield* paged('attempts', restoredJournalDomainSubmissions(state, personal))
yield* paged('attempts', restoredJournalDomainSubmissions(state, personal, retiredAccepted))
yield* paged(
'removed-turns',
[...memoryJournalSet(state.removedTurnIds)].map((turnId) => ({ turnId }))
@@ -239,6 +239,28 @@ it('requeues a proven-unrun linked attempt but never requeues an uncertain hando
})
})
it('fronts current unlinked sends while preserving an explicit linked card and remains stable on reopen', () => {
const f = fixture()
f.seed('H')
f.seed('C')
f.db.prepare("UPDATE journal_queue_cards SET position=4 WHERE message_id='C'").run()
f.admit('A')
f.admit('H-handoff', { body: message('H') }, { messageId: 'H', expect: 'waiting' })
f.admit('B')
const order = () =>
f.db
.prepare(
"SELECT message_id FROM journal_queue_cards WHERE state='waiting' ORDER BY position,message_id"
)
.all()
.map((row) => row.message_id)
f.recover({ kind: 'hold-unsent', fence: 1, hold: { cause: 'hostRestarted' } }, f.head())
expect(order()).toEqual(['A', 'B', 'H', 'C'])
f.restart()
f.recover({ kind: 'hold-unsent', fence: 1, hold: { cause: 'hostRestarted' } }, f.head())
expect(order()).toEqual(['A', 'B', 'H', 'C'])
})
it('keeps only admitted personal input or commands and never defaults an unknown source to personal', () => {
const f = fixture()
f.admit('personal', { source: undefined })
@@ -0,0 +1,56 @@
import type Database from '../../sqlite/sync-database'
import type { AgentJournalCursor } from '../../../shared/agent-session/journal/agent-session-journal-identity'
import { readJournalDomainQueueRow } from './journal-domain-queue-rows'
import { readCommandReceipt } from './writing/command-receipt-table'
import { applyJournalDomainQueueCommand } from './journal-domain-queue-command'
/** Derived restart work is replaced by the new owner, while ordinary queued work retains custody. */
export function withdrawOpeningDerivedQueue(
db: Database.Database,
stateDirectory: string,
target: { sessionId: string; epoch: string },
openedThrough: AgentJournalCursor,
savedAt: number
): void {
let after: [number, string] | undefined
for (;;) {
const rows = db
.prepare(`SELECT position,message_id,state,revision,payload_json FROM journal_queue_cards
INDEXED BY journal_queue_order WHERE session_id=? AND epoch=? AND state IN ('waiting','returned')
${after ? 'AND (position,message_id)>(?,?)' : ''}
ORDER BY position,message_id LIMIT 64`)
.all(target.sessionId, target.epoch, ...(after ?? []))
const ids: string[] = []
for (const row of rows) {
const { card } = readJournalDomainQueueRow(target.sessionId, row)
if (
card.queuedAt?.epoch === openedThrough.epoch &&
card.queuedAt.sequence > openedThrough.sequence
) {
continue
}
const receipt = readCommandReceipt(db, { kind: 'global' }, card.messageId)
if (
(card.body.from?.orchestration?.message === 'task' && card.deliveryMode === 'steer') ||
(receipt.verdict === 'readable' &&
receipt.receipt.callerKey === 'trusted-local:restart-continuation')
) {
ids.push(card.messageId)
}
}
if (ids.length > 0) {
applyJournalDomainQueueCommand(
db,
stateDirectory,
target,
{ kind: 'queue-withdraw', messageIds: ids },
savedAt
)
}
if (rows.length < 64) {
return
}
const last = rows.at(-1)!
after = [Number(last.position), String(last.message_id)]
}
}
@@ -5,12 +5,21 @@ import { journalDomainSubmission } from './journal-domain-status-inputs'
import { journalDomainQueueRow } from './journal-domain-queue-rows'
import { JournalSqlScope } from './journal-sql-scope'
import { nextJournalDomainQueueCard } from './journal-domain-submission-consume'
import { agentSessionRecordFixture } from '../agent-session-record-test-fixture'
import { encodeAgentSessionRecord } from '../../../shared/agent-session/lifecycle/agent-session-record-stored-form'
import { readJournalDomainQueuePauses } from './journal-domain-queue-reads'
const cleanups: (() => void)[] = []
afterEach(() => cleanups.splice(0).forEach((cleanup) => cleanup()))
async function fixture() {
const f = domainLifecycleFixture(cleanups)
f.db
.prepare('INSERT INTO agent_session_records(session_id,record_json) VALUES(?,?)')
.run(
'chat',
JSON.stringify(encodeAgentSessionRecord(agentSessionRecordFixture({ sessionId: 'chat' })))
)
await f.run({
kind: 'submission',
input: {
@@ -42,6 +51,9 @@ it('opens at its original boundary, recovers sends and holds the recovered perso
const opened = await f.run({ kind: 'open', recover: { fence: 1 } })
expect(opened.value).toEqual({ epoch: 'epoch', sequence: 2 })
expect(opened.openRecovery).toEqual({ failures: [], reopenFloor: null })
expect(readJournalDomainQueuePauses(f.db, f.scope, null).map((pause) => pause.reason)).toEqual([
'restarted'
])
expect(journalDomainSubmission(f.scope, 'handed')).toMatchObject({
dispatchState: 'unknown',
recovered: true
@@ -88,6 +100,91 @@ it('retains earlier recovery and a conservative reopen floor when the final mark
).toEqual({ kind: 'selected', card: null })
})
it('recovers ordinary custody and withdraws only derived tasks or the selected readable continuation receipt', async () => {
const f = await fixture()
for (const [id, deliveryMode] of [
['derived', 'steer'],
['queued', 'queue'],
['legacy', undefined]
] as const) {
await f.run({
kind: 'queue-insert',
input: {
messageId: id,
body: {
...automaticBody(id),
from: {
kind: 'agent',
senders: [],
orchestration: { message: 'task', runId: 'run', taskId: 'task', dispatchId: 'dispatch' }
}
},
...(deliveryMode ? { deliveryMode } : {})
}
})
}
for (const messageId of ['continuation', 'unreadable', 'other-first']) {
await f.run({ kind: 'queue-insert', input: { messageId, body: automaticBody(messageId) } })
}
const receipt = f.db.prepare(`INSERT INTO agent_session_command_receipts
(operation_id,session_id,caller_key,method,fingerprint,status,result_json,rejection_json,accepted_at)
VALUES (?,'chat',?,'agentSession.send','fp','accepted',?,NULL,100)`)
receipt.run(
'continuation',
'trusted-local:restart-continuation',
JSON.stringify({ kind: 'journal-row', epoch: 'epoch', sequence: 1 })
)
receipt.run('unreadable', 'trusted-local:restart-continuation', 'broken-json')
receipt.run(
'other-first',
'other',
JSON.stringify({ kind: 'journal-row', epoch: 'epoch', sequence: 1 })
)
receipt.run(
'other-first',
'trusted-local:restart-continuation',
JSON.stringify({ kind: 'journal-row', epoch: 'epoch', sequence: 1 })
)
const opened = await f.run({ kind: 'open' })
expect(opened.openRecovery?.failures).toEqual([])
expect(readJournalDomainQueuePauses(f.db, f.scope, null)).toEqual([])
expect(journalDomainSubmission(f.scope, 'handed')?.dispatchState).toBe('pending')
expect(journalDomainSubmission(f.scope, 'unsent')?.dispatchState).toBe('rejected')
for (const id of ['derived', 'continuation']) {
expect(journalDomainQueueRow(f.db, f.scope, id)?.card.state).toBe('withdrawn')
}
for (const id of ['unsent', 'queued', 'legacy', 'unreadable', 'other-first']) {
expect(journalDomainQueueRow(f.db, f.scope, id)?.card.state).toBe('waiting')
}
})
it('keeps completed holds when derived withdrawal fails and retries withdrawal on the next open', async () => {
const f = await fixture()
await f.run({
kind: 'queue-insert',
input: { messageId: 'continuation', body: automaticBody('continuation') }
})
f.db
.prepare(`INSERT INTO agent_session_command_receipts
(operation_id,session_id,caller_key,method,fingerprint,status,result_json,rejection_json,accepted_at)
VALUES ('continuation','chat','trusted-local:restart-continuation','agentSession.send','fp','accepted',?,NULL,100)`)
.run(JSON.stringify({ kind: 'journal-row', epoch: 'epoch', sequence: 1 }))
f.db.exec(`CREATE TRIGGER reject_withdraw BEFORE UPDATE OF state ON journal_queue_cards
WHEN NEW.message_id='continuation' AND NEW.state='withdrawn'
BEGIN SELECT RAISE(ABORT,'withdraw unavailable'); END`)
const opened = await f.run({ kind: 'open' })
expect(opened.openRecovery?.failures).toMatchObject([
{ step: 'withdraw-derived', failure: { message: 'withdraw unavailable' } }
])
expect(journalDomainSubmission(f.scope, 'unsent')?.dispatchState).toBe('rejected')
expect(journalDomainQueueRow(f.db, f.scope, 'unsent')?.card.state).toBe('waiting')
expect(journalDomainQueueRow(f.db, f.scope, 'continuation')?.card.state).toBe('waiting')
f.db.exec('DROP TRIGGER reject_withdraw')
expect((await f.run({ kind: 'open' })).openRecovery?.failures).toEqual([])
expect(journalDomainQueueRow(f.db, f.scope, 'unsent')?.card.state).toBe('waiting')
expect(journalDomainQueueRow(f.db, f.scope, 'continuation')?.card.state).toBe('withdrawn')
})
it('marks a working subagent as unverifiable when another handle opens its saved journal', async () => {
const f = domainLifecycleFixture(cleanups)
await f.run({
@@ -14,10 +14,11 @@ import {
} from '../../../shared/agent-session/journal/agent-session-journal-types'
import { staleSubagentRosterRevisions } from './item-lifecycle/journal-subagent-liveness'
import { applyJournalDomainLifecycle } from './journal-domain-lifecycle'
import { withdrawOpeningDerivedQueue } from './journal-open-queue-withdrawal'
export type JournalOpenRecoveryResult = {
failures: {
step: 'pending-unknown' | 'hold-unsent' | 'queue-reopen'
step: 'pending-unknown' | 'hold-unsent' | 'withdraw-derived' | 'queue-reopen'
failure: JournalSqlFailure
}[]
reopenFloor: AgentJournalCursor | null
@@ -37,6 +38,13 @@ export function applyJournalSqlOpenCommand(
const { target } = command
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: open returns its physical pre-recovery cursor.
const openedThrough = opened.value as AgentJournalCursor
const fence =
recover?.fence ??
Number(
db
.prepare('SELECT highest_fence FROM journal_chat_state WHERE session_id=? AND epoch=?')
.get(target.sessionId, target.epoch)!.highest_fence
)
settleOpeningLiveWork(db, target, savedAt)
const recovery: JournalOpenRecoveryResult = { failures: [], reopenFloor: null }
const attempt = <Value>(
@@ -56,23 +64,28 @@ export function applyJournalSqlOpenCommand(
return undefined
}
}
if (recover) {
for (const kind of ['pending-unknown', 'hold-unsent'] as const) {
const result = attempt(kind, () =>
applyJournalDomainInputRecovery(
db,
target,
kind === 'pending-unknown'
? { kind, fence: recover.fence }
: { kind, fence: recover.fence, hold: { cause: 'hostRestarted' } },
{ openedThrough },
savedAt
)
for (const kind of recover
? (['pending-unknown', 'hold-unsent'] as const)
: (['hold-unsent'] as const)) {
const result = attempt(kind, () =>
applyJournalDomainInputRecovery(
db,
target,
kind === 'pending-unknown'
? { kind, fence }
: { kind, fence, hold: { cause: 'hostRestarted' } },
{ openedThrough },
savedAt
)
if (result && 'failure' in result.value) {
recovery.failures.push({ step: kind, failure: result.value.failure })
}
)
if (result && 'failure' in result.value) {
recovery.failures.push({ step: kind, failure: result.value.failure })
}
}
attempt('withdraw-derived', () =>
withdrawOpeningDerivedQueue(db, stateDirectory, target, openedThrough, savedAt)
)
if (recover) {
attempt('queue-reopen', () => {
if (journalDomainQueueAwaitsReopen(db, target)) {
const head = db
@@ -82,7 +95,7 @@ export function applyJournalSqlOpenCommand(
applyJournalDomainControlCommand(
db,
target,
{ kind: 'queue-reopen-mark', fence: recover.fence },
{ kind: 'queue-reopen-mark', fence },
{},
savedAt
)
@@ -15,9 +15,13 @@ import { structuredAgentSessionPayloadFingerprint } from '../../../../shared/age
import type { JournalWriterOperation } from '../journal-writer-command'
type UnsentSendHold = Extract<JournalWriterOperation, { kind: 'hold-unsent' }>['hold']
import type { AgentSessionJournal } from '../journal-store'
import type { JournalRow, JournalSubmissionRow } from '../durable-rows/journal-row-schema'
import type { JournalQueueRow } from './journal-queue-events'
import { seedTestJournalHistoricalRows } from '../replay/journal-historical-inputs.test-support'
import {
closeTestJournalHostDatabases,
createTrackedJournalOpener
createTrackedJournalOpener,
openTestJournalHostDatabase
} from '../store-lifecycle/journal-host-database-test-support'
export function createFixture() {
@@ -116,6 +120,58 @@ export function createFixture() {
return open()
}
const historicalBase = (seq: number) => ({
v: 4,
epoch: 'old-queue',
seq,
fence: 0,
ts: 1_000 + seq
})
function historicalQueue(
seq: number,
messageId: string,
body = message(messageId),
position = seq
): JournalQueueRow {
return {
...historicalBase(seq),
kind: 'queue',
messageId,
change: { op: 'queued', body, position, queuedAt: { epoch: 'old-queue', sequence: seq } }
}
}
function historicalSend(
seq: number,
clientMessageId: string,
fields: Partial<JournalSubmissionRow> = {}
): JournalSubmissionRow {
const body = fields.body ?? message(clientMessageId)
return {
...historicalBase(seq),
kind: 'submission',
clientMessageId,
body,
payloadFingerprint: fingerprint(body),
providerHandle: claudeProviderHandle('native-1', null),
origin: 'client',
source: { kind: 'user' },
handoverRecorded: true,
...fields
}
}
async function openHistorical(rows: JournalRow[]) {
seedTestJournalHistoricalRows(openTestJournalHostDatabase(root).db, IDENTITY, [
{
...historicalBase(1),
kind: 'epoch',
reason: 'session_created',
providerHandle: claudeProviderHandle('native-1', null)
},
...rows
])
return open()
}
async function cardOrder(journal: AgentSessionJournal): Promise<string[]> {
return (await journal.queuedMessages.list())
.filter((card) => card.state === 'waiting')
@@ -155,6 +211,10 @@ export function createFixture() {
},
fingerprint,
hold,
historicalBase,
historicalQueue,
historicalSend,
openHistorical,
journals,
message,
open,
@@ -16,32 +16,22 @@ export function registerJournalUnsentSendHoldQueueTests(
])(
'in acceptance order, ahead of the cards already waiting, a hand-off by $by among them',
async ({ origin, order }) => {
const journal = await fixture.afterRestart(async (earlier) => {
await earlier.queuedMessages.insert({
messageId: 'H',
body: fixture.message('H')
})
await earlier.queuedMessages.insert({
messageId: 'C',
body: fixture.message('C')
})
await fixture.accept(earlier, 'A', { origin: 'client', source: USER_MESSAGE_SOURCE })
await earlier.appendSubmission(
{
clientMessageId: 'H-handoff',
payloadFingerprint: fixture.fingerprint(fixture.message('H')),
body: fixture.message('H'),
fence: 0,
handoverRecorded: true,
origin
},
{ messageId: 'H', expect: 'waiting' }
)
await fixture.accept(earlier, 'B', { origin: 'client', source: USER_MESSAGE_SOURCE })
})
const journal = await fixture.openHistorical([
fixture.historicalQueue(2, 'H'),
fixture.historicalQueue(3, 'C'),
fixture.historicalSend(4, 'A'),
fixture.historicalSend(5, 'H-handoff', {
body: fixture.message('H'),
queuedMessageId: 'H',
origin
}),
fixture.historicalSend(6, 'B')
])
await fixture.hold(journal)
expect(await fixture.cardOrder(journal)).toEqual(order)
await journal.close()
expect(await fixture.cardOrder(await fixture.open())).toEqual(order)
}
)
})
@@ -81,43 +71,49 @@ export function registerJournalUnsentSendHoldQueueTests(
it.each(['stopped', 'reserved-command', 'moved-command'] as const)(
'preserves the %s ordering boundary',
async (shape) => {
const journal = await fixture.afterRestart(async (earlier) => {
if (shape === 'stopped') {
await earlier.queuedMessages.insert({ messageId: 'W', body: fixture.message('W') })
await earlier.appendStopEvent({ reason: 'user-stop' }, 0)
await fixture.accept(earlier, 'S', { origin: 'client', source: USER_MESSAGE_SOURCE })
return
}
await earlier.queuedMessages.insert({
messageId: 'C1',
body: { ...fixture.message('/compact'), command: { name: 'compact' as const } }
})
await fixture.accept(earlier, 'S', { origin: 'client', source: USER_MESSAGE_SOURCE })
await earlier.queuedMessages.insert({ messageId: 'C2', body: fixture.message('C2') })
await earlier.queuedMessages.move({ messageId: 'C2', beforeMessageId: 'C1' })
if (shape === 'reserved-command') {
await earlier.appendSubmission(
{
clientMessageId: 'command-attempt',
origin: 'client',
body: { ...fixture.message('/compact'), command: { name: 'compact' as const } },
payloadFingerprint: fixture.fingerprint({
...fixture.message('/compact'),
command: { name: 'compact' as const }
}),
handoverRecorded: true,
fence: 0
},
{ messageId: 'C1', expect: 'waiting' }
)
await earlier.resolveDispatch({
clientMessageId: 'command-attempt',
state: 'pending',
turnScope: { kind: 'thread' },
fence: 0
})
}
})
const command = { ...fixture.message('/compact'), command: { name: 'compact' as const } }
const journal = await fixture.openHistorical(
shape === 'stopped'
? [
fixture.historicalQueue(2, 'W'),
{
...fixture.historicalBase(3),
kind: 'tombstone',
itemId: 'orca:stop',
revision: 1,
stopEvent: { reason: 'user-stop', at: 1_003 }
},
fixture.historicalSend(4, 'S')
]
: [
fixture.historicalQueue(2, 'C1', command),
fixture.historicalSend(3, 'S'),
fixture.historicalQueue(4, 'C2'),
{
...fixture.historicalBase(5),
kind: 'queue',
messageId: 'C2',
change: { op: 'moved', position: 1 }
},
...(shape === 'reserved-command'
? [
fixture.historicalSend(6, 'command-attempt', {
body: command,
queuedMessageId: 'C1'
}),
{
...fixture.historicalBase(7),
kind: 'dispatch' as const,
clientMessageId: 'command-attempt',
state: 'pending' as const,
providerItemId: null,
reason: null,
turnScope: { kind: 'thread' as const }
}
]
: [])
]
)
expect(await fixture.cardOrder(journal)).toEqual(
shape === 'stopped'
? ['S', 'W']
@@ -2,7 +2,9 @@ import { seedTestJournalHistoricalRows } from '../replay/journal-historical-inpu
import { openTestJournalHostDatabase } from '../store-lifecycle/journal-host-database-test-support'
import type { createFixture } from './queued-message-pause-contract-test-fixture'
import { describe, expect, it } from 'vitest'
import { QueuedMessageNotConsumableError } from './queued-message-consume-error'
import { agentSessionRecordFixture } from '../../agent-session-record-test-fixture'
import { encodeAgentSessionRecord } from '../../../../shared/agent-session/lifecycle/agent-session-record-stored-form'
import type { JournalAutomaticSelection } from '../journal-automatic-submission'
export function registerQueuedMessageStopPauseTests(
fixture: ReturnType<typeof createFixture>
@@ -35,29 +37,58 @@ export function registerQueuedMessageStopPauseTests(
})
it("the queue's own consume is refused in its transaction while the pause holds the card; Send-now is not", async () => {
const record = agentSessionRecordFixture({ sessionId: fixture.IDENTITY.sessionId })
openTestJournalHostDatabase(fixture.root)
.db.prepare('INSERT INTO agent_session_records(session_id,record_json) VALUES(?,?)')
.run(record.sessionId, JSON.stringify(encodeAgentSessionRecord(record)))
const journal = await fixture.open()
await fixture.queueDraft(journal, 'draft-1')
await fixture.userStop(journal)
const consume = (id: string, automatic: boolean) =>
journal.appendSubmission(
{
clientMessageId: id,
payloadFingerprint: fixture.fingerprintOf('draft-1'),
body: fixture.message('draft-1'),
fence: 0,
handoverRecorded: true
const selection: JournalAutomaticSelection = {
authority: {
handle: journal.observationHandle(),
record: {
currentFence: 1,
ownerObservation: { claimStatus: 'live', unreconciled: false, handoffStage: null }
},
{
messageId: 'draft-1',
expect: 'waiting',
...(automatic ? { yieldsToPause: true as const } : {})
}
child: null,
handingOver: [],
continuingUnadmitted: [],
finalizing: [],
openedThrough: journal.openingCursor(),
operationalRevision: 0
},
child: null,
steering: { kind: 'codex-unavailable' }
}
const automatic = (clientMessageId: string) =>
journal.claimAutomaticSubmission(
{ clientMessageId, fence: 1 },
async () => selection,
() => 'owed'
)
await expect(consume('drain-1', true)).rejects.toBeInstanceOf(QueuedMessageNotConsumableError)
expect(await automatic('drain-1')).toEqual({ kind: 'waiting' })
expect(await journal.reads.submission('drain-1')).toBeUndefined()
expect((await journal.queuedMessages.get('draft-1'))?.state).toBe('waiting')
await consume('send-now-1', false)
expect((await journal.queuedMessages.get('draft-1'))?.state).toBe('dispatched')
await journal.appendQueueResume(1)
expect(await automatic('drain-after-resume')).toEqual({
kind: 'claimed',
messageId: 'draft-1',
clientMessageId: 'drain-after-resume'
})
await fixture.queueDraft(journal, 'draft-2')
await journal.appendStopEvent({ reason: 'user-stop' }, 1)
await journal.appendSubmission(
{
clientMessageId: 'send-now-1',
payloadFingerprint: fixture.fingerprintOf('draft-2'),
body: fixture.message('draft-2'),
fence: 1,
handoverRecorded: true
},
{ messageId: 'draft-2', expect: 'waiting' }
)
expect((await journal.queuedMessages.get('draft-2'))?.state).toBe('dispatched')
})
it("a rewind's epoch replacement restates a Stop still pausing, and not one a turn ended", async () => {
@@ -194,17 +225,16 @@ export function registerQueuedMessageStopPauseTests(
state: 'accepted',
providerItemId: null,
reason: null
},
{
...row(6),
kind: 'tombstone',
itemId: 'orca:retired-person',
revision: 1,
retireSubmission: true
}
],
true
)
const db = openTestJournalHostDatabase(fixture.root).db
const source = db.prepare('SELECT generation,through_seq FROM journal_outcome_epochs').get()!
db.prepare(
"INSERT INTO journal_outcomes (generation,seq,field,entry_key,format,ordinal,value_json,deleted) VALUES (?,?,'submissions',?,2,'s0000000000000001:0000000000000001','null',1)"
).run(source.generation, Number(source.through_seq) + 1, JSON.stringify('retired-person'))
db.prepare('UPDATE journal_outcome_epochs SET through_seq=through_seq+1').run()
const restored = await fixture.open()
expect(await restored.reads.submission('retired-person')).toBeUndefined()
expect((await restored.queuedMessages.nextInputCard())?.messageId).toBe('W')
@@ -105,6 +105,7 @@ export async function openStructuredAgentSessionConversationJournal(
const messages = {
'pending-unknown': 'marking pending sends unknown on open failed',
'hold-unsent': 'settling sends an earlier process left queued failed on open',
'withdraw-derived': 'withdrawing replaced restart work failed on open',
'queue-reopen': 'marking a reopened queue failed'
}
deps.logger.warn(messages[step], {