mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 11:24:37 +00:00
Preserve saved queue order and recover restart custody on SQL open
This commit is contained in:
@@ -0,0 +1,156 @@
|
||||
import type {
|
||||
AgentJournalMessageItem,
|
||||
AgentJournalSubmission
|
||||
} from '../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key'
|
||||
import { agentSessionSendBodyFingerprint } from '../../../shared/agent-session/requests/structured-agent-session-send-mutation'
|
||||
import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session/requests/agent-session-queued-submission'
|
||||
import type { JournalReducerState } from './projection/journal-reducer'
|
||||
import { memoryJournalMap } from './journal-memory-collections'
|
||||
import { journalInputOrder } from './input-order/journal-input-order'
|
||||
import { compareQueuedRows } from './input-order/journal-queued-row-index'
|
||||
import { unsentSendKeptAsCard } from './queued-messages/journal-unsent-send-hold'
|
||||
import type { QueuedMessageRow } from './queued-messages/queued-message-table'
|
||||
|
||||
/** Materialize old places once while unready; ready SQL rows never own a legacy decoder. */
|
||||
export function placeImportedLegacyQueue(
|
||||
state: JournalReducerState,
|
||||
cards: Map<string, QueuedMessageRow>,
|
||||
personal: (submission: AgentJournalSubmission) => boolean
|
||||
): void {
|
||||
const owner = journalInputOrder(state)
|
||||
const legacy = owner.legacy
|
||||
if (!legacy) {
|
||||
return
|
||||
}
|
||||
const batch = [...memoryJournalMap(state.submissions).values()]
|
||||
.filter(isQueuedAgentJournalSubmission)
|
||||
.sort((a, b) => (a.acceptedSequence ?? 0) - (b.acceptedSequence ?? 0))
|
||||
.map((submission) => ({
|
||||
submission,
|
||||
body: unsentSendKeptAsCard(
|
||||
submission,
|
||||
state.items.get(agentJournalSubmissionKey(submission.clientMessageId))?.body ?? null
|
||||
)
|
||||
}))
|
||||
if (batch.length === 0) {
|
||||
return
|
||||
}
|
||||
const placed = legacyQueuePositions(state, cards, batch)
|
||||
for (const [id, position] of placed) {
|
||||
const card = cards.get(id)
|
||||
if (card) {
|
||||
cards.set(id, { ...card, position })
|
||||
}
|
||||
}
|
||||
for (const { submission, body } of batch) {
|
||||
if (!body || cards.has(submission.clientMessageId)) {
|
||||
continue
|
||||
}
|
||||
cards.set(submission.clientMessageId, {
|
||||
messageId: submission.clientMessageId,
|
||||
body,
|
||||
position: placed.get(submission.clientMessageId)!,
|
||||
fingerprint: agentSessionSendBodyFingerprint(state.sessionId, body),
|
||||
state: 'waiting',
|
||||
holdReason: null,
|
||||
returnedReason: null,
|
||||
returnedRejection: null,
|
||||
consumedAs: null,
|
||||
queuedAt: { epoch: state.epoch, sequence: submission.submittedSequence ?? 0 },
|
||||
deliveryMode: 'queue',
|
||||
unattended: !personal(submission),
|
||||
...(submission.options ? { options: submission.options } : {})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function legacyQueuePositions(
|
||||
state: JournalReducerState,
|
||||
allCards: ReadonlyMap<string, QueuedMessageRow>,
|
||||
batch: readonly { submission: AgentJournalSubmission; body: AgentJournalMessageItem | null }[]
|
||||
): Map<string, number> {
|
||||
const owner = journalInputOrder(state)
|
||||
const legacy = owner.legacy!
|
||||
const cards = [...allCards.values()]
|
||||
.filter((card) => card.state !== 'withdrawn')
|
||||
.sort(compareQueuedRows)
|
||||
const forcedBatch = batch.some(({ submission }) => legacy.forced(submission.clientMessageId))
|
||||
const explicitHeads = new Map(
|
||||
batch.flatMap(({ submission }) => {
|
||||
const id = submission.origin === 'client' ? submission.queuedMessageId : undefined
|
||||
return id && allCards.get(id)?.consumedAs === submission.clientMessageId
|
||||
? [[submission.clientMessageId, id] as const]
|
||||
: []
|
||||
})
|
||||
)
|
||||
const explicit = new Set(explicitHeads.values())
|
||||
const earlier: string[] = []
|
||||
const rest: QueuedMessageRow[] = []
|
||||
for (const card of cards) {
|
||||
if (explicit.has(card.messageId)) {
|
||||
continue
|
||||
}
|
||||
if (
|
||||
card.state === 'waiting' &&
|
||||
!forcedBatch &&
|
||||
card.queuedAt &&
|
||||
card.queuedAt.epoch !== state.epoch
|
||||
) {
|
||||
earlier.push(card.messageId)
|
||||
} else {
|
||||
rest.push(card)
|
||||
}
|
||||
}
|
||||
const at = new Map<number, string[]>()
|
||||
const head: string[] = []
|
||||
const firstCut = batch
|
||||
.flatMap(({ submission }) => legacy.place(submission.clientMessageId)?.context.cut ?? [])
|
||||
.reduce((a, b) => Math.min(a, b), Infinity)
|
||||
let previousBoundary = 0
|
||||
for (let i = 0; i < rest.length; i++) {
|
||||
if (
|
||||
rest[i].state === 'waiting' &&
|
||||
owner.keptCard(rest[i].messageId) &&
|
||||
(legacy.lineage.cards.get(rest[i].messageId)?.introduced ?? -Infinity) < firstCut
|
||||
) {
|
||||
previousBoundary = i + 1
|
||||
}
|
||||
}
|
||||
for (const { submission, body } of batch) {
|
||||
const id = body ? submission.clientMessageId : explicitHeads.get(submission.clientMessageId)
|
||||
if (!id || (body && owner.keptCard(id))) {
|
||||
continue
|
||||
}
|
||||
if (!body && previousBoundary > 0) {
|
||||
head.push(id)
|
||||
continue
|
||||
}
|
||||
const after = legacy.forced(submission.clientMessageId)
|
||||
? legacy.predecessors(state, submission.clientMessageId).position
|
||||
: -Infinity
|
||||
let low = 0,
|
||||
high = rest.length
|
||||
while (low < high) {
|
||||
const mid = (low + high) >>> 1
|
||||
if (rest[mid].position <= after) {
|
||||
low = mid + 1
|
||||
} else {
|
||||
high = mid
|
||||
}
|
||||
}
|
||||
const boundary = Math.max(previousBoundary, low)
|
||||
previousBoundary = boundary
|
||||
const bucket = at.get(boundary) ?? []
|
||||
bucket.push(id)
|
||||
at.set(boundary, bucket)
|
||||
}
|
||||
const order = [...earlier, ...head]
|
||||
for (let i = 0; i <= rest.length; i++) {
|
||||
order.push(...(at.get(i) ?? []))
|
||||
if (i < rest.length) {
|
||||
order.push(rest[i].messageId)
|
||||
}
|
||||
}
|
||||
return new Map(order.map((id, index) => [id, index]))
|
||||
}
|
||||
@@ -2,6 +2,9 @@ import { journalInputOrder } from './input-order/journal-input-order'
|
||||
import { memoryJournalMap } from './journal-memory-collections'
|
||||
import { readAnsweredTurn } from './projection/journal-dispatch-reducer'
|
||||
import type Database from '../../sqlite/sync-database'
|
||||
import { journalOutcomeSource } from './outcomes/journal-outcome-format'
|
||||
import { decodeJournalOutcome } from './outcomes/journal-outcome-codec'
|
||||
import { outcomeFormat1EntryParsers } from './outcomes/journal-outcome-format-1'
|
||||
import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key'
|
||||
import type { AgentJournalSubmission } from '../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
import {
|
||||
@@ -59,7 +62,8 @@ export function restoredJournalDomainSubmission(
|
||||
submission,
|
||||
sequence: journalInputSourceSequence(source) ?? submission.submittedSequence ?? 0,
|
||||
index: source?.kind === 'outcome-admission' ? (source.item.sequenceIndex ?? 0) : 0,
|
||||
ordinal,
|
||||
// A retired outcome may outlive the reconstructed admission's ordinal.
|
||||
ordinal: Number.isFinite(ordinal) ? ordinal : (submission.submittedSequence ?? 0),
|
||||
candidate: state.inputs.candidates.has(id),
|
||||
admittedPersonal,
|
||||
echoObserved: state.inputs.echoes.has(agentJournalSubmissionKey(id)),
|
||||
@@ -145,7 +149,8 @@ export function insertJournalDomainImportedSubmission(
|
||||
/** Restore current dispatch membership from each saved submission. */
|
||||
export function restoredJournalDomainSubmissions(
|
||||
state: JournalReducerState,
|
||||
personal: (submission: AgentJournalSubmission) => boolean
|
||||
personal: (submission: AgentJournalSubmission) => boolean,
|
||||
retiredAccepted: ReadonlyMap<string, AgentJournalSubmission> = new Map()
|
||||
): JournalDomainImportAttempt[] {
|
||||
const order = journalInputOrder(state)
|
||||
const current = [...memoryJournalMap(state.submissions).values()].map((submission) =>
|
||||
@@ -156,20 +161,56 @@ export function restoredJournalDomainSubmissions(
|
||||
personal(submission)
|
||||
)
|
||||
)
|
||||
const retained = [...memoryJournalMap(state.inputs.sources)].flatMap(([id, source]) =>
|
||||
state.submissions.has(id)
|
||||
? []
|
||||
: [
|
||||
{
|
||||
...restoredJournalDomainSubmission(
|
||||
state,
|
||||
source.kind === 'submission' ? journalSubmissionFromRow(source) : source.submission,
|
||||
order.inputOrdinal(id),
|
||||
false
|
||||
),
|
||||
current: false
|
||||
}
|
||||
]
|
||||
)
|
||||
const retained = [
|
||||
...new Set([...memoryJournalMap(state.inputs.sources).keys(), ...retiredAccepted.keys()])
|
||||
].flatMap((id) => {
|
||||
if (state.submissions.has(id)) {
|
||||
return []
|
||||
}
|
||||
const source = state.inputs.sources.get(id)
|
||||
const submission =
|
||||
retiredAccepted.get(id) ??
|
||||
(source?.kind === 'submission' ? journalSubmissionFromRow(source) : source?.submission)
|
||||
if (!submission) {
|
||||
return []
|
||||
}
|
||||
return [
|
||||
{
|
||||
...restoredJournalDomainSubmission(
|
||||
state,
|
||||
submission,
|
||||
order.inputOrdinal(id),
|
||||
retiredAccepted.has(id) && personal(submission)
|
||||
),
|
||||
current: false
|
||||
}
|
||||
]
|
||||
})
|
||||
return [...current, ...retained]
|
||||
}
|
||||
|
||||
/** Preserve historical acceptance without restoring a deleted current submission key. */
|
||||
export function readImportedRetiredAcceptances(
|
||||
db: Database.Database,
|
||||
state: JournalReducerState
|
||||
): Map<string, AgentJournalSubmission> {
|
||||
const source = journalOutcomeSource(db, state.sessionId, state.epoch)
|
||||
const accepted = new Map<string, AgentJournalSubmission>()
|
||||
if (!source) {
|
||||
return accepted
|
||||
}
|
||||
const rows = db.prepare(`SELECT seq,field,entry_key,format,ordinal,value_json,deleted
|
||||
FROM journal_outcomes WHERE generation=? AND seq<=? AND field='submissions' AND deleted=0 ORDER BY seq DESC`)
|
||||
for (const row of rows.iterate(source.generation, source.through)) {
|
||||
const value = decodeJournalOutcome(row)
|
||||
const [id, submission] = outcomeFormat1EntryParsers.submissions.parse(JSON.parse(value.json))
|
||||
if (
|
||||
submission.dispatchState === 'accepted' &&
|
||||
!state.submissions.has(id) &&
|
||||
!accepted.has(id)
|
||||
) {
|
||||
accepted.set(id, submission)
|
||||
}
|
||||
}
|
||||
return accepted
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type Database from '../../sqlite/sync-database'
|
||||
import {
|
||||
restoredJournalDomainSubmissions,
|
||||
readImportedRetiredAcceptances,
|
||||
type JournalDomainImportAttempt
|
||||
} from './journal-domain-import-submissions'
|
||||
export type { JournalDomainImportAttempt } from './journal-domain-import-submissions'
|
||||
@@ -29,6 +30,7 @@ import { journalOutcomeSource } from './outcomes/journal-outcome-format'
|
||||
import { memoryJournalMap, memoryJournalSet } from './journal-memory-collections'
|
||||
import type { JournalReducerState } from './projection/journal-reducer'
|
||||
import { journalDomainStopNote } from './journal-domain-stop-notes'
|
||||
import { placeImportedLegacyQueue } from './journal-domain-import-queue'
|
||||
import type { JournalQueuePauseMarks } from './queued-messages/queued-message-pause-marks'
|
||||
import type { JournalHistoryOrigin } from './context-epochs/journal-history-origin'
|
||||
import {
|
||||
@@ -129,6 +131,7 @@ export function prepareJournalDomainImport(
|
||||
journalOutcomeSource(db, sessionId, state.epoch) ?? undefined,
|
||||
cards
|
||||
)
|
||||
placeImportedLegacyQueue(state, cards, personal)
|
||||
return {
|
||||
sessionId,
|
||||
workspaceId: String(
|
||||
@@ -158,7 +161,7 @@ export function prepareJournalDomainImport(
|
||||
},
|
||||
queueFailureBoundary: journalInputOrder(state).newestTurn.importBoundary()
|
||||
},
|
||||
pages: importPages(state, cards, personal)
|
||||
pages: importPages(state, cards, personal, readImportedRetiredAcceptances(db, state))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -190,7 +193,8 @@ function currentCards(
|
||||
function* importPages(
|
||||
state: JournalReducerState,
|
||||
cards: ReadonlyMap<string, QueuedMessageRow>,
|
||||
personal: (submission: AgentJournalSubmission) => boolean
|
||||
personal: (submission: AgentJournalSubmission) => boolean,
|
||||
retiredAccepted: ReadonlyMap<string, AgentJournalSubmission>
|
||||
): Generator<JournalDomainImportPage> {
|
||||
// Ties keep the restored map order, exactly as the restored timeline sorts them.
|
||||
const items = [...memoryJournalMap(state.items)].toSorted(([, a], [, b]) =>
|
||||
@@ -234,7 +238,7 @@ function* importPages(
|
||||
...(echoes.has(providerItemId) ? { echo: { value: echoes.get(providerItemId) } } : {})
|
||||
}))
|
||||
)
|
||||
yield* paged('attempts', restoredJournalDomainSubmissions(state, personal))
|
||||
yield* paged('attempts', restoredJournalDomainSubmissions(state, personal, retiredAccepted))
|
||||
yield* paged(
|
||||
'removed-turns',
|
||||
[...memoryJournalSet(state.removedTurnIds)].map((turnId) => ({ turnId }))
|
||||
|
||||
@@ -239,6 +239,28 @@ it('requeues a proven-unrun linked attempt but never requeues an uncertain hando
|
||||
})
|
||||
})
|
||||
|
||||
it('fronts current unlinked sends while preserving an explicit linked card and remains stable on reopen', () => {
|
||||
const f = fixture()
|
||||
f.seed('H')
|
||||
f.seed('C')
|
||||
f.db.prepare("UPDATE journal_queue_cards SET position=4 WHERE message_id='C'").run()
|
||||
f.admit('A')
|
||||
f.admit('H-handoff', { body: message('H') }, { messageId: 'H', expect: 'waiting' })
|
||||
f.admit('B')
|
||||
const order = () =>
|
||||
f.db
|
||||
.prepare(
|
||||
"SELECT message_id FROM journal_queue_cards WHERE state='waiting' ORDER BY position,message_id"
|
||||
)
|
||||
.all()
|
||||
.map((row) => row.message_id)
|
||||
f.recover({ kind: 'hold-unsent', fence: 1, hold: { cause: 'hostRestarted' } }, f.head())
|
||||
expect(order()).toEqual(['A', 'B', 'H', 'C'])
|
||||
f.restart()
|
||||
f.recover({ kind: 'hold-unsent', fence: 1, hold: { cause: 'hostRestarted' } }, f.head())
|
||||
expect(order()).toEqual(['A', 'B', 'H', 'C'])
|
||||
})
|
||||
|
||||
it('keeps only admitted personal input or commands and never defaults an unknown source to personal', () => {
|
||||
const f = fixture()
|
||||
f.admit('personal', { source: undefined })
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import type Database from '../../sqlite/sync-database'
|
||||
import type { AgentJournalCursor } from '../../../shared/agent-session/journal/agent-session-journal-identity'
|
||||
import { readJournalDomainQueueRow } from './journal-domain-queue-rows'
|
||||
import { readCommandReceipt } from './writing/command-receipt-table'
|
||||
import { applyJournalDomainQueueCommand } from './journal-domain-queue-command'
|
||||
|
||||
/** Derived restart work is replaced by the new owner, while ordinary queued work retains custody. */
|
||||
export function withdrawOpeningDerivedQueue(
|
||||
db: Database.Database,
|
||||
stateDirectory: string,
|
||||
target: { sessionId: string; epoch: string },
|
||||
openedThrough: AgentJournalCursor,
|
||||
savedAt: number
|
||||
): void {
|
||||
let after: [number, string] | undefined
|
||||
for (;;) {
|
||||
const rows = db
|
||||
.prepare(`SELECT position,message_id,state,revision,payload_json FROM journal_queue_cards
|
||||
INDEXED BY journal_queue_order WHERE session_id=? AND epoch=? AND state IN ('waiting','returned')
|
||||
${after ? 'AND (position,message_id)>(?,?)' : ''}
|
||||
ORDER BY position,message_id LIMIT 64`)
|
||||
.all(target.sessionId, target.epoch, ...(after ?? []))
|
||||
const ids: string[] = []
|
||||
for (const row of rows) {
|
||||
const { card } = readJournalDomainQueueRow(target.sessionId, row)
|
||||
if (
|
||||
card.queuedAt?.epoch === openedThrough.epoch &&
|
||||
card.queuedAt.sequence > openedThrough.sequence
|
||||
) {
|
||||
continue
|
||||
}
|
||||
const receipt = readCommandReceipt(db, { kind: 'global' }, card.messageId)
|
||||
if (
|
||||
(card.body.from?.orchestration?.message === 'task' && card.deliveryMode === 'steer') ||
|
||||
(receipt.verdict === 'readable' &&
|
||||
receipt.receipt.callerKey === 'trusted-local:restart-continuation')
|
||||
) {
|
||||
ids.push(card.messageId)
|
||||
}
|
||||
}
|
||||
if (ids.length > 0) {
|
||||
applyJournalDomainQueueCommand(
|
||||
db,
|
||||
stateDirectory,
|
||||
target,
|
||||
{ kind: 'queue-withdraw', messageIds: ids },
|
||||
savedAt
|
||||
)
|
||||
}
|
||||
if (rows.length < 64) {
|
||||
return
|
||||
}
|
||||
const last = rows.at(-1)!
|
||||
after = [Number(last.position), String(last.message_id)]
|
||||
}
|
||||
}
|
||||
@@ -5,12 +5,21 @@ import { journalDomainSubmission } from './journal-domain-status-inputs'
|
||||
import { journalDomainQueueRow } from './journal-domain-queue-rows'
|
||||
import { JournalSqlScope } from './journal-sql-scope'
|
||||
import { nextJournalDomainQueueCard } from './journal-domain-submission-consume'
|
||||
import { agentSessionRecordFixture } from '../agent-session-record-test-fixture'
|
||||
import { encodeAgentSessionRecord } from '../../../shared/agent-session/lifecycle/agent-session-record-stored-form'
|
||||
import { readJournalDomainQueuePauses } from './journal-domain-queue-reads'
|
||||
|
||||
const cleanups: (() => void)[] = []
|
||||
afterEach(() => cleanups.splice(0).forEach((cleanup) => cleanup()))
|
||||
|
||||
async function fixture() {
|
||||
const f = domainLifecycleFixture(cleanups)
|
||||
f.db
|
||||
.prepare('INSERT INTO agent_session_records(session_id,record_json) VALUES(?,?)')
|
||||
.run(
|
||||
'chat',
|
||||
JSON.stringify(encodeAgentSessionRecord(agentSessionRecordFixture({ sessionId: 'chat' })))
|
||||
)
|
||||
await f.run({
|
||||
kind: 'submission',
|
||||
input: {
|
||||
@@ -42,6 +51,9 @@ it('opens at its original boundary, recovers sends and holds the recovered perso
|
||||
const opened = await f.run({ kind: 'open', recover: { fence: 1 } })
|
||||
expect(opened.value).toEqual({ epoch: 'epoch', sequence: 2 })
|
||||
expect(opened.openRecovery).toEqual({ failures: [], reopenFloor: null })
|
||||
expect(readJournalDomainQueuePauses(f.db, f.scope, null).map((pause) => pause.reason)).toEqual([
|
||||
'restarted'
|
||||
])
|
||||
expect(journalDomainSubmission(f.scope, 'handed')).toMatchObject({
|
||||
dispatchState: 'unknown',
|
||||
recovered: true
|
||||
@@ -88,6 +100,91 @@ it('retains earlier recovery and a conservative reopen floor when the final mark
|
||||
).toEqual({ kind: 'selected', card: null })
|
||||
})
|
||||
|
||||
it('recovers ordinary custody and withdraws only derived tasks or the selected readable continuation receipt', async () => {
|
||||
const f = await fixture()
|
||||
for (const [id, deliveryMode] of [
|
||||
['derived', 'steer'],
|
||||
['queued', 'queue'],
|
||||
['legacy', undefined]
|
||||
] as const) {
|
||||
await f.run({
|
||||
kind: 'queue-insert',
|
||||
input: {
|
||||
messageId: id,
|
||||
body: {
|
||||
...automaticBody(id),
|
||||
from: {
|
||||
kind: 'agent',
|
||||
senders: [],
|
||||
orchestration: { message: 'task', runId: 'run', taskId: 'task', dispatchId: 'dispatch' }
|
||||
}
|
||||
},
|
||||
...(deliveryMode ? { deliveryMode } : {})
|
||||
}
|
||||
})
|
||||
}
|
||||
for (const messageId of ['continuation', 'unreadable', 'other-first']) {
|
||||
await f.run({ kind: 'queue-insert', input: { messageId, body: automaticBody(messageId) } })
|
||||
}
|
||||
const receipt = f.db.prepare(`INSERT INTO agent_session_command_receipts
|
||||
(operation_id,session_id,caller_key,method,fingerprint,status,result_json,rejection_json,accepted_at)
|
||||
VALUES (?,'chat',?,'agentSession.send','fp','accepted',?,NULL,100)`)
|
||||
receipt.run(
|
||||
'continuation',
|
||||
'trusted-local:restart-continuation',
|
||||
JSON.stringify({ kind: 'journal-row', epoch: 'epoch', sequence: 1 })
|
||||
)
|
||||
receipt.run('unreadable', 'trusted-local:restart-continuation', 'broken-json')
|
||||
receipt.run(
|
||||
'other-first',
|
||||
'other',
|
||||
JSON.stringify({ kind: 'journal-row', epoch: 'epoch', sequence: 1 })
|
||||
)
|
||||
receipt.run(
|
||||
'other-first',
|
||||
'trusted-local:restart-continuation',
|
||||
JSON.stringify({ kind: 'journal-row', epoch: 'epoch', sequence: 1 })
|
||||
)
|
||||
const opened = await f.run({ kind: 'open' })
|
||||
expect(opened.openRecovery?.failures).toEqual([])
|
||||
expect(readJournalDomainQueuePauses(f.db, f.scope, null)).toEqual([])
|
||||
expect(journalDomainSubmission(f.scope, 'handed')?.dispatchState).toBe('pending')
|
||||
expect(journalDomainSubmission(f.scope, 'unsent')?.dispatchState).toBe('rejected')
|
||||
for (const id of ['derived', 'continuation']) {
|
||||
expect(journalDomainQueueRow(f.db, f.scope, id)?.card.state).toBe('withdrawn')
|
||||
}
|
||||
for (const id of ['unsent', 'queued', 'legacy', 'unreadable', 'other-first']) {
|
||||
expect(journalDomainQueueRow(f.db, f.scope, id)?.card.state).toBe('waiting')
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps completed holds when derived withdrawal fails and retries withdrawal on the next open', async () => {
|
||||
const f = await fixture()
|
||||
await f.run({
|
||||
kind: 'queue-insert',
|
||||
input: { messageId: 'continuation', body: automaticBody('continuation') }
|
||||
})
|
||||
f.db
|
||||
.prepare(`INSERT INTO agent_session_command_receipts
|
||||
(operation_id,session_id,caller_key,method,fingerprint,status,result_json,rejection_json,accepted_at)
|
||||
VALUES ('continuation','chat','trusted-local:restart-continuation','agentSession.send','fp','accepted',?,NULL,100)`)
|
||||
.run(JSON.stringify({ kind: 'journal-row', epoch: 'epoch', sequence: 1 }))
|
||||
f.db.exec(`CREATE TRIGGER reject_withdraw BEFORE UPDATE OF state ON journal_queue_cards
|
||||
WHEN NEW.message_id='continuation' AND NEW.state='withdrawn'
|
||||
BEGIN SELECT RAISE(ABORT,'withdraw unavailable'); END`)
|
||||
const opened = await f.run({ kind: 'open' })
|
||||
expect(opened.openRecovery?.failures).toMatchObject([
|
||||
{ step: 'withdraw-derived', failure: { message: 'withdraw unavailable' } }
|
||||
])
|
||||
expect(journalDomainSubmission(f.scope, 'unsent')?.dispatchState).toBe('rejected')
|
||||
expect(journalDomainQueueRow(f.db, f.scope, 'unsent')?.card.state).toBe('waiting')
|
||||
expect(journalDomainQueueRow(f.db, f.scope, 'continuation')?.card.state).toBe('waiting')
|
||||
f.db.exec('DROP TRIGGER reject_withdraw')
|
||||
expect((await f.run({ kind: 'open' })).openRecovery?.failures).toEqual([])
|
||||
expect(journalDomainQueueRow(f.db, f.scope, 'unsent')?.card.state).toBe('waiting')
|
||||
expect(journalDomainQueueRow(f.db, f.scope, 'continuation')?.card.state).toBe('withdrawn')
|
||||
})
|
||||
|
||||
it('marks a working subagent as unverifiable when another handle opens its saved journal', async () => {
|
||||
const f = domainLifecycleFixture(cleanups)
|
||||
await f.run({
|
||||
|
||||
@@ -14,10 +14,11 @@ import {
|
||||
} from '../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
import { staleSubagentRosterRevisions } from './item-lifecycle/journal-subagent-liveness'
|
||||
import { applyJournalDomainLifecycle } from './journal-domain-lifecycle'
|
||||
import { withdrawOpeningDerivedQueue } from './journal-open-queue-withdrawal'
|
||||
|
||||
export type JournalOpenRecoveryResult = {
|
||||
failures: {
|
||||
step: 'pending-unknown' | 'hold-unsent' | 'queue-reopen'
|
||||
step: 'pending-unknown' | 'hold-unsent' | 'withdraw-derived' | 'queue-reopen'
|
||||
failure: JournalSqlFailure
|
||||
}[]
|
||||
reopenFloor: AgentJournalCursor | null
|
||||
@@ -37,6 +38,13 @@ export function applyJournalSqlOpenCommand(
|
||||
const { target } = command
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: open returns its physical pre-recovery cursor.
|
||||
const openedThrough = opened.value as AgentJournalCursor
|
||||
const fence =
|
||||
recover?.fence ??
|
||||
Number(
|
||||
db
|
||||
.prepare('SELECT highest_fence FROM journal_chat_state WHERE session_id=? AND epoch=?')
|
||||
.get(target.sessionId, target.epoch)!.highest_fence
|
||||
)
|
||||
settleOpeningLiveWork(db, target, savedAt)
|
||||
const recovery: JournalOpenRecoveryResult = { failures: [], reopenFloor: null }
|
||||
const attempt = <Value>(
|
||||
@@ -56,23 +64,28 @@ export function applyJournalSqlOpenCommand(
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
if (recover) {
|
||||
for (const kind of ['pending-unknown', 'hold-unsent'] as const) {
|
||||
const result = attempt(kind, () =>
|
||||
applyJournalDomainInputRecovery(
|
||||
db,
|
||||
target,
|
||||
kind === 'pending-unknown'
|
||||
? { kind, fence: recover.fence }
|
||||
: { kind, fence: recover.fence, hold: { cause: 'hostRestarted' } },
|
||||
{ openedThrough },
|
||||
savedAt
|
||||
)
|
||||
for (const kind of recover
|
||||
? (['pending-unknown', 'hold-unsent'] as const)
|
||||
: (['hold-unsent'] as const)) {
|
||||
const result = attempt(kind, () =>
|
||||
applyJournalDomainInputRecovery(
|
||||
db,
|
||||
target,
|
||||
kind === 'pending-unknown'
|
||||
? { kind, fence }
|
||||
: { kind, fence, hold: { cause: 'hostRestarted' } },
|
||||
{ openedThrough },
|
||||
savedAt
|
||||
)
|
||||
if (result && 'failure' in result.value) {
|
||||
recovery.failures.push({ step: kind, failure: result.value.failure })
|
||||
}
|
||||
)
|
||||
if (result && 'failure' in result.value) {
|
||||
recovery.failures.push({ step: kind, failure: result.value.failure })
|
||||
}
|
||||
}
|
||||
attempt('withdraw-derived', () =>
|
||||
withdrawOpeningDerivedQueue(db, stateDirectory, target, openedThrough, savedAt)
|
||||
)
|
||||
if (recover) {
|
||||
attempt('queue-reopen', () => {
|
||||
if (journalDomainQueueAwaitsReopen(db, target)) {
|
||||
const head = db
|
||||
@@ -82,7 +95,7 @@ export function applyJournalSqlOpenCommand(
|
||||
applyJournalDomainControlCommand(
|
||||
db,
|
||||
target,
|
||||
{ kind: 'queue-reopen-mark', fence: recover.fence },
|
||||
{ kind: 'queue-reopen-mark', fence },
|
||||
{},
|
||||
savedAt
|
||||
)
|
||||
|
||||
+61
-1
@@ -15,9 +15,13 @@ import { structuredAgentSessionPayloadFingerprint } from '../../../../shared/age
|
||||
import type { JournalWriterOperation } from '../journal-writer-command'
|
||||
type UnsentSendHold = Extract<JournalWriterOperation, { kind: 'hold-unsent' }>['hold']
|
||||
import type { AgentSessionJournal } from '../journal-store'
|
||||
import type { JournalRow, JournalSubmissionRow } from '../durable-rows/journal-row-schema'
|
||||
import type { JournalQueueRow } from './journal-queue-events'
|
||||
import { seedTestJournalHistoricalRows } from '../replay/journal-historical-inputs.test-support'
|
||||
import {
|
||||
closeTestJournalHostDatabases,
|
||||
createTrackedJournalOpener
|
||||
createTrackedJournalOpener,
|
||||
openTestJournalHostDatabase
|
||||
} from '../store-lifecycle/journal-host-database-test-support'
|
||||
|
||||
export function createFixture() {
|
||||
@@ -116,6 +120,58 @@ export function createFixture() {
|
||||
return open()
|
||||
}
|
||||
|
||||
const historicalBase = (seq: number) => ({
|
||||
v: 4,
|
||||
epoch: 'old-queue',
|
||||
seq,
|
||||
fence: 0,
|
||||
ts: 1_000 + seq
|
||||
})
|
||||
function historicalQueue(
|
||||
seq: number,
|
||||
messageId: string,
|
||||
body = message(messageId),
|
||||
position = seq
|
||||
): JournalQueueRow {
|
||||
return {
|
||||
...historicalBase(seq),
|
||||
kind: 'queue',
|
||||
messageId,
|
||||
change: { op: 'queued', body, position, queuedAt: { epoch: 'old-queue', sequence: seq } }
|
||||
}
|
||||
}
|
||||
function historicalSend(
|
||||
seq: number,
|
||||
clientMessageId: string,
|
||||
fields: Partial<JournalSubmissionRow> = {}
|
||||
): JournalSubmissionRow {
|
||||
const body = fields.body ?? message(clientMessageId)
|
||||
return {
|
||||
...historicalBase(seq),
|
||||
kind: 'submission',
|
||||
clientMessageId,
|
||||
body,
|
||||
payloadFingerprint: fingerprint(body),
|
||||
providerHandle: claudeProviderHandle('native-1', null),
|
||||
origin: 'client',
|
||||
source: { kind: 'user' },
|
||||
handoverRecorded: true,
|
||||
...fields
|
||||
}
|
||||
}
|
||||
async function openHistorical(rows: JournalRow[]) {
|
||||
seedTestJournalHistoricalRows(openTestJournalHostDatabase(root).db, IDENTITY, [
|
||||
{
|
||||
...historicalBase(1),
|
||||
kind: 'epoch',
|
||||
reason: 'session_created',
|
||||
providerHandle: claudeProviderHandle('native-1', null)
|
||||
},
|
||||
...rows
|
||||
])
|
||||
return open()
|
||||
}
|
||||
|
||||
async function cardOrder(journal: AgentSessionJournal): Promise<string[]> {
|
||||
return (await journal.queuedMessages.list())
|
||||
.filter((card) => card.state === 'waiting')
|
||||
@@ -155,6 +211,10 @@ export function createFixture() {
|
||||
},
|
||||
fingerprint,
|
||||
hold,
|
||||
historicalBase,
|
||||
historicalQueue,
|
||||
historicalSend,
|
||||
openHistorical,
|
||||
journals,
|
||||
message,
|
||||
open,
|
||||
|
||||
+56
-60
@@ -16,32 +16,22 @@ export function registerJournalUnsentSendHoldQueueTests(
|
||||
])(
|
||||
'in acceptance order, ahead of the cards already waiting, a hand-off by $by among them',
|
||||
async ({ origin, order }) => {
|
||||
const journal = await fixture.afterRestart(async (earlier) => {
|
||||
await earlier.queuedMessages.insert({
|
||||
messageId: 'H',
|
||||
body: fixture.message('H')
|
||||
})
|
||||
await earlier.queuedMessages.insert({
|
||||
messageId: 'C',
|
||||
body: fixture.message('C')
|
||||
})
|
||||
await fixture.accept(earlier, 'A', { origin: 'client', source: USER_MESSAGE_SOURCE })
|
||||
await earlier.appendSubmission(
|
||||
{
|
||||
clientMessageId: 'H-handoff',
|
||||
payloadFingerprint: fixture.fingerprint(fixture.message('H')),
|
||||
body: fixture.message('H'),
|
||||
fence: 0,
|
||||
handoverRecorded: true,
|
||||
origin
|
||||
},
|
||||
{ messageId: 'H', expect: 'waiting' }
|
||||
)
|
||||
await fixture.accept(earlier, 'B', { origin: 'client', source: USER_MESSAGE_SOURCE })
|
||||
})
|
||||
const journal = await fixture.openHistorical([
|
||||
fixture.historicalQueue(2, 'H'),
|
||||
fixture.historicalQueue(3, 'C'),
|
||||
fixture.historicalSend(4, 'A'),
|
||||
fixture.historicalSend(5, 'H-handoff', {
|
||||
body: fixture.message('H'),
|
||||
queuedMessageId: 'H',
|
||||
origin
|
||||
}),
|
||||
fixture.historicalSend(6, 'B')
|
||||
])
|
||||
await fixture.hold(journal)
|
||||
|
||||
expect(await fixture.cardOrder(journal)).toEqual(order)
|
||||
await journal.close()
|
||||
expect(await fixture.cardOrder(await fixture.open())).toEqual(order)
|
||||
}
|
||||
)
|
||||
})
|
||||
@@ -81,43 +71,49 @@ export function registerJournalUnsentSendHoldQueueTests(
|
||||
it.each(['stopped', 'reserved-command', 'moved-command'] as const)(
|
||||
'preserves the %s ordering boundary',
|
||||
async (shape) => {
|
||||
const journal = await fixture.afterRestart(async (earlier) => {
|
||||
if (shape === 'stopped') {
|
||||
await earlier.queuedMessages.insert({ messageId: 'W', body: fixture.message('W') })
|
||||
await earlier.appendStopEvent({ reason: 'user-stop' }, 0)
|
||||
await fixture.accept(earlier, 'S', { origin: 'client', source: USER_MESSAGE_SOURCE })
|
||||
return
|
||||
}
|
||||
await earlier.queuedMessages.insert({
|
||||
messageId: 'C1',
|
||||
body: { ...fixture.message('/compact'), command: { name: 'compact' as const } }
|
||||
})
|
||||
await fixture.accept(earlier, 'S', { origin: 'client', source: USER_MESSAGE_SOURCE })
|
||||
await earlier.queuedMessages.insert({ messageId: 'C2', body: fixture.message('C2') })
|
||||
await earlier.queuedMessages.move({ messageId: 'C2', beforeMessageId: 'C1' })
|
||||
if (shape === 'reserved-command') {
|
||||
await earlier.appendSubmission(
|
||||
{
|
||||
clientMessageId: 'command-attempt',
|
||||
origin: 'client',
|
||||
body: { ...fixture.message('/compact'), command: { name: 'compact' as const } },
|
||||
payloadFingerprint: fixture.fingerprint({
|
||||
...fixture.message('/compact'),
|
||||
command: { name: 'compact' as const }
|
||||
}),
|
||||
handoverRecorded: true,
|
||||
fence: 0
|
||||
},
|
||||
{ messageId: 'C1', expect: 'waiting' }
|
||||
)
|
||||
await earlier.resolveDispatch({
|
||||
clientMessageId: 'command-attempt',
|
||||
state: 'pending',
|
||||
turnScope: { kind: 'thread' },
|
||||
fence: 0
|
||||
})
|
||||
}
|
||||
})
|
||||
const command = { ...fixture.message('/compact'), command: { name: 'compact' as const } }
|
||||
const journal = await fixture.openHistorical(
|
||||
shape === 'stopped'
|
||||
? [
|
||||
fixture.historicalQueue(2, 'W'),
|
||||
{
|
||||
...fixture.historicalBase(3),
|
||||
kind: 'tombstone',
|
||||
itemId: 'orca:stop',
|
||||
revision: 1,
|
||||
stopEvent: { reason: 'user-stop', at: 1_003 }
|
||||
},
|
||||
fixture.historicalSend(4, 'S')
|
||||
]
|
||||
: [
|
||||
fixture.historicalQueue(2, 'C1', command),
|
||||
fixture.historicalSend(3, 'S'),
|
||||
fixture.historicalQueue(4, 'C2'),
|
||||
{
|
||||
...fixture.historicalBase(5),
|
||||
kind: 'queue',
|
||||
messageId: 'C2',
|
||||
change: { op: 'moved', position: 1 }
|
||||
},
|
||||
...(shape === 'reserved-command'
|
||||
? [
|
||||
fixture.historicalSend(6, 'command-attempt', {
|
||||
body: command,
|
||||
queuedMessageId: 'C1'
|
||||
}),
|
||||
{
|
||||
...fixture.historicalBase(7),
|
||||
kind: 'dispatch' as const,
|
||||
clientMessageId: 'command-attempt',
|
||||
state: 'pending' as const,
|
||||
providerItemId: null,
|
||||
reason: null,
|
||||
turnScope: { kind: 'thread' as const }
|
||||
}
|
||||
]
|
||||
: [])
|
||||
]
|
||||
)
|
||||
expect(await fixture.cardOrder(journal)).toEqual(
|
||||
shape === 'stopped'
|
||||
? ['S', 'W']
|
||||
|
||||
+54
-24
@@ -2,7 +2,9 @@ import { seedTestJournalHistoricalRows } from '../replay/journal-historical-inpu
|
||||
import { openTestJournalHostDatabase } from '../store-lifecycle/journal-host-database-test-support'
|
||||
import type { createFixture } from './queued-message-pause-contract-test-fixture'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { QueuedMessageNotConsumableError } from './queued-message-consume-error'
|
||||
import { agentSessionRecordFixture } from '../../agent-session-record-test-fixture'
|
||||
import { encodeAgentSessionRecord } from '../../../../shared/agent-session/lifecycle/agent-session-record-stored-form'
|
||||
import type { JournalAutomaticSelection } from '../journal-automatic-submission'
|
||||
|
||||
export function registerQueuedMessageStopPauseTests(
|
||||
fixture: ReturnType<typeof createFixture>
|
||||
@@ -35,29 +37,58 @@ export function registerQueuedMessageStopPauseTests(
|
||||
})
|
||||
|
||||
it("the queue's own consume is refused in its transaction while the pause holds the card; Send-now is not", async () => {
|
||||
const record = agentSessionRecordFixture({ sessionId: fixture.IDENTITY.sessionId })
|
||||
openTestJournalHostDatabase(fixture.root)
|
||||
.db.prepare('INSERT INTO agent_session_records(session_id,record_json) VALUES(?,?)')
|
||||
.run(record.sessionId, JSON.stringify(encodeAgentSessionRecord(record)))
|
||||
const journal = await fixture.open()
|
||||
await fixture.queueDraft(journal, 'draft-1')
|
||||
await fixture.userStop(journal)
|
||||
const consume = (id: string, automatic: boolean) =>
|
||||
journal.appendSubmission(
|
||||
{
|
||||
clientMessageId: id,
|
||||
payloadFingerprint: fixture.fingerprintOf('draft-1'),
|
||||
body: fixture.message('draft-1'),
|
||||
fence: 0,
|
||||
handoverRecorded: true
|
||||
const selection: JournalAutomaticSelection = {
|
||||
authority: {
|
||||
handle: journal.observationHandle(),
|
||||
record: {
|
||||
currentFence: 1,
|
||||
ownerObservation: { claimStatus: 'live', unreconciled: false, handoffStage: null }
|
||||
},
|
||||
{
|
||||
messageId: 'draft-1',
|
||||
expect: 'waiting',
|
||||
...(automatic ? { yieldsToPause: true as const } : {})
|
||||
}
|
||||
child: null,
|
||||
handingOver: [],
|
||||
continuingUnadmitted: [],
|
||||
finalizing: [],
|
||||
openedThrough: journal.openingCursor(),
|
||||
operationalRevision: 0
|
||||
},
|
||||
child: null,
|
||||
steering: { kind: 'codex-unavailable' }
|
||||
}
|
||||
const automatic = (clientMessageId: string) =>
|
||||
journal.claimAutomaticSubmission(
|
||||
{ clientMessageId, fence: 1 },
|
||||
async () => selection,
|
||||
() => 'owed'
|
||||
)
|
||||
await expect(consume('drain-1', true)).rejects.toBeInstanceOf(QueuedMessageNotConsumableError)
|
||||
expect(await automatic('drain-1')).toEqual({ kind: 'waiting' })
|
||||
expect(await journal.reads.submission('drain-1')).toBeUndefined()
|
||||
expect((await journal.queuedMessages.get('draft-1'))?.state).toBe('waiting')
|
||||
await consume('send-now-1', false)
|
||||
expect((await journal.queuedMessages.get('draft-1'))?.state).toBe('dispatched')
|
||||
await journal.appendQueueResume(1)
|
||||
expect(await automatic('drain-after-resume')).toEqual({
|
||||
kind: 'claimed',
|
||||
messageId: 'draft-1',
|
||||
clientMessageId: 'drain-after-resume'
|
||||
})
|
||||
await fixture.queueDraft(journal, 'draft-2')
|
||||
await journal.appendStopEvent({ reason: 'user-stop' }, 1)
|
||||
await journal.appendSubmission(
|
||||
{
|
||||
clientMessageId: 'send-now-1',
|
||||
payloadFingerprint: fixture.fingerprintOf('draft-2'),
|
||||
body: fixture.message('draft-2'),
|
||||
fence: 1,
|
||||
handoverRecorded: true
|
||||
},
|
||||
{ messageId: 'draft-2', expect: 'waiting' }
|
||||
)
|
||||
expect((await journal.queuedMessages.get('draft-2'))?.state).toBe('dispatched')
|
||||
})
|
||||
|
||||
it("a rewind's epoch replacement restates a Stop still pausing, and not one a turn ended", async () => {
|
||||
@@ -194,17 +225,16 @@ export function registerQueuedMessageStopPauseTests(
|
||||
state: 'accepted',
|
||||
providerItemId: null,
|
||||
reason: null
|
||||
},
|
||||
{
|
||||
...row(6),
|
||||
kind: 'tombstone',
|
||||
itemId: 'orca:retired-person',
|
||||
revision: 1,
|
||||
retireSubmission: true
|
||||
}
|
||||
],
|
||||
true
|
||||
)
|
||||
const db = openTestJournalHostDatabase(fixture.root).db
|
||||
const source = db.prepare('SELECT generation,through_seq FROM journal_outcome_epochs').get()!
|
||||
db.prepare(
|
||||
"INSERT INTO journal_outcomes (generation,seq,field,entry_key,format,ordinal,value_json,deleted) VALUES (?,?,'submissions',?,2,'s0000000000000001:0000000000000001','null',1)"
|
||||
).run(source.generation, Number(source.through_seq) + 1, JSON.stringify('retired-person'))
|
||||
db.prepare('UPDATE journal_outcome_epochs SET through_seq=through_seq+1').run()
|
||||
const restored = await fixture.open()
|
||||
expect(await restored.reads.submission('retired-person')).toBeUndefined()
|
||||
expect((await restored.queuedMessages.nextInputCard())?.messageId).toBe('W')
|
||||
|
||||
+1
@@ -105,6 +105,7 @@ export async function openStructuredAgentSessionConversationJournal(
|
||||
const messages = {
|
||||
'pending-unknown': 'marking pending sends unknown on open failed',
|
||||
'hold-unsent': 'settling sends an earlier process left queued failed on open',
|
||||
'withdraw-derived': 'withdrawing replaced restart work failed on open',
|
||||
'queue-reopen': 'marking a reopened queue failed'
|
||||
}
|
||||
deps.logger.warn(messages[step], {
|
||||
|
||||
Reference in New Issue
Block a user