refactor: use one chat durability mode and ready batch selector

This commit is contained in:
Brennan Benson
2026-11-08 05:22:30 -08:00
parent 2a06b43270
commit 5dae0b92d5
25 changed files with 205 additions and 638 deletions
@@ -83,24 +83,20 @@ it('rolls back the entire Codex exit settlement when a later row fails', async (
const diskBefore = liveTestJournalRows(database.db, SESSION)
const sawEarlierInserts: number[] = []
const transaction = database.transaction.bind(database)
vi.spyOn(database, 'transaction').mockImplementation((run, streamed, committed) =>
transaction(
(db) => {
try {
return run(db)
} catch (error) {
sawEarlierInserts.push(liveTestJournalRows(db, SESSION).length - diskBefore.length)
expect(journal.snapshot()).toEqual(before)
expect(commits).not.toHaveBeenCalled()
expect(
published.every((snapshot) => JSON.stringify(snapshot) === JSON.stringify(before))
).toBe(true)
throw error
}
},
streamed,
committed
)
vi.spyOn(database, 'transaction').mockImplementation((run, committed) =>
transaction((db) => {
try {
return run(db)
} catch (error) {
sawEarlierInserts.push(liveTestJournalRows(db, SESSION).length - diskBefore.length)
expect(journal.snapshot()).toEqual(before)
expect(commits).not.toHaveBeenCalled()
expect(
published.every((snapshot) => JSON.stringify(snapshot) === JSON.stringify(before))
).toBe(true)
throw error
}
}, committed)
)
database.db.exec(`CREATE TEMP TRIGGER reject_codex_later BEFORE INSERT ON journal_rows
WHEN NEW.session_id = '${SESSION}' AND NEW.seq > ${before.cursor.sequence + 1}
@@ -74,7 +74,6 @@ export function publishNewEpoch(input: {
publishJournalSessionEpoch(db, input.identity, input.epoch)
return publishJournalOutcomeEpoch(db, sessionId, state.epoch, rows)
},
false,
(publication) => {
// Adopt, then finish the nonthrowing queue bookkeeping, before the fallible publication.
const state = publication.adopt()
@@ -10,7 +10,7 @@ try {
if (db.prepare('PRAGMA journal_mode').get().journal_mode !== 'wal') {
throw new Error('chat journal is not in WAL mode');
}
db.exec('PRAGMA busy_timeout = 0; PRAGMA synchronous = FULL; PRAGMA checkpoint_fullfsync = ON;');
db.exec('PRAGMA busy_timeout = 0; PRAGMA synchronous = NORMAL; PRAGMA checkpoint_fullfsync = ON;');
db.exec('PRAGMA journal_size_limit = ' + workerData.sizeLimit);
const frames = db.prepare('PRAGMA wal_checkpoint(NOOP)');
const checkpoint = db.prepare('PRAGMA wal_checkpoint(PASSIVE)');
@@ -16,7 +16,7 @@ import { ensureAgentSessionAttachmentClaimTables } from '../../agent-session-att
export const JOURNAL_BUSY_TIMEOUT_MS = 5000
export const JOURNAL_SIZE_LIMIT_BYTES = 32 * 1024 * 1024
const JOURNAL_SYNCHRONOUS = 'FULL'
const JOURNAL_SYNCHRONOUS = 'NORMAL'
export type OpenJournalDatabase = {
db: Database.Database
@@ -97,7 +97,7 @@ function configureJournalPragmas(db: Database.Database, stored: number): void {
db.pragma('journal_mode = WAL')
db.pragma(`busy_timeout = ${JOURNAL_BUSY_TIMEOUT_MS}`)
db.pragma('foreign_keys = ON')
// FULL fsyncs submissions before dispatch; checkpoint_fullfsync also flushes macOS drive caches.
// WAL checkpoints flush committed pages and macOS drive caches.
db.pragma(`synchronous = ${JOURNAL_SYNCHRONOUS}`)
db.pragma('checkpoint_fullfsync = ON')
db.pragma(`journal_size_limit = ${JOURNAL_SIZE_LIMIT_BYTES}`)
@@ -72,35 +72,13 @@ export class JournalHostDatabase {
return readJournalSessionEpoch(this.db, sessionId)
}
/** Streamed text may lose only its trailing suffix on OS crash; the next FULL commit makes it durable. */
transaction<T>(
run: (db: Database.Database) => T,
streamed = false,
committed?: (result: T) => void
): T {
transaction<T>(run: (db: Database.Database) => T, committed?: (result: T) => void): T {
const db = this.db
this.transactionNumber += 1
if (!streamed) {
const result = runJournalTransaction(db, run, () => {
this.stranded = true
})
committed?.(result)
this.checkpoint?.afterCommit()
return result
}
// Synchronous level changes require no transaction; stranded recovery restores FULL first.
db.pragma('synchronous = NORMAL')
let result: T
try {
result = runJournalTransaction(db, run, () => {
this.stranded = true
})
committed?.(result)
} finally {
if (!db.isTransaction) {
db.pragma('synchronous = FULL')
}
}
const result = runJournalTransaction(db, run, () => {
this.stranded = true
})
committed?.(result)
this.checkpoint?.afterCommit()
return result
}
@@ -139,7 +117,6 @@ export class JournalHostDatabase {
throw journalOpenRefusalError(error)
}
}
connection.pragma('synchronous = FULL')
this.stranded = false
}
}
@@ -1,7 +1,9 @@
import { afterEach, expect, it, vi } from 'vitest'
import Database from '../../sqlite/sync-database'
import { JournalSqlBatch, JournalSqlBatchError } from './journal-sql-batch'
import { takeJournalSqlReadyBatch } from './journal-sql-ready-batch'
import { prepareJournalSqlRequest, takeJournalSqlReadyBatch } from './journal-sql-ready-batch'
import { prepareJournalSqlMessages } from './journal-sql-message-commit'
import { message } from './journal-sql-message.test-fixture'
import type { JournalSqlRequest } from './journal-sql-protocol'
const databases: Database.Database[] = []
@@ -22,21 +24,14 @@ function fixture() {
it('rolls back a failed command while later commands see preceding successful writes', async () => {
const { db, batch } = fixture()
const failure = new Error('command failed')
const answer = await batch.run(
[
{ command: 1, normal: true },
{ command: 2, normal: true },
{ command: 3, normal: false }
],
(id) => {
const previous = db.prepare('SELECT id FROM writes ORDER BY id').all()
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written')
if (id === 2) {
throw failure
}
return previous
const answer = await batch.run([1, 2, 3], (id) => {
const previous = db.prepare('SELECT id FROM writes ORDER BY id').all()
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written')
if (id === 2) {
throw failure
}
)
return previous
})
expect(answer).toEqual({
outcomes: [
{ committed: true, value: [] },
@@ -51,38 +46,19 @@ it('rolls back a failed command while later commands see preceding successful wr
expect(db.isTransaction).toBe(false)
})
it.each([
{ normal: [true, true], synchronous: 1 },
{ normal: [true, false], synchronous: 2 }
])('uses batch durability $synchronous and restores FULL before the next batch', async (input) => {
it('leaves newly admitted work for a new drain', async () => {
const { db, batch } = fixture()
const answer = await batch.run(
input.normal.map((normal, id) => ({ command: id, normal })),
(id) => {
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written')
return db.pragma('synchronous', { simple: true })
}
)
expect(answer.outcomes).toEqual([
{ committed: true, value: input.synchronous },
{ committed: true, value: input.synchronous }
])
expect(db.pragma('synchronous', { simple: true })).toBe(2)
})
it('leaves newly admitted work for a new drain with its own durability', async () => {
const { db, batch } = fixture()
const admitted = [{ command: 1, normal: true }]
const admitted = [1]
const first = await batch.run(admitted, (id) => {
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'stream')
admitted.push({ command: 2, normal: false })
admitted.push(2)
return id
})
expect(first.outcomes).toEqual([{ committed: true, value: 1 }])
expect(db.prepare('SELECT id, value FROM writes').all()).toEqual([{ id: 1, value: 'stream' }])
const send = await batch.run(admitted.slice(1), (id) => {
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'send')
return db.pragma('synchronous', { simple: true })
return id
})
expect(send.outcomes).toEqual([{ committed: true, value: 2 }])
expect(db.prepare('SELECT id, value FROM writes ORDER BY id').all()).toEqual([
@@ -91,43 +67,39 @@ it('leaves newly admitted work for a new drain with its own durability', async (
])
})
it('commits a control receipt before later streamed work with its own durability', async () => {
it('keeps bounded snapshot prefixes and controls in enqueue order', async () => {
const { db, batch } = fixture()
const stream = (id: number): JournalSqlRequest => ({
id,
command: {
type: 'write',
command: {
target: { sessionId: 'session', epoch: 'epoch', ownerToken: 'owner' },
operation: {
kind: 'item',
identity: { provider: 'orca', clientMessageId: 'output' },
body: { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'output' }] },
options: { fence: 1, turnScope: { kind: 'thread' } }
}
}
}
})
const stream = (id: number, steps: number, text = 'output'): JournalSqlRequest =>
prepareJournalSqlRequest(id, {
type: 'messages',
job: prepareJournalSqlMessages({
target: { sessionId: `session${id}`, epoch: 'epoch' },
steps: Array.from({ length: steps }, () => message(text)),
savedAt: 100
})
})
const queue: JournalSqlRequest[] = [
stream(1),
{ id: 2, command: { type: 'effects', effects: [] } },
stream(3)
stream(1, 32),
stream(2, 32),
stream(3, 1),
{ id: 4, command: { type: 'effects', effects: [] } },
stream(5, 1, 'x'.repeat(700 * 1024)),
stream(6, 1, 'x'.repeat(700 * 1024))
]
const commit = async () =>
batch.run(
takeJournalSqlReadyBatch(queue, () => true).map(({ request, normal }) => ({
command: request,
normal
})),
(request) => {
db.prepare('INSERT INTO writes VALUES (?, ?)').run(request.id, 'written')
return { id: request.id, durability: db.pragma('synchronous', { simple: true }) }
}
)
expect((await commit()).outcomes).toEqual([{ committed: true, value: { id: 1, durability: 1 } }])
expect((await commit()).outcomes).toEqual([{ committed: true, value: { id: 2, durability: 2 } }])
batch.run(takeJournalSqlReadyBatch(queue), (request) => {
db.prepare('INSERT INTO writes VALUES (?, ?)').run(request.id, 'written')
return request.id
})
expect((await commit()).outcomes).toEqual([
{ committed: true, value: 1 },
{ committed: true, value: 2 }
])
expect(db.prepare('SELECT id FROM writes ORDER BY id').all()).toEqual([{ id: 1 }, { id: 2 }])
expect((await commit()).outcomes).toEqual([{ committed: true, value: { id: 3, durability: 1 } }])
expect((await commit()).outcomes).toEqual([{ committed: true, value: 3 }])
expect((await commit()).outcomes).toEqual([{ committed: true, value: 4 }])
expect((await commit()).outcomes).toEqual([{ committed: true, value: 5 }])
expect((await commit()).outcomes).toEqual([{ committed: true, value: 6 }])
})
it.each([false, true])(
@@ -146,7 +118,7 @@ it.each([false, true])(
})
let failure: unknown
try {
await batch.run([{ command: 1, normal: false }], (id) => {
await batch.run([1], (id) => {
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written')
return id
})
@@ -156,34 +128,13 @@ it.each([false, true])(
expect(failure).toBeInstanceOf(JournalSqlBatchError)
expect(failure).toMatchObject({ outcome: 'indeterminate' })
expect(db.prepare('SELECT id FROM writes').all()).toEqual(committed ? [{ id: 1 }] : [])
await expect(batch.run([{ command: 2, normal: false }], (id) => id)).rejects.toThrow(
await expect(batch.run([2], (id) => id)).rejects.toThrow(
expect.objectContaining({ outcome: 'not-dispatched' })
)
expect(db.prepare('SELECT id FROM writes').all()).toEqual(committed ? [{ id: 1 }] : [])
}
)
it('acknowledges a committed stream even when restoring FULL fails afterwards', async () => {
const { db, batch } = fixture()
const pragma = db.pragma.bind(db)
let committed = false
const failure = new Error('FULL setting failed')
vi.spyOn(db, 'pragma').mockImplementation((sql, options) => {
if (sql === 'synchronous = FULL' && committed) {
throw failure
}
return pragma(sql, options)
})
const answer = await batch.run([{ command: 1, normal: true }], (id) => {
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written')
committed = true
return id
})
expect(answer).toEqual({ outcomes: [{ committed: true, value: 1 }], postcommit: failure })
expect(db.prepare('SELECT id FROM writes').all()).toEqual([{ id: 1 }])
expect(batch.fault).toBe(failure)
})
it('does not claim rollback when both command and outer rollback fail', async () => {
const { db, batch } = fixture()
const exec = db.exec.bind(db)
@@ -195,13 +146,13 @@ it('does not claim rollback when both command and outer rollback fail', async ()
exec(sql)
})
await expect(
batch.run([{ command: 1, normal: false }], (id) => {
batch.run([1], (id) => {
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'uncommitted')
throw new Error('command failed')
})
).rejects.toThrow(expect.objectContaining({ outcome: 'indeterminate', rollback }))
expect(db.isTransaction).toBe(true)
await expect(batch.run([{ command: 2, normal: false }], (id) => id)).rejects.toThrow(
await expect(batch.run([2], (id) => id)).rejects.toThrow(
expect.objectContaining({ outcome: 'not-dispatched' })
)
exec('ROLLBACK')
@@ -6,7 +6,6 @@ export type JournalSqlBatchOutcome<Value> =
export type JournalSqlBatchResult<Value> = {
outcomes: JournalSqlBatchOutcome<Value>[]
postcommit?: unknown
}
export class JournalSqlBatchError extends Error {
@@ -33,7 +32,7 @@ export class JournalSqlBatch {
/** Commands may await only microtasks: the worker owns the connection while one is open. */
async run<Command, Value>(
commands: readonly { command: Command; normal: boolean }[],
commands: readonly Command[],
apply: (command: Command) => Value | Promise<Value>
): Promise<JournalSqlBatchResult<Value>> {
if (this.failure) {
@@ -44,20 +43,13 @@ export class JournalSqlBatch {
return { outcomes }
}
const drain = commands.slice()
const normal = drain.every((entry) => entry.normal)
let began = false
let committing = false
let failed: JournalSqlBatchError | undefined
let postcommit: unknown
try {
// Re-prove FULL before every drain, including after a failed PRAGMA from an earlier drain.
this.db.pragma('synchronous = FULL')
if (normal) {
this.db.pragma('synchronous = NORMAL')
}
this.db.exec('BEGIN IMMEDIATE')
began = true
for (const { command } of drain) {
for (const command of drain) {
this.db.exec('SAVEPOINT journal_command')
try {
const value = await apply(command)
@@ -88,20 +80,10 @@ export class JournalSqlBatch {
if (committing || rollback) {
this.failure = failed
}
} finally {
if (normal && !this.db.isTransaction) {
try {
this.db.pragma('synchronous = FULL')
} catch (error) {
// The COMMIT answer remains true; the owner reports this fault after its acknowledgement.
postcommit = error
this.failure ??= error
}
}
}
if (failed) {
throw failed
}
return { outcomes, ...(postcommit ? { postcommit } : {}) }
return { outcomes }
}
}
@@ -71,13 +71,10 @@ it('commits a step prefix without leaking the rolled-back session or admitting i
const answer = (
await batch.run(
[
{
command: command({
kind: 'steps',
steps: [step('first'), step('refused'), step('suffix')]
}),
normal: false
}
command({
kind: 'steps',
steps: [step('first'), step('refused'), step('suffix')]
})
],
(entry) => engine.apply(entry, 100)
)
@@ -161,10 +158,7 @@ it('assigns revisions after preceding successes and rolls back failed command pr
]
}
const results = await batch.run(
[accepted, duplicate, command(item('second')), command(item('stale', 0))].map((entry) => ({
command: entry,
normal: engine.streamed(entry)
})),
[accepted, duplicate, command(item('second')), command(item('stale', 0))],
(entry) => engine.apply(entry, 100)
)
expect(results.outcomes.map((result) => result.committed)).toEqual([true, false, true, false])
@@ -240,11 +234,8 @@ it('keeps exact saved records across command shells and publishes metadata witho
memory = persistJournalOutcomeRows(oracle, memory, source, rows, 100, false, (state) =>
journalInputOrder(state).cards.ensure(state)
).adopt()
const result = (
await batch.run([{ command: command(operation), normal: false }], (entry) =>
engine.apply(entry, 100)
)
).outcomes[0]!
const result = (await batch.run([command(operation)], (entry) => engine.apply(entry, 100)))
.outcomes[0]!
expect(result.committed).toBe(true)
}
const records = (connection: Database.Database) =>
@@ -254,9 +245,7 @@ it('keeps exact saved records across command shells and publishes metadata witho
.all()
expect(records(db)).toEqual(records(oracle))
const final = (
await batch.run([{ command: command(item('after deletion')), normal: false }], (entry) =>
engine.apply(entry, 100)
)
await batch.run([command(item('after deletion'))], (entry) => engine.apply(entry, 100))
).outcomes[0]!
expect(final).toMatchObject({
committed: true,
@@ -265,55 +254,6 @@ it('keeps exact saved records across command shells and publishes metadata witho
expect(JSON.stringify(final)).not.toContain('after deletion')
})
it('chooses NORMAL only for proven text revisions and retains FULL for settlements and effects', async () => {
const { db, engine, batch } = fixture()
expect(engine.streamed(command(item('first')))).toBe(false)
await batch.run([{ command: command(item('first')), normal: false }], (entry) =>
engine.apply(entry, 100)
)
const revision = command(item('next'))
const textStep: JournalWriterStep = {
kind: 'resolved-item',
reservedBytes: 4096,
resolution: {
kind: 'text-snapshot',
identity,
body: { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'streamed' }] }
},
options: { fence: 1, turnScope: { kind: 'thread' } }
}
const steps = command({ kind: 'steps', steps: [textStep] })
db.pragma('query_only = ON')
try {
expect(engine.streamed(revision)).toBe(true)
expect(engine.streamed(steps)).toBe(true)
expect(
engine.streamed(
command({
kind: 'steps',
steps: [{ ...textStep, options: { ...textStep.options, executionOwner: 2 } }]
})
)
).toBe(false)
} finally {
db.pragma('query_only = OFF')
}
expect(
engine.streamed({
...revision,
effects: [{ kind: 'queue-delete', sessionId: 'session', messageIds: [] }]
})
).toBe(false)
expect(engine.streamed(command({ kind: 'queue-resume', fence: 1 }))).toBe(false)
let synchronous = -1
await batch.run([{ command: steps, normal: engine.streamed(steps) }], (entry) => {
synchronous = Number(db.pragma('synchronous', { simple: true }))
return engine.apply(entry, 100)
})
expect(synchronous).toBe(1)
expect(Number(db.pragma('synchronous', { simple: true }))).toBe(2)
})
it('refuses stale ownership, changed epochs and log coverage even for an empty plan', async () => {
const { db, engine, batch } = fixture()
const noOp = command({
@@ -322,8 +262,7 @@ it('refuses stale ownership, changed epochs and log coverage even for an empty p
onlyIfUnsettled: true
})
const run = async (entry = noOp) =>
(await batch.run([{ command: entry, normal: false }], (value) => engine.apply(value, 100)))
.outcomes[0]!
(await batch.run([entry], (value) => engine.apply(value, 100))).outcomes[0]!
expect(
await run({
...noOp,
@@ -354,8 +293,7 @@ it('refuses stale ownership, changed epochs and log coverage even for an empty p
it('replaces the epoch inside one writer command, moving the logical cursor epoch', async () => {
const { engine, batch } = fixture()
const run = async (entry: JournalWriterCommand) =>
(await batch.run([{ command: entry, normal: false }], (value) => engine.apply(value, 100)))
.outcomes[0]!
(await batch.run([entry], (value) => engine.apply(value, 100))).outcomes[0]!
const before = await run(command(item('before')))
const replaced = await run(
command({
@@ -5,11 +5,8 @@ import type { JournalWriterCommit } from './journal-writer-changes'
import { applyJournalWriterEffect, type JournalWriterEffectResult } from './journal-writer-effects'
import { JournalWriterSession } from './journal-writer-session'
import { runJournalWriterOperation } from './journal-writer-operations'
import { planJournalWriterRows } from './journal-writer-plan'
import type { JournalOperationReceipt } from './writing/journal-row-writer'
import { runJournalSqlSteps } from './journal-sql-steps'
import { isJournalStreamedTextRevision } from './writing/journal-streamed-text-revision'
import { planJournalWriterStep } from './writing/journal-writer-step-plan'
import {
claimJournalSqlProjection,
journalLogNeedsFounding,
@@ -22,40 +19,6 @@ export class JournalSqlCommand {
private readonly stateDirectory: string
) {}
/** Conservative preflight only; all authority and revision assignment are repeated in apply(). */
streamed(command: JournalWriterCommand): boolean {
if (command.effects) {
return false
}
try {
const { state } = this.session(command, 0)
const { operation } = command
if (operation.kind === 'steps') {
return operation.steps.every((step) => {
if (step.kind === 'settlement' || step.kind === 'creation-fact') {
return false
}
const planned = planJournalWriterStep(state, step, () => {})
return (
planned === null ||
planned.every((build) =>
isJournalStreamedTextRevision(state, build(state.lastSequence + 1, 0))
)
)
})
}
if (operation.kind !== 'item') {
return false
}
return planJournalWriterRows(state, operation).every((build) =>
isJournalStreamedTextRevision(state, build(state.lastSequence + 1, 0))
)
} catch {
// A stale/invalid command fails its own SAVEPOINT; it must not downgrade the batch.
return false
}
}
async apply(command: JournalWriterCommand, savedAt: number): Promise<JournalWriterCommit> {
if (!this.db.isTransaction) {
throw new Error('journal SQL command requires its transaction')
@@ -145,9 +145,7 @@ export async function sides() {
facts: { identity, ...facts },
savedAt: NOW
}
const [outcome] = (
await batch.run([{ command, normal: false }], (entry) => engine.apply(entry, NOW))
).outcomes
const [outcome] = (await batch.run([command], (entry) => engine.apply(entry, NOW))).outcomes
if (!outcome!.committed) {
// Exactly what reaches the host through the worker port.
throw restoreJournalSqlFailure(journalSqlFailure(outcome!.error))
@@ -39,11 +39,6 @@ export type JournalSqlMaintenanceResult =
| boolean
| null
/** Each call is one writer batch; only the final publication needs FULL durability. */
export function journalSqlMaintenanceIsNormal(operation: JournalSqlMaintenance): boolean {
return operation.kind !== 'conversion-finish' && operation.kind !== 'projection-state'
}
export function applyJournalSqlMaintenance(
db: Database.Database,
operation: JournalSqlMaintenance
@@ -1,12 +1,7 @@
import type Database from '../../sqlite/sync-database'
import type {
JournalSqlBatch,
JournalSqlBatchOutcome,
JournalSqlBatchResult
} from './journal-sql-batch'
import type { JournalSqlBatch, JournalSqlBatchResult } from './journal-sql-batch'
import {
applyJournalSqlMessage,
JournalMessageNeedsFull,
type JournalSqlMessage,
type JournalSqlMessageCommit
} from './journal-sql-message'
@@ -20,7 +15,6 @@ type MessageCommand = {
export type JournalSqlMessageJob = {
command: MessageCommand
bytes: number
normal: boolean
}
export type JournalSqlMessagesCommit = Omit<JournalSqlMessageCommit, 'outcome'> & {
@@ -32,47 +26,10 @@ export type JournalSqlMessagesCommit = Omit<JournalSqlMessageCommit, 'outcome'>
export function prepareJournalSqlMessages(command: MessageCommand): JournalSqlMessageJob {
return {
command,
bytes: Buffer.byteLength(JSON.stringify(command), 'utf8'),
normal: command.steps.length > 0 && command.steps.every((step) => step.kind === 'text-snapshot')
bytes: Buffer.byteLength(JSON.stringify(command), 'utf8')
}
}
/** Select one ready prefix; controls and repeated chats keep their serialization boundary. */
export function journalSqlMessageBatchSize<T>(
ready: readonly T[],
jobFor: (entry: T) => JournalSqlMessageJob | undefined,
at = 0
): number {
const sessions = new Set<string>()
let bytes = 0
let events = 0
let count = 0
for (let next = at; next < ready.length; next++) {
const job = jobFor(ready[next]!)
if (!job) {
break
}
const size = Math.max(1, job.command.steps.length)
if (
count &&
(!job.normal ||
sessions.has(job.command.target.sessionId) ||
bytes + job.bytes > 1024 * 1024 ||
events + size > 64)
) {
break
}
count++
bytes += job.bytes
events += size
sessions.add(job.command.target.sessionId)
if (!job.normal || bytes >= 1024 * 1024 || events >= 64) {
break
}
}
return count
}
/** Called by the existing writer admission lane; publication completes before the next drain. */
export async function commitJournalSqlMessages(
db: Database.Database,
@@ -80,48 +37,13 @@ export async function commitJournalSqlMessages(
jobs: readonly JournalSqlMessageJob[],
publish: (commit: JournalSqlMessagesCommit, index: number) => void
): Promise<JournalSqlBatchResult<JournalSqlMessagesCommit>> {
const outcomes: JournalSqlBatchOutcome<JournalSqlMessagesCommit>[] = []
let at = 0
while (at < jobs.length) {
const ready = jobs.slice(at, at + journalSqlMessageBatchSize(jobs, (job) => job, at))
const result = await batch.run(
ready.map((job) => ({ command: job, normal: job.normal })),
(job) => applyMessageSteps(db, job)
)
for (const [index, outcome] of result.outcomes.entries()) {
if (outcome.committed) {
// Payloads are owned by this just-committed result, never reloaded through a reader.
publish(outcome.value, at + index)
}
const result = await batch.run(jobs, (job) => applyMessageSteps(db, job))
for (const [index, outcome] of result.outcomes.entries()) {
if (outcome.committed) {
publish(outcome.value, index)
}
for (let index = 0; index < result.outcomes.length; index++) {
let outcome = result.outcomes[index]!
if (
!outcome.committed &&
outcome.error instanceof JournalMessageNeedsFull &&
!result.postcommit
) {
const job = { ...ready[index]!, normal: false }
const rerouted = await batch.run([{ command: job, normal: false }], (job) =>
applyMessageSteps(db, job)
)
outcome = rerouted.outcomes[0]!
if (outcome.committed) {
publish(outcome.value, at + index)
}
if (rerouted.postcommit) {
outcomes.push(outcome)
return { outcomes, postcommit: rerouted.postcommit }
}
}
outcomes.push(outcome)
}
if (result.postcommit) {
return { outcomes, postcommit: result.postcommit }
}
at += ready.length
}
return { outcomes }
return result
}
function applyMessageSteps(
@@ -146,7 +68,7 @@ function applyMessageSteps(
for (const step of steps) {
db.exec('SAVEPOINT journal_message_step')
try {
const commit = applyJournalSqlMessage(db, target, step, savedAt, job.normal)
const commit = applyJournalSqlMessage(db, target, step, savedAt)
db.exec('RELEASE journal_message_step')
result.cursor = commit.cursor
result.outcomes.push(commit.outcome)
@@ -160,9 +82,6 @@ function applyMessageSteps(
}
} catch (error) {
db.exec('ROLLBACK TO journal_message_step; RELEASE journal_message_step')
if (error instanceof JournalMessageNeedsFull) {
throw error
}
result.failure = { message: error instanceof Error ? error.message : String(error) }
break
}
@@ -16,17 +16,14 @@ afterEach(() => {
}
})
it('reroutes a first snapshot to FULL and publishes exact committed revisions in order', async () => {
it('publishes exact committed snapshots and their final body in order', async () => {
const { db, batch, commit, item, published } = fixture()
db.exec(`CREATE TEMP TRIGGER durable_first BEFORE INSERT ON journal_items
WHEN (SELECT synchronous FROM pragma_synchronous)<>2 BEGIN SELECT RAISE(ABORT,'creation was not FULL'); END`)
expect((await commit(message('first'))).outcomes[0]).toMatchObject({
committed: true,
value: {
cursor: { epoch: 'epoch', sequence: 11 }
}
})
db.exec('DROP TRIGGER durable_first')
const result = await commitJournalSqlMessages(
db,
batch,
@@ -42,9 +39,6 @@ it('reroutes a first snapshot to FULL and publishes exact committed revisions in
{ kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'third' }] }
])
expect(item('orca:assistant')).toMatchObject({ revision: 3, sequence: 11, observedAt: 100 })
db.exec(`CREATE TEMP TRIGGER durable_message_end BEFORE UPDATE ON journal_items
WHEN (SELECT synchronous FROM pragma_synchronous)<>2 AND json_extract(NEW.payload_json,'$.body.completedAt') IS NOT NULL
BEGIN SELECT RAISE(ABORT,'message end was not FULL'); END`)
const ended = message('third')
ended.body.completedAt = 300
expect((await commit(ended)).outcomes[0]).toMatchObject({
@@ -54,7 +48,6 @@ it('reroutes a first snapshot to FULL and publishes exact committed revisions in
items: [{ body: { completedAt: 300 } }]
}
})
expect(db.pragma('user_version', { simple: true })).toBe(4)
})
it('commits ready revisions from different chats atomically before publishing either payload', async () => {
@@ -132,8 +125,6 @@ it('accepts an optimistic echo once, preserving its local body and moved placeme
`INSERT INTO journal_card_versions(session_id,epoch,message_id,version_sequence,version_index,current,state,command_turn_item_id,payload_json)
VALUES('chat','epoch','card',1,0,1,'returned','orca:reported-command',?)`
).run(JSON.stringify(card))
db.exec(`CREATE TEMP TRIGGER durable_echo BEFORE UPDATE ON journal_submission_attempts
WHEN (SELECT synchronous FROM pragma_synchronous)<>2 BEGIN SELECT RAISE(ABORT,'acceptance was not FULL'); END`)
const input = message('original', 'unused', 'item')
input.identity = { provider: 'codex', threadId: 'thread', turnId: 'turn', ordinal: 0 }
input.body.role = 'user'
@@ -1,4 +1,3 @@
import { isDeepStrictEqual } from 'node:util'
import type Database from '../../sqlite/sync-database'
import {
agentJournalItemKey,
@@ -28,7 +27,6 @@ import {
import { journalRenderItem } from './projection/journal-render-item'
import { resolveJournalGuardedItem } from './writing/journal-timeline-resolution'
import { journalRowBase } from './writing/journal-row-builders'
import { isJournalTextRevision } from './writing/journal-streamed-text-revision'
import { assertJournalFence } from './writing/journal-write-guards'
import type { JournalItemRow } from './durable-rows/journal-row-schema'
import type { QueuedMessageRow } from './queued-messages/queued-message-table'
@@ -60,15 +58,12 @@ export type JournalSqlMessageCommit = {
editDeadline?: number | null
}
export class JournalMessageNeedsFull extends Error {}
/** Runs only on the owning writer connection, inside its command SAVEPOINT. */
export function applyJournalSqlMessage(
db: Database.Database,
target: { sessionId: string; epoch: string },
input: JournalSqlMessage,
savedAt: number,
normal: boolean
savedAt: number
): JournalSqlMessageCommit {
if (!db.isTransaction) {
throw new Error('journal message requires its writer transaction')
@@ -186,22 +181,6 @@ export function applyJournalSqlMessage(
JOIN journal_items AS item ON item.session_id=card.session_id AND item.epoch=card.epoch AND item.item_id=card.command_turn_item_id
WHERE card.session_id=? AND card.epoch=? AND card.current=1 AND card.state='returned'`)
.all(...prefix)
if (
normal &&
!(
input.kind === 'text-snapshot' &&
!echo &&
!declaration &&
returned.length === 0 &&
isJournalTextRevision(row, previous) &&
row.ownerFence === stored?.owner_fence &&
(row.executionOwner === undefined || row.executionOwner === stored?.execution_owner) &&
isDeepStrictEqual(scope, previous?.turnScope) &&
isDeepStrictEqual(agentJournalLinkageFields(row), agentJournalLinkageFields(previous))
)
) {
throw new JournalMessageNeedsFull()
}
const result: JournalSqlMessageCommit = {
cursor: { epoch, sequence },
outcome: { kind: 'committed', wrote: true },
@@ -50,9 +50,7 @@ function fixture() {
operation,
facts: { identity: { ...identity, sessionId } }
}
const [outcome] = (
await batch.run([{ command, normal: false }], (entry) => engine.apply(entry, 100))
).outcomes
const [outcome] = (await batch.run([command], (entry) => engine.apply(entry, 100))).outcomes
if (!outcome!.committed) {
throw outcome!.error
}
@@ -6,6 +6,7 @@ import {
} from '../../worker-threads/worker-thread-entry-path'
import type { JournalSqlRequest, JournalSqlResponse } from './journal-sql-protocol'
import { restoreJournalSqlFailure } from './journal-sql-failure'
import { prepareJournalSqlRequest } from './journal-sql-ready-batch'
import type { JournalWriterCommand } from './journal-writer-command'
import type { JournalWriterCommit } from './journal-writer-changes'
import type { JournalWriterEffect, JournalWriterEffectResult } from './journal-writer-effects'
@@ -130,7 +131,7 @@ export class JournalSqlOwner {
!(command.type === 'maintenance' && command.operation.kind === 'projection-state')
})
try {
this.worker.postMessage({ id, command } satisfies JournalSqlRequest)
this.worker.postMessage(prepareJournalSqlRequest(id, command))
} catch (error) {
this.pending.delete(id)
reject(error)
@@ -170,9 +171,6 @@ export class JournalSqlOwner {
return
}
pending?.resolve(response)
if ('postcommit' in response && response.postcommit) {
this.fail(restoreJournalSqlFailure(response.postcommit))
}
}
private fail(error: Error): void {
@@ -7,6 +7,7 @@ import type { JournalSqlMessageJob, JournalSqlMessagesCommit } from './journal-s
export type JournalSqlRequest = {
id: number
batch?: { sessionId: string; steps: number; bytes: number }
command:
| { type: 'write'; command: JournalWriterCommand }
| { type: 'messages'; job: JournalSqlMessageJob }
@@ -17,20 +18,18 @@ export type JournalSqlRequest = {
export type JournalSqlResponse =
| { type: 'ready'; readOnly: boolean }
| { type: 'committed'; id: number; commit: JournalWriterCommit; postcommit?: JournalSqlFailure }
| { type: 'committed'; id: number; commit: JournalWriterCommit }
| { type: 'messages'; id: number; commit: JournalSqlMessagesCommit }
| {
type: 'effects'
id: number
results: JournalWriterEffectResult[]
postcommit?: JournalSqlFailure
}
| { type: 'done'; id: number }
| {
type: 'maintenance'
id: number
result: JournalSqlMaintenanceResult
postcommit?: JournalSqlFailure
}
| {
type: 'failed'
@@ -1,53 +1,87 @@
import type { AgentJournalMessageItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
import type { JournalSqlRequest } from './journal-sql-protocol'
import type { JournalWriterCommand } from './journal-writer-command'
import { journalSqlMaintenanceIsNormal } from './journal-sql-maintenance'
const MAX_EVENTS = 64
const MAX_STEPS = 64
const MAX_BYTES = 1024 * 1024
/** Control commands keep their own boundary; an oversized atomic command runs alone. */
export function takeJournalSqlReadyBatch(
queue: JournalSqlRequest[],
streamed: (command: JournalWriterCommand) => boolean
): { request: JournalSqlRequest; normal: boolean }[] {
const batch: { request: JournalSqlRequest; normal: boolean }[] = []
let events = 0
/** Sender work only: classification never opens a saved projection. */
export function prepareJournalSqlRequest(
id: number,
command: JournalSqlRequest['command']
): JournalSqlRequest {
let batch: JournalSqlRequest['batch']
if (
command.type === 'messages' &&
command.job.command.steps.length > 0 &&
command.job.command.steps.every(
(step) => step.kind === 'text-snapshot' && streamingText(step.body)
)
) {
batch = {
sessionId: command.job.command.target.sessionId,
steps: command.job.command.steps.length,
bytes: command.job.bytes
}
} else if (command.type === 'write' && !command.command.effects) {
const { operation } = command.command
if (
operation.kind === 'steps' &&
operation.steps.length > 0 &&
operation.steps.every(
(step) =>
step.kind === 'resolved-item' &&
step.resolution.kind === 'text-snapshot' &&
streamingText(step.resolution.body)
)
) {
batch = {
sessionId: command.command.target.sessionId,
steps: operation.steps.length,
bytes: Buffer.byteLength(JSON.stringify(command.command), 'utf8')
}
}
}
return { id, command, ...(batch ? { batch } : {}) }
}
/** Controls, repeated chats and an oversized atomic command keep their own boundary. */
export function takeJournalSqlReadyBatch(queue: JournalSqlRequest[]): JournalSqlRequest[] {
const first = queue[0]
if (!first?.batch) {
return queue.splice(0, 1)
}
const sessions = new Set<string>()
let count = 0
let steps = 0
let bytes = 0
for (const request of queue) {
const { command } = request
const normal = command.type === 'write' && streamed(command.command)
if (!normal) {
if (batch.length === 0) {
batch.push({
request,
normal: command.type === 'maintenance' && journalSqlMaintenanceIsNormal(command.operation)
})
}
const next = request.batch
if (
!next ||
request.command.type !== first.command.type ||
(count > 0 &&
(sessions.has(next.sessionId) ||
steps + next.steps > MAX_STEPS ||
bytes + next.bytes > MAX_BYTES))
) {
break
}
const count =
command.command.operation.kind === 'steps'
? Math.max(1, command.command.operation.steps.length)
: 1
let size: number
try {
size = Buffer.byteLength(JSON.stringify(command.command), 'utf8')
} catch {
if (batch.length === 0) {
batch.push({ request, normal: false })
}
break
}
if (batch.length > 0 && (events + count > MAX_EVENTS || bytes + size > MAX_BYTES)) {
break
}
batch.push({ request, normal: true })
events += count
bytes += size
if (events >= MAX_EVENTS || bytes >= MAX_BYTES) {
count++
steps += next.steps
bytes += next.bytes
sessions.add(next.sessionId)
if (steps >= MAX_STEPS || bytes >= MAX_BYTES) {
break
}
}
queue.splice(0, batch.length)
return batch
return queue.splice(0, count)
}
function streamingText(body: AgentJournalMessageItem): boolean {
return (
(body.role === 'assistant' || body.role === 'reasoning') &&
body.completedAt === undefined &&
body.state === undefined &&
body.blocks.every((block) => block.type === 'text')
)
}
@@ -155,12 +155,9 @@ it.each([
await new JournalSqlBatch(persisted.db).run(
[
{
command: {
target: { sessionId: 'session', epoch: 'epoch', ownerToken: `open-${index}` },
operation: { kind: 'open' as const },
facts: { identity }
},
normal: false
target: { sessionId: 'session', epoch: 'epoch', ownerToken: `open-${index}` },
operation: { kind: 'open' as const },
facts: { identity }
}
],
(entry) => engine.apply(entry, 100)
@@ -14,7 +14,7 @@ import { assertJournalWritable } from './writing/journal-write-guards'
import { applyJournalSqlMaintenance } from './journal-sql-maintenance'
import { initializeJournalSqlProjection } from './journal-sql-schema-migration'
import { migrateJournalDomainSchema } from './journal-domain-schema'
import { commitJournalSqlMessages, journalSqlMessageBatchSize } from './journal-sql-message-commit'
import { commitJournalSqlMessages } from './journal-sql-message-commit'
if (
!parentPort ||
@@ -94,24 +94,19 @@ type MessageRequest = {
/** Bound ready work before opening a transaction, then yield after its acknowledgements. */
async function drain(): Promise<void> {
if (queue[0]?.command.type === 'messages') {
const count = journalSqlMessageBatchSize(queue, ({ command }) =>
command.type === 'messages' ? command.job : undefined
)
const pending = takeJournalSqlReadyBatch(queue)
if (pending[0]?.command.type === 'messages') {
await drainMessages(
queue
.splice(0, count)
.filter((request): request is MessageRequest => request.command.type === 'messages')
pending.filter((request): request is MessageRequest => request.command.type === 'messages')
)
scheduleNextDrain()
return
}
const pending = takeJournalSqlReadyBatch(queue, (command) => commands?.streamed(command) === true)
const writes = pending.filter(
(entry): entry is { request: WriteRequest; normal: boolean } =>
entry.request.command.type !== 'close' && entry.request.command.type !== 'messages'
(request): request is WriteRequest =>
request.command.type !== 'close' && request.command.type !== 'messages'
)
const close = pending.find(({ request }) => request.command.type === 'close')?.request
const close = pending.find((request) => request.command.type === 'close')
const database = db
if (writes.length > 0 && database && batch && commands) {
const executor = commands
@@ -122,32 +117,25 @@ async function drain(): Promise<void> {
throw fault
}
assertJournalWritable(readOnly, 'database')
const result = await batch.run<WriteRequest, CommitResponse>(
writes.map(({ request, normal }) => ({
command: request,
normal
})),
async (request) => {
const { command, id } = request
if (command.type === 'maintenance') {
return {
type: 'maintenance',
id,
result: applyJournalSqlMaintenance(database, command.operation)
}
const result = await batch.run<WriteRequest, CommitResponse>(writes, async (request) => {
const { command, id } = request
if (command.type === 'maintenance') {
return {
type: 'maintenance',
id,
result: applyJournalSqlMaintenance(database, command.operation)
}
return command.type === 'write'
? { type: 'committed', id, commit: await executor.apply(command.command, Date.now()) }
: {
type: 'effects',
id,
results: command.effects.map((effect) =>
applyJournalWriterEffect(database, stateDirectory, effect)
)
}
}
)
postcommit = result.postcommit ? journalSqlFailure(result.postcommit) : undefined
return command.type === 'write'
? { type: 'committed', id, commit: await executor.apply(command.command, Date.now()) }
: {
type: 'effects',
id,
results: command.effects.map((effect) =>
applyJournalWriterEffect(database, stateDirectory, effect)
)
}
})
try {
checkpoint?.afterCommit()
} catch (error) {
@@ -159,18 +147,18 @@ async function drain(): Promise<void> {
? outcome.value
: {
type: 'failed',
id: writes[index]!.request.id,
id: writes[index]!.id,
outcome: 'rolled-back',
error: journalSqlFailure(outcome.error)
}
)
if (postcommit) {
fault = result.postcommit ?? postcommit
fault = postcommit
}
} catch (error) {
const outcome = error instanceof JournalSqlBatchError ? error.outcome : 'not-dispatched'
fault ??= batch.fault
responses = writes.map(({ request }) => ({
responses = writes.map((request) => ({
type: 'failed',
id: request.id,
outcome,
@@ -240,7 +228,7 @@ async function drainMessages(requests: readonly MessageRequest[]): Promise<void>
acknowledged.add(id)
}
}
let postcommit = result.postcommit
let postcommit: unknown
try {
checkpoint?.afterCommit()
} catch (error) {
@@ -77,7 +77,7 @@ export class JournalWriterSession {
db,
readOnly: false,
stateDirectory,
transaction: (run, _streamed, committed) => this.savepoint(run, committed)
transaction: (run, committed) => this.savepoint(run, committed)
}
const held = queuedMessageEditLeaseHolds(facts.editLeases ?? [])
this.collaborators = createJournalStoreCollaborators({
@@ -31,10 +31,7 @@ import {
import { claimAgentSessionAttachmentsInTransaction } from '../../agent-session-attachments/agent-session-attachment-claims'
import { JournalGoalAttachmentInverse } from '../writing/journal-goal-attachment-inverse'
import { JournalEpochReplacements } from '../context-epochs/journal-epoch-replacements'
import {
isJournalOutputRevision,
isJournalStreamedTextRevision
} from '../writing/journal-streamed-text-revision'
import { isJournalOutputRevision } from '../writing/journal-output-revision'
export type JournalStoreHost = {
/** Notify readers of durable draft-only transactions that append no journal row. */
@@ -97,7 +94,6 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal
prepare: host.prepare,
commit: host.commit,
project: (rows) => projectJournalRows(host.state(), rows),
streamedTextRevision: (row) => isJournalStreamedTextRevision(host.state(), row),
outputRevision: (row) => isJournalOutputRevision(host.state(), row)
})
const inputReceipt = (rows: readonly JournalRow[], receipt?: JournalOperationReceipt) =>
@@ -2,48 +2,6 @@ import { isDeepStrictEqual } from 'node:util'
import { agentJournalLinkageFields } from '../../../../shared/agent-session/journal/agent-session-journal-producer'
import type { JournalReducerState } from '../projection/journal-reducer'
import type { JournalRow } from '../durable-rows/journal-row-schema'
import type { AgentJournalRenderItem } from '../../../../shared/agent-session/journal/agent-session-journal-types'
/** Only streamed assistant text may skip per-commit fsync; FULL turn settlement makes it durable. */
export function isJournalStreamedTextRevision(
state: JournalReducerState,
row: JournalRow
): boolean {
if (row.kind !== 'item' || row.recovered) {
return false
}
const previous = state.items.get(row.itemId)
return isJournalTextRevision(row, previous) && sameRevisionPlacement(state, row, previous!)
}
export function isJournalTextRevision(
row: Extract<JournalRow, { kind: 'item' }>,
previous: AgentJournalRenderItem | undefined
): boolean {
if (row.recovered) {
return false
}
const { body } = row
if (
body.kind !== 'message' ||
body.role !== 'assistant' ||
!body.blocks.every((block) => block.type === 'text') ||
body.sentAs !== undefined ||
body.command !== undefined ||
body.from !== undefined ||
previous?.body.kind !== 'message' ||
previous.body.role !== 'assistant' ||
previous.body.sentAs !== undefined ||
previous.body.command !== undefined ||
previous.body.from !== undefined ||
!previous.body.blocks.every((block) => block.type === 'text')
) {
return false
}
const { blocks: _oldText, ...before } = previous.body
const { blocks: _newText, ...after } = body
return isDeepStrictEqual(before, after)
}
/** Output revisions cannot change queue eligibility; lifecycle and unknown changes still wake it. */
export function isJournalOutputRevision(state: JournalReducerState, row: JournalRow): boolean {
@@ -102,8 +102,6 @@ export type JournalRowWriterDeps = {
finishFold?: (draft: JournalReducerState) => void
) => () => void
commit: (rows: readonly JournalRow[], savedAt: number, inputChanged?: boolean) => void
/** Streamed-text-only revisions may skip per-commit fsync; settlement makes them durable. */
streamedTextRevision?: (row: JournalRow) => boolean
outputRevision?: (row: JournalRow) => boolean
}
@@ -120,7 +118,6 @@ export class JournalRowWriter {
const row = build(this.deps.nextSequence(), savedAt)
assertJournalFence(row.fence, this.deps.highestFence())
const receipt = typeof prepareReceipt === 'function' ? prepareReceipt([row]) : prepareReceipt
const streamed = !receipt && this.streamed([row])
const inputChanged = Boolean(receipt) || !this.outputOnly([row])
this.deps.database().transaction(
(db) => {
@@ -131,7 +128,6 @@ export class JournalRowWriter {
receipt?.write(db, this.project([row])[0])
return adopt
},
streamed,
(adopt) => adopt?.()
)
receipt?.committed()
@@ -166,7 +162,6 @@ export class JournalRowWriter {
assertJournalFence(row.fence, this.deps.highestFence())
}
const receipt = typeof prepareReceipt === 'function' ? prepareReceipt(rows) : prepareReceipt
const streamed = !receipt && this.streamed(rows)
const inputChanged = Boolean(receipt) || !this.outputOnly(rows)
this.deps.database().transaction(
(db) => {
@@ -179,7 +174,6 @@ export class JournalRowWriter {
receipt?.write(db, receiptRow?.(this.project(rows)))
return adopt
},
streamed,
(adopt) => adopt?.()
)
receipt?.committed()
@@ -205,9 +199,4 @@ export class JournalRowWriter {
private outputOnly(rows: readonly JournalRow[]): boolean {
return this.deps.outputRevision !== undefined && rows.every(this.deps.outputRevision)
}
private streamed(rows: readonly JournalRow[]): boolean {
const { streamedTextRevision } = this.deps
return streamedTextRevision !== undefined && rows.every((row) => streamedTextRevision(row))
}
}
@@ -1,78 +0,0 @@
import { mkdtemp, rm, mkdir, writeFile } from 'node:fs/promises'
import { randomUUID } from 'node:crypto'
import { tmpdir } from 'node:os'
import { join, dirname } from 'node:path'
import { afterEach, beforeEach, vi } from 'vitest'
import type { AgentJournalMessageItem } from '../../../../shared/agent-session/journal/agent-session-journal-types'
import { journalPragmaNumber } from '../database/journal-database'
import {
closeTestJournalHostDatabases,
createTrackedJournalOpener,
openTestJournalHostDatabase
} from '../store-lifecycle/journal-host-database-test-support'
import { formatNativeChatFileReference } from '../../../../shared/agent-image-paste'
import * as attachmentReferences from '../../agent-session-attachments/agent-session-attachment-references'
export const journals = createTrackedJournalOpener({ legacyInputs: true })
let root: string
beforeEach(async () => {
root = await mkdtemp(join(tmpdir(), 'orca-streamed-revision-'))
})
afterEach(async () => {
vi.restoreAllMocks()
await journals.closeAll()
await closeTestJournalHostDatabases()
await rm(root, { recursive: true, force: true })
})
export const text = (role: 'assistant' | 'user', value: string): AgentJournalMessageItem => ({
kind: 'message',
role,
blocks: [{ type: 'text', text: value }]
})
export function openJournal(sessionId = 'stream') {
return journals.open({
stateDirectory: root,
identity: {
sessionId,
workspaceId: 'folder',
hostId: 'local',
agent: 'grok',
providerHandle: null
}
})
}
export function commitLevels() {
const { db } = openTestJournalHostDatabase(root)
const levels: number[] = []
const exec = db.exec.bind(db)
vi.spyOn(db, 'exec').mockImplementation((sql) => {
if (sql === 'COMMIT') {
levels.push(journalPragmaNumber(db, 'synchronous'))
}
exec(sql)
})
return levels
}
export async function uploadedInput() {
const id = randomUUID()
const path = join(attachmentReferences.agentSessionAttachmentStoreRoot(root), id, 'source.txt')
await mkdir(dirname(path), { recursive: true })
await writeFile(path, 'retained input')
return { id, body: text('user', formatNativeChatFileReference(path)) }
}
export const fixture = {
get root() {
return root
},
set root(value: typeof root) {
root = value
}
}