perf(native-chat): bound fork scratch presentation work

This commit is contained in:
Brennan Benson
2026-11-11 08:52:55 -08:00
parent 6409e04b23
commit b76334cd54
9 changed files with 753 additions and 395 deletions
@@ -80,18 +80,22 @@ export function journalDomainDisplayTurnState(
export function readJournalDomainDisplayedRow(
db: Database.Database,
target: JournalDomainItemTarget,
row: SqliteRow
row: SqliteRow,
stopStates?: ReadonlyMap<string, unknown>
): { item: AgentJournalRenderItem; payload: string } {
const source: AgentJournalRenderItem = JSON.parse(String(row.payload_json))
const states =
row.wire_version === AGENT_SESSION_ITEM_WIRE_VERSION
? undefined
: readJournalDomainStopStates(db, target, [String(row.item_id)])
: (stopStates ?? readJournalDomainStopStates(db, target, [String(row.item_id)]))
const item = resolveJournalDomainDisplay(
source,
row.wire_version,
row.wire_json,
states && journalDomainDisplayTurnState(source, states)
)
return { item, payload: item === source ? String(row.payload_json) : JSON.stringify(item) }
return {
item,
payload: item === source ? String(row.payload_json) : JSON.stringify(item)
}
}
@@ -50,3 +50,42 @@ export function assertJournalMetadataPlan(db: Database.Database, sql: string): v
}
}
}
export function captureJournalForkPresentationPlans(db: Database.Database): () => void {
const statements = vi.spyOn(db, 'prepare')
return () => {
const queries = [...new Set(statements.mock.calls.map(([sql]) => sql))]
statements.mockRestore()
const root = db
.prepare("SELECT rootpage FROM temp.sqlite_master WHERE name='fork_bodies'")
.get()!.rootpage
const payload = db
.prepare('PRAGMA temp.table_info(fork_bodies)')
.all()
.find((row) => row.name === 'payload')!.cid
expect(queries.some((sql) => sql.includes('body.lifecycle'))).toBe(true)
for (const sql of queries) {
const code = db.prepare(`EXPLAIN ${sql}`).all()
expect(
code.filter((row) => ['OpenRead', 'OpenWrite'].includes(String(row.opcode)) && row.p3 === 0)
).toEqual([])
expect(
db
.prepare(`EXPLAIN QUERY PLAN ${sql}`)
.all()
.map((row) => String(row.detail))
.join('\n')
).not.toContain('TEMP B-TREE')
if (sql.includes('body.lifecycle')) {
const cursors = new Set(
code
.filter((row) => row.opcode === 'OpenRead' && row.p2 === root && row.p3 === 1)
.map((row) => row.p1)
)
for (const column of code.filter((row) => row.opcode === 'Column' && cursors.has(row.p1))) {
expect(Number(column.p2)).toBeLessThan(Number(payload))
}
}
}
}
}
@@ -0,0 +1,102 @@
import type Database from '../../sqlite/sync-database'
import {
agentJournalItemKey,
parseAgentJournalItemKey
} from '../../../shared/agent-session/journal/agent-session-journal-item-key'
import { readAgentJournalTurn } from '../../../shared/agent-session/lifecycle/agent-session-turn-record'
import {
forkDependencies,
omitForkPresentation
} from '../agent-session-wire/acquisition/structured-agent-session-fork-snapshot'
import {
readJournalDomainDisplayedRow,
readJournalDomainStopStates,
type JournalDomainItemTarget
} from './journal-domain-item-display'
import type { JournalForkPreparationInput } from './journal-fork-preparation'
import { journalForkRowBytes } from './journal-fork-workspace'
/** One dependency item and scalar admission metadata survive a bounded capture step. */
export class JournalForkCapture {
private after = 0
private edges?: Generator<readonly [string, string]>
constructor(
private readonly db: Database.Database,
private readonly target: JournalDomainItemTarget,
private readonly input: JournalForkPreparationInput,
private readonly end: number
) {}
next(budget: () => boolean, remaining: () => number, account: (bytes: number) => void): boolean {
const drainEdges = () => {
while (this.edges && budget()) {
const edge = this.edges.next()
if (edge.done) {
this.edges = undefined
} else {
this.db.prepare('INSERT OR IGNORE INTO temp.fork_edges VALUES(?,?)').run(...edge.value)
account(journalForkRowBytes(edge.value[0]) + journalForkRowBytes(edge.value[1]))
}
}
}
drainEdges()
if (!budget()) {
return false
}
const prefix = [this.target.sessionId, this.target.epoch] as const
const selection = `FROM temp.fork_headers header
WHERE header.rank>? AND (header.rank<? OR EXISTS(SELECT 1 FROM temp.fork_kept WHERE member=header.member))
ORDER BY header.rank LIMIT ?`
const ids = this.db
.prepare(`SELECT header.item_id ${selection}`)
.all(this.after, this.end, remaining())
.map((row) => String(row.item_id))
const states = readJournalDomainStopStates(this.db, this.target, ids)
let exhausted = true
for (const row of this.db
.prepare(`SELECT header.rank,header.item_id,header.provider_key,
item.wire_version,item.wire_json,item.payload_json
FROM temp.fork_headers header LEFT JOIN journal_items item
ON item.session_id=?3 AND item.epoch=?4 AND item.item_id=header.item_id
WHERE header.rank>?1 AND (header.rank<?2 OR EXISTS(SELECT 1 FROM temp.fork_kept WHERE member=header.member))
ORDER BY header.rank LIMIT ?5`)
.iterate(this.after, this.end, ...prefix, remaining())) {
if (typeof row.payload_json !== 'string') {
throw new Error('agent_session_fork:missing-source')
}
const { item, payload } = readJournalDomainDisplayedRow(this.db, this.target, row, states)
const identity = parseAgentJournalItemKey(String(row.provider_key))
if (!identity || item.itemId !== row.item_id) {
throw new Error('agent_session_fork:missing-identity')
}
const mapped = agentJournalItemKey(
identity.provider === 'codex' &&
identity.threadId === this.input.sourceThreadId &&
this.input.childThreadId
? { ...identity, threadId: this.input.childThreadId }
: identity
)
const bodyBytes = Buffer.byteLength(JSON.stringify(item.body))
const lifecycle = Number(readAgentJournalTurn(item.body) !== null)
const turnScope = item.turnScope?.kind === 'turn' ? item.turnScope.turnItemId : null
this.db
.prepare(
`INSERT INTO temp.fork_bodies(rank,item_id,payload,mapped,body_bytes,lifecycle,turn_scope_id) VALUES(?,?,?,?,?,?,?)`
)
.run(row.rank, item.itemId, payload, mapped, bodyBytes, lifecycle, turnScope)
this.edges = forkDependencies(item)
if (omitForkPresentation(item.body)) {
this.db.prepare('INSERT OR IGNORE INTO temp.fork_omitted(id) VALUES(?)').run(item.itemId)
}
this.after = Number(row.rank)
account(journalForkRowBytes(row))
drainEdges()
if (!budget()) {
exhausted = false
break
}
}
return exhausted
}
}
@@ -1,19 +1,16 @@
import { readJournalDomainDisplayedRow } from './journal-domain-item-display'
import { createJournalForkWorkspace, journalForkHeaderMember } from './journal-fork-workspace'
import { JournalForkCapture } from './journal-fork-capture'
import {
createJournalForkWorkspace,
journalForkHeaderMember,
journalForkRowBytes
} from './journal-fork-workspace'
import { JournalForkPresentation } from './journal-fork-presentation'
import type Database from '../../sqlite/sync-database'
import type { SqliteRow } from '../../sqlite/sqlite-statement'
import {
agentJournalItemKey,
parseAgentJournalItemKey
} from '../../../shared/agent-session/journal/agent-session-journal-item-key'
import { agentJournalItemKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key'
import { savedRewindKeys } from '../../../shared/agent-session/lifecycle/agent-session-rewind'
import type { AgentJournalSubmission } from '../../../shared/agent-session/journal/agent-session-journal-types'
import type { JournalReplacementItem } from './context-epochs/journal-epoch-replacement'
import {
omitForkPresentation,
forkDependencies
} from '../agent-session-wire/acquisition/structured-agent-session-fork-snapshot'
import { validateJournalForkSource, type JournalForkSourceProof } from './journal-fork-source-proof'
export type JournalForkPreparationInput = {
@@ -31,7 +28,8 @@ const MAX_BYTES = 1024 * 1024
/** One reserved connection, with disk-backed preparation surviving only this finite copy. */
export class JournalForkPreparation {
private phase: 'submissions' | 'headers' | 'members' | 'kept' | 'capture' = 'submissions'
private phase: 'submissions' | 'headers' | 'members' | 'kept' | 'capture' | 'closed' =
'submissions'
private presentation?: JournalForkPresentation
private sourceOpen = false
private epoch = ''
@@ -40,7 +38,7 @@ export class JournalForkPreparation {
private position = [0, 0, 0]
private after = -1
private end = Number.MAX_SAFE_INTEGER
private edges?: Generator<readonly [string, string]>
private captureState?: JournalForkCapture
constructor(
private readonly db: Database.Database,
@@ -49,8 +47,8 @@ export class JournalForkPreparation {
if (db.isTransaction) {
throw new Error('fork preparation requires an exclusively reserved reader')
}
createJournalForkWorkspace(db)
try {
createJournalForkWorkspace(db)
db.exec('BEGIN')
this.sourceOpen = true
const validated = validateJournalForkSource(db, input.source)
@@ -63,7 +61,7 @@ export class JournalForkPreparation {
.get(input.source.sessionId, this.epoch)!.floor
)
} catch (error) {
this.close()
this.abort()
throw error
}
}
@@ -73,18 +71,37 @@ export class JournalForkPreparation {
}
next(): JournalForkPreparationResult {
if (this.presentation) {
return this.presentation.next()
if (this.phase === 'closed') {
throw new Error('fork preparation is closed')
}
try {
if (this.presentation) {
if (this.sourceOpen || this.db.isTransaction) {
throw new Error('fork presentation requires a released source snapshot')
}
this.db.exec('SAVEPOINT fork_presentation')
const result = this.presentation.next()
this.db.exec('RELEASE fork_presentation')
return result
}
return this.capture()
} catch (error) {
this.abort()
throw error
}
}
private capture(): JournalForkPreparationResult {
const start = performance.now()
let units = 0
let bytes = 0
const budget = () =>
units < MAX_ROWS && bytes < MAX_BYTES && (units === 0 || performance.now() - start < 8)
const account = (row: SqliteRow | string) => {
const accountBytes = (size: number) => {
units++
bytes += Buffer.byteLength(typeof row === 'string' ? row : JSON.stringify(row))
bytes += size
}
const account = (row: SqliteRow | string) => accountBytes(journalForkRowBytes(row))
const prefix = [this.input.source.sessionId, this.epoch] as const
while (budget()) {
switch (this.phase) {
@@ -222,75 +239,55 @@ export class JournalForkPreparation {
break
}
case 'capture': {
if (this.edges) {
const edge = this.edges.next()
if (edge.done) {
this.edges = undefined
} else {
this.db
.prepare('INSERT OR IGNORE INTO temp.fork_edges VALUES(?,?)')
.run(...edge.value)
account(JSON.stringify(edge.value))
}
continue
}
const row = this.db
.prepare(`SELECT header.rank,header.item_id,header.provider_key,item.wire_version,item.wire_json,item.payload_json
FROM temp.fork_headers header CROSS JOIN journal_items item
WHERE header.rank>? AND (header.rank<? OR EXISTS(SELECT 1 FROM temp.fork_kept WHERE member=header.member))
AND item.session_id=? AND item.epoch=? AND item.item_id=header.item_id
ORDER BY header.rank LIMIT 1`)
.get(this.after, this.end, ...prefix)
if (!row) {
this.captureState ??= new JournalForkCapture(
this.db,
{ sessionId: prefix[0], epoch: prefix[1] },
this.input,
this.end
)
if (this.captureState.next(budget, () => MAX_ROWS - units, accountBytes)) {
// No domain writer acknowledgement is needed to release this WAL snapshot.
this.db.exec('COMMIT')
this.sourceOpen = false
this.captureState = undefined
this.presentation = new JournalForkPresentation(this.db, this.input)
this.after = 0
return { kind: 'progress', sourceReleased: true }
}
const { item, payload } = readJournalDomainDisplayedRow(
this.db,
{ sessionId: prefix[0], epoch: prefix[1] },
row
)
const identity = parseAgentJournalItemKey(String(row.provider_key))
if (!identity) {
throw new Error('agent_session_fork:missing-identity')
}
const mapped = agentJournalItemKey(
identity.provider === 'codex' &&
identity.threadId === this.input.sourceThreadId &&
this.input.childThreadId
? { ...identity, threadId: this.input.childThreadId }
: identity
)
this.db
.prepare('INSERT INTO temp.fork_bodies(rank,item_id,payload,mapped) VALUES(?,?,?,?)')
.run(row.rank, item.itemId, payload, mapped)
this.edges = forkDependencies(item)
if (omitForkPresentation(item.body)) {
this.omit(item.itemId)
}
this.after = Number(row.rank)
account(row)
break
}
case 'closed':
throw new Error('fork preparation is closed')
}
}
return { kind: 'progress', sourceReleased: !this.sourceOpen }
}
close(): void {
if (this.sourceOpen && this.db.isTransaction) {
this.db.exec('ROLLBACK')
try {
if (this.db.isTransaction) {
this.db.exec('ROLLBACK')
}
this.db.exec(`DROP TABLE IF EXISTS temp.fork_headers; DROP TABLE IF EXISTS temp.fork_aliases;
DROP TABLE IF EXISTS temp.fork_kept; DROP TABLE IF EXISTS temp.fork_bodies; DROP TABLE IF EXISTS temp.fork_edges;
DROP TABLE IF EXISTS temp.fork_omitted; DROP TABLE IF EXISTS temp.fork_work`)
} catch (error) {
try {
this.db.close()
} catch {}
throw error
} finally {
this.sourceOpen = false
this.captureState = undefined
this.presentation = undefined
this.phase = 'closed'
}
this.sourceOpen = false
this.edges = undefined
this.db.exec(`DROP TABLE IF EXISTS temp.fork_headers; DROP TABLE IF EXISTS temp.fork_aliases;
DROP TABLE IF EXISTS temp.fork_kept; DROP TABLE IF EXISTS temp.fork_bodies; DROP TABLE IF EXISTS temp.fork_edges;
DROP TABLE IF EXISTS temp.fork_omitted; DROP TABLE IF EXISTS temp.fork_work`)
this.presentation = undefined
}
private abort(): void {
try {
this.close()
} catch {}
}
private omit(id: string): void {
@@ -1,9 +1,9 @@
import { journalForkRowBytes } from './journal-fork-workspace'
import type Database from '../../sqlite/sync-database'
import type { SqliteRow } from '../../sqlite/sqlite-statement'
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
import { agentJournalLinkageFields } from '../../../shared/agent-session/journal/agent-session-journal-producer'
import { parseAgentJournalItemKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key'
import { readAgentJournalTurn } from '../../../shared/agent-session/lifecycle/agent-session-turn-record'
import { mapBodyLinks } from '../agent-session-wire/acquisition/structured-agent-session-fork-snapshot'
import { continuedOutsideRecordId } from '../agent-session-wire/acquisition/structured-agent-session-history-catch-up'
import {
@@ -41,7 +41,7 @@ export class JournalForkPresentation {
units < 64 && bytes < MAX_BYTES && (units === 0 || performance.now() - start < 8)
const account = (row: SqliteRow | string) => {
units++
bytes += Buffer.byteLength(typeof row === 'string' ? row : JSON.stringify(row))
bytes += journalForkRowBytes(row)
}
while (budget()) {
switch (this.phase) {
@@ -86,55 +86,72 @@ export class JournalForkPresentation {
break
}
case 'notices': {
const row = this.retainedRow(this.after)
if (!row) {
this.phase = 'output'
this.after = 0
continue
}
const item: AgentJournalRenderItem = JSON.parse(String(row.payload))
const key = this.db
.prepare('SELECT provider_key,member FROM temp.fork_headers WHERE rank=?')
.get(row.rank)!
const identity = parseAgentJournalItemKey(String(key.provider_key))!
const scoped = (...parts: string[]) => JSON.stringify(parts)
const noticePrefix = continuedOutsideRecordId('')
if (identity.provider === 'legacy' && identity.recordId.startsWith(noticePrefix)) {
const anchor = identity.recordId.slice(noticePrefix.length)
const notice =
identity.agent === 'codex'
? scoped('codex', anchor)
: identity.agent === 'claude'
? scoped('row', anchor)
: scoped(identity.agent, identity.sessionId, providerTimelineKeyPart(anchor))
this.db
.prepare('UPDATE temp.fork_bodies SET notice=? WHERE rank=?')
.run(notice, row.rank)
} else if (!readAgentJournalTurn(item.body)) {
const work = (anchor: string) =>
const anchors = new Map<string, number>()
let exhausted = true
for (const row of this.db
.prepare(`SELECT body.rank,body.item_id,body.lifecycle,body.turn_scope_id,
header.provider_key,header.member,root.provider_key AS root_provider_key,root.turn_id AS root_turn_id
FROM temp.fork_bodies body JOIN temp.fork_headers header ON header.rank=body.rank
LEFT JOIN temp.fork_headers root ON root.item_id=body.turn_scope_id
WHERE body.rank>? AND NOT EXISTS(SELECT 1 FROM temp.fork_omitted WHERE id=body.item_id)
ORDER BY body.rank LIMIT ?`)
.iterate(this.after, 64 - units)) {
const identity = parseAgentJournalItemKey(String(row.provider_key))!
const scoped = (...parts: string[]) => JSON.stringify(parts)
const noticePrefix = continuedOutsideRecordId('')
if (identity.provider === 'legacy' && identity.recordId.startsWith(noticePrefix)) {
const anchor = identity.recordId.slice(noticePrefix.length)
const notice =
identity.agent === 'codex'
? scoped('codex', anchor)
: identity.agent === 'claude'
? scoped('row', anchor)
: scoped(identity.agent, identity.sessionId, providerTimelineKeyPart(anchor))
this.db
.prepare(
'INSERT INTO temp.fork_work VALUES(?,?) ON CONFLICT(anchor) DO UPDATE SET last_rank=excluded.last_rank'
)
.run(anchor, row.rank)
work(scoped('row', item.itemId))
work(scoped('row', String(key.provider_key)))
if (this.input.sourceThreadId && key.member !== null) {
work(scoped('codex', String(key.member)))
} else if (item.turnScope?.kind === 'turn') {
const root = this.db
.prepare('SELECT provider_key,turn_id FROM temp.fork_headers WHERE item_id=?')
.get(item.turnScope.turnItemId)
const rootIdentity = root && parseAgentJournalItemKey(String(root.provider_key))
const keyPart =
root?.turn_id == null ? null : providerTimelineTurnKeyPart(String(root.turn_id))
if (rootIdentity?.provider === 'legacy' && keyPart !== null) {
work(scoped(rootIdentity.agent, rootIdentity.sessionId, keyPart))
.prepare('UPDATE temp.fork_bodies SET notice=? WHERE rank=?')
.run(notice, row.rank)
bytes += Buffer.byteLength(notice)
} else if (!row.lifecycle) {
const work = (anchor: string) => {
anchors.set(anchor, Number(row.rank))
bytes += Buffer.byteLength(anchor)
}
work(scoped('row', String(row.item_id)))
work(scoped('row', String(row.provider_key)))
if (this.input.sourceThreadId && row.member !== null) {
work(scoped('codex', String(row.member)))
} else if (row.turn_scope_id !== null) {
const rootIdentity =
row.root_provider_key === null
? null
: parseAgentJournalItemKey(String(row.root_provider_key))
const keyPart =
row.root_turn_id === null
? null
: providerTimelineTurnKeyPart(String(row.root_turn_id))
if (rootIdentity?.provider === 'legacy' && keyPart !== null) {
work(scoped(rootIdentity.agent, rootIdentity.sessionId, keyPart))
}
}
}
this.after = Number(row.rank)
account(row)
if (!budget()) {
exhausted = false
break
}
}
if (anchors.size) {
this.db
.prepare(`INSERT INTO temp.fork_work(anchor,last_rank)
SELECT json_extract(value,'$[0]'),json_extract(value,'$[1]') FROM json_each(?) WHERE 1
ON CONFLICT(anchor) DO UPDATE SET last_rank=MAX(fork_work.last_rank,excluded.last_rank)`)
.run(JSON.stringify([...anchors]))
}
if (exhausted) {
this.phase = 'output'
this.after = 0
}
this.after = Number(row.rank)
account(row)
break
}
case 'output': {
@@ -148,15 +165,21 @@ export class JournalForkPresentation {
break
}
const item: AgentJournalRenderItem = JSON.parse(String(row.payload))
const rewrite = (id: string) =>
String(
let rewritten = false
const rewrite = (id: string) => {
const mapped = String(
this.db.prepare('SELECT mapped FROM temp.fork_bodies WHERE item_id=?').get(id)
?.mapped ?? id
)
rewritten ||= mapped !== id
return mapped
}
const body = mapBodyLinks(item.body, rewrite)
const bodyUnchanged = !rewritten
const identity = parseAgentJournalItemKey(String(row.mapped))!
const output: JournalReplacementItem = {
identity,
body: mapBodyLinks(item.body, rewrite),
body,
observedAt: item.observedAt,
ownerFence: 0,
...agentJournalLinkageFields(item),
@@ -173,7 +196,11 @@ export class JournalForkPresentation {
rank: Number(row.rank),
mapped: String(row.mapped),
item: output,
bytes: Buffer.byteLength(JSON.stringify(output))
bytes: bodyUnchanged
? Number(row.body_bytes) +
Buffer.byteLength(JSON.stringify({ ...output, body: null })) -
4
: Buffer.byteLength(JSON.stringify(output))
}
}
const output = this.pendingOutput
@@ -204,7 +231,7 @@ export class JournalForkPresentation {
}
private retainedRow(after: number, notices = false): SqliteRow | undefined {
return this.db
.prepare(`SELECT body.rank,body.item_id,body.payload,body.mapped FROM temp.fork_bodies body
.prepare(`SELECT body.rank,body.item_id,body.payload,body.mapped,body.body_bytes FROM temp.fork_bodies body
WHERE rank>? AND NOT EXISTS(SELECT 1 FROM temp.fork_omitted WHERE id=body.item_id)
${notices ? 'AND (notice IS NULL OR EXISTS(SELECT 1 FROM temp.fork_work WHERE anchor=body.notice AND last_rank>body.rank))' : ''}
ORDER BY rank LIMIT 1`)
@@ -21,13 +21,25 @@ export function createJournalForkWorkspace(db: Database.Database): void {
CREATE INDEX temp.fork_headers_member ON fork_headers(member,rank);
CREATE TEMP TABLE fork_aliases(item_id TEXT PRIMARY KEY,provider_key TEXT NOT NULL);
CREATE TEMP TABLE fork_kept(member TEXT PRIMARY KEY);
CREATE TEMP TABLE fork_bodies(rank INTEGER PRIMARY KEY,item_id TEXT UNIQUE,payload TEXT NOT NULL,mapped TEXT NOT NULL,notice TEXT);
CREATE TEMP TABLE fork_bodies(rank INTEGER PRIMARY KEY,item_id TEXT UNIQUE,mapped TEXT NOT NULL,notice TEXT,
body_bytes INTEGER NOT NULL,lifecycle INTEGER NOT NULL,turn_scope_id TEXT,payload TEXT NOT NULL);
CREATE TEMP TABLE fork_edges(parent TEXT NOT NULL,child TEXT NOT NULL,PRIMARY KEY(parent,child)) WITHOUT ROWID;
CREATE TEMP TABLE fork_omitted(id TEXT UNIQUE,processed INTEGER NOT NULL DEFAULT 0);
CREATE INDEX temp.fork_omitted_pending ON fork_omitted(processed);
CREATE TEMP TABLE fork_work(anchor TEXT PRIMARY KEY,last_rank INTEGER NOT NULL);`)
}
/** UTF-8 bound strings and numeric scalars, without serializing rows containing source JSON. */
export function journalForkRowBytes(row: SqliteRow | string): number {
return typeof row === 'string'
? Buffer.byteLength(row)
: Object.values(row).reduce<number>(
(bytes, value) =>
bytes + (typeof value === 'string' ? Buffer.byteLength(value) : value === null ? 0 : 8),
0
)
}
export function journalForkHeaderMember(row: SqliteRow, sourceThreadId?: string): string | null {
const identity = parseAgentJournalItemKey(String(row.provider_key))
let member: string | null = null
@@ -2,7 +2,7 @@ import type { JournalReplacementItem } from './context-epochs/journal-epoch-repl
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { expect, it } from 'vitest'
import { expect, it, vi } from 'vitest'
import Database from '../../sqlite/sync-database'
import { agentSessionRecordFixture } from '../agent-session-record-test-fixture'
import { encodeAgentSessionRecord } from '../../../shared/agent-session/lifecycle/agent-session-record-stored-form'
@@ -21,239 +21,177 @@ import { JournalSqlScope } from './journal-sql-scope'
import { JournalSqlBatch } from './journal-sql-batch'
import { seedJournalForkSource } from './journal-fork-source.test-fixture'
import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key'
import { captureJournalMetadataPlans } from './journal-domain-metadata-plan.test-fixture'
import {
captureJournalMetadataPlans,
captureJournalForkPresentationPlans
} from './journal-domain-metadata-plan.test-fixture'
it('releases a WAL-pressured capture before staging and leaves no visible fork after a pre-install crash', async () => {
const directory = await mkdtemp(join(tmpdir(), 'orca-sql-fork-'))
const filename = join(directory, 'copy.sqlite')
const writer = new Database(filename)
let writerClosed = false
writer.pragma('journal_mode=WAL')
writer.pragma('wal_autocheckpoint=0')
messageFixture(writer)
let reader: Database.Database | undefined
let preparation: JournalForkPreparation | undefined
try {
const record = agentSessionRecordFixture({
sessionId: 'source-chat',
provider: 'codex',
providerHandleChain: [
it.each(['current', 'stale', 'mixed', 'temp-write', 'release', 'cleanup'])(
'releases a WAL-pressured capture and keeps an interrupted private fork invisible (%s)',
async (failure) => {
const directory = await mkdtemp(join(tmpdir(), 'orca-sql-fork-'))
const filename = join(directory, 'copy.sqlite')
const writer = new Database(filename)
let writerClosed = false
writer.pragma('journal_mode=WAL')
writer.pragma('wal_autocheckpoint=0')
messageFixture(writer)
let reader: Database.Database | undefined
let preparation: JournalForkPreparation | undefined
try {
const record = agentSessionRecordFixture({
sessionId: 'source-chat',
provider: 'codex',
providerHandleChain: [
{
linkId: 'head',
origin: 'created',
mintedAtFence: 1,
observedAt: 1,
handle: codexProviderHandle('source')
}
]
})
writer
.prepare('INSERT INTO agent_session_records(session_id,record_json) VALUES(?,?)')
.run(record.sessionId, JSON.stringify(encodeAgentSessionRecord(record)))
const items: AgentJournalRenderItem[] = [
{
linkId: 'head',
origin: 'created',
mintedAtFence: 1,
itemId: agentJournalSubmissionKey('first-send'),
revision: 1,
sequence: 1,
observedAt: 1,
handle: codexProviderHandle('source')
body: {
kind: 'message',
role: 'user',
blocks: [{ type: 'text', text: 'Keep the frozen answer' }]
}
},
{
itemId: 'legacy:codex:source:root',
revision: 1,
sequence: 2,
observedAt: 2,
body: {
kind: 'turn',
turnId: 'T1',
userItemId: agentJournalSubmissionKey('first-send'),
state: 'interrupted'
}
}
]
})
writer
.prepare('INSERT INTO agent_session_records(session_id,record_json) VALUES(?,?)')
.run(record.sessionId, JSON.stringify(encodeAgentSessionRecord(record)))
const items: AgentJournalRenderItem[] = [
{
itemId: agentJournalSubmissionKey('first-send'),
revision: 1,
sequence: 1,
observedAt: 1,
body: {
kind: 'message',
role: 'user',
blocks: [{ type: 'text', text: 'Keep the frozen answer' }]
const frozenText = '雪🙂 codex:source:T1:140 "quoted" \\ end\n'.repeat(32768)
const stopId = agentJournalSubmissionKey('stop:fork')
for (let ordinal = 1; ordinal <= 140; ordinal++) {
items.push({
itemId: ordinal === 3 ? stopId : `codex:source:T1:${ordinal}`,
revision: 1,
sequence: ordinal + 2,
observedAt: ordinal + 2,
turnScope: { kind: 'turn', turnItemId: 'legacy:codex:source:root' },
body:
ordinal === 3
? JSON.parse(
'{"kind":"status","text":"Saved cancellation failure.","failure":{"kind":"cancelUnconfirmed"}}'
)
: {
kind: 'message',
role: 'assistant',
...(ordinal === 69 ? { state: 'running' as const } : {}),
blocks: [
{
type: 'text',
text: ordinal === 1 ? frozenText : `frozen-${ordinal} ${'x'.repeat(8192)}`,
...([69, 130, 131].includes(ordinal)
? {
usageLimitTurnItemId: `codex:source:T1:${ordinal === 130 ? 69 : 130}`
}
: {})
},
...(ordinal === 2
? [
{
type: 'tool-result' as const,
output: '雪🙂 codex:source:T1:1',
fullOutputItemId: 'codex:source:T1:1'
},
{
type: 'tool-result' as const,
output: 'Missing reference stays frozen',
fullOutputItemId: 'codex:source:missing:1'
}
]
: [])
]
}
})
}
for (const [ordinal, id] of [
[4, 'legacy:codex:source:continued-outside%3AT1'],
[5, 'legacy:claude:source:continued-outside%3Acodex%3Asource%3AT1%3A130']
] as const) {
const item = items[ordinal + 1]!
item.itemId = id
item.body = { kind: 'status', text: 'Continued elsewhere' }
}
const submissions: AgentJournalSubmission[] = [
{
clientMessageId: 'first-send',
fence: 1,
submittedAt: 1,
submittedSequence: 1,
payloadFingerprint: 'accepted-first-send',
dispatchState: 'accepted',
providerItemId: 'codex:source:T1:0',
acceptedSequence: 1,
reason: null,
resolvedAt: 1
}
},
{
itemId: 'legacy:codex:source:root',
revision: 1,
sequence: 2,
observedAt: 2,
body: {
kind: 'turn',
turnId: 'T1',
userItemId: agentJournalSubmissionKey('first-send'),
state: 'completed'
}
}
]
for (let ordinal = 1; ordinal <= 140; ordinal++) {
items.push({
itemId: `codex:source:T1:${ordinal}`,
revision: 1,
sequence: ordinal + 2,
observedAt: ordinal + 2,
turnScope: { kind: 'turn', turnItemId: 'legacy:codex:source:root' },
body: {
kind: 'message',
role: 'assistant',
...(ordinal === 69 ? { state: 'running' as const } : {}),
blocks: [
{
type: 'text',
text: `frozen-${ordinal} ${'x'.repeat(8192)}`,
...([69, 130, 131].includes(ordinal)
? {
usageLimitTurnItemId: `codex:source:T1:${ordinal === 130 ? 69 : 130}`
}
: {})
}
]
}
})
}
const submissions: AgentJournalSubmission[] = [
{
clientMessageId: 'first-send',
fence: 1,
submittedAt: 1,
submittedSequence: 1,
payloadFingerprint: 'accepted-first-send',
dispatchState: 'accepted',
providerItemId: 'codex:source:T1:0',
acceptedSequence: 1,
reason: null,
resolvedAt: 1
}
]
seedJournalForkSource(
writer,
{
sessionId: 'source-chat',
cursor: { epoch: 'epoch', sequence: 142 },
items,
submissions
},
record
)
writer.pragma('wal_checkpoint(TRUNCATE)')
reader = new Database(filename)
const source = readJournalDomainForkBoundary(
new JournalSqlScope(writer, 'source-chat', 'epoch'),
'codex:source:T1:140',
undefined,
false
)
const boundary = structuredAgentSessionForkBoundary(
source.snapshot,
record,
'codex:source:T1:140'
)!
expect(boundary).not.toBeNull()
const assertPlans = captureJournalMetadataPlans(
reader,
'SELECT item_id,sequence,sequence_index,ordinal,turn_item_id,'
)
preparation = new JournalForkPreparation(reader, {
source: {
sessionId: 'source-chat',
throughItemId: 'codex:source:T1:140',
capability: 'checkpoint',
retained: boundary.retained,
location: record.location,
submissionsToken: null
},
sourceThreadId: 'source',
childThreadId: 'child'
})
expect(preparation.next()).toEqual({ kind: 'progress', sourceReleased: false })
writer
.prepare(
"UPDATE journal_items SET payload_json=json_set(payload_json,'$.body.blocks[0].text','new parent body') WHERE session_id='source-chat' AND item_id='codex:source:T1:140'"
)
.run()
writer
.prepare(
"DELETE FROM journal_items WHERE session_id='source-chat' AND item_id='codex:source:T1:139'"
)
.run()
const blocked = writer.prepare('PRAGMA wal_checkpoint(PASSIVE)').get()!
expect(Number(blocked.checkpointed)).toBeLessThan(Number(blocked.log))
for (;;) {
const step = preparation.next()
expect(step.kind).toBe('progress')
if (step.kind === 'progress' && step.sourceReleased) {
break
}
}
expect(reader.isTransaction).toBe(false)
assertPlans()
const released = writer.prepare('PRAGMA wal_checkpoint(PASSIVE)').get()!
expect(Number(released.checkpointed)).toBe(Number(released.log))
const output: JournalReplacementItem[] = []
for (;;) {
const step = preparation.next()
if (step.kind === 'complete') {
break
}
if (step.kind === 'page') {
expect(step.items.length).toBeLessThanOrEqual(64)
expect(step.bytes).toBeLessThanOrEqual(1024 * 1024)
output.push(...step.items)
}
}
expect(
output.filter(
({ identity }) => 'ordinal' in identity && [69, 130, 131].includes(identity.ordinal)
)
).toEqual([])
expect(output).toHaveLength(139)
expect(output[0]!.identity).toEqual({
provider: 'codex',
threadId: 'child',
turnId: 'T1',
ordinal: 0
})
expect(output).toEqual(
prepareForkSnapshot({
snapshot: {
]
seedJournalForkSource(
writer,
{
sessionId: 'source-chat',
cursor: { epoch: 'epoch', sequence: 142 },
submissions,
items
items,
submissions
},
nextItemId: null,
sourceThreadId: 'source',
childThreadId: 'child'
})
)
writer
.prepare(
"INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) VALUES('workspace','child','empty',0,1)"
record
)
.run()
const stages = new JournalForkStages(writer, directory)
const batch = new JournalSqlBatch(writer, {
beforeWrite: async () => {
if (reader!.isTransaction) {
throw new Error('checkpoint admission is waiting for capture')
}
},
afterCommit: () => {}
})
const run = async (operation: Parameters<typeof stages.apply>[0]) => {
const checkpoint = stages.checkpoint()
const committed = await batch.run([operation], (op) => stages.apply(op, 150))
const result = committed.outcomes[0]!
if (!result.committed) {
stages.rollback(checkpoint)
throw result.error
const stop = items.find((item) => item.itemId === stopId)!
writer
.prepare('UPDATE journal_items SET original_stop_note_json=?,wire_json=? WHERE item_id=?')
.run(
JSON.stringify(stop.body),
JSON.stringify({ ...stop, body: { kind: 'status', text: 'Cancellation requested.' } }),
stopId
)
if (failure === 'stale' || failure === 'mixed') {
writer
.prepare(
`UPDATE journal_items SET wire_version=NULL,wire_json=NULL WHERE ${failure === 'stale' ? '1' : 'ordinal%2=0'}`
)
.run()
}
stages.acknowledge()
return result.value
}
const begun = await run({
kind: 'fork-stage-begin',
input: {
target: { sessionId: 'child', epoch: 'empty', ownerToken: 'child-owner' },
identity: {
sessionId: 'child',
workspaceId: 'workspace',
hostId: 'local',
agent: 'codex',
providerHandle: codexProviderHandle('child')
},
fence: 1,
operationId: 'create',
fingerprint: 'fork-intent',
writer.pragma('wal_checkpoint(TRUNCATE)')
reader = new Database(filename, { readonly: true, fileMustExist: true })
const source = readJournalDomainForkBoundary(
new JournalSqlScope(writer, 'source-chat', 'epoch'),
'codex:source:T1:140',
undefined,
false
)
const boundary = structuredAgentSessionForkBoundary(
source.snapshot,
record,
'codex:source:T1:140'
)!
expect(boundary).not.toBeNull()
const assertPlans = captureJournalMetadataPlans(
reader,
'SELECT item_id,sequence,sequence_index,ordinal,turn_item_id,'
)
preparation = new JournalForkPreparation(reader, {
source: {
sessionId: 'source-chat',
throughItemId: 'codex:source:T1:140',
@@ -261,57 +199,283 @@ it('releases a WAL-pressured capture before staging and leaves no visible fork a
retained: boundary.retained,
location: record.location,
submissionsToken: null
}
}
})
if (!begun || !('token' in begun)) {
throw new Error('no private stage')
}
await run({
kind: 'fork-stage-page',
token: begun.token,
ordinal: 0,
items: output.slice(0, 64)
})
expect(
},
sourceThreadId: 'source',
childThreadId: 'child'
})
expect(preparation.next()).toEqual({ kind: 'progress', sourceReleased: false })
writer
.prepare(
"SELECT COUNT(*) AS count FROM journal_items WHERE session_id='child' AND epoch<>'empty'"
"UPDATE journal_items SET payload_json=json_set(payload_json,'$.body.blocks[0].text','new parent body') WHERE session_id='source-chat' AND item_id='codex:source:T1:140'"
)
.get()!.count
).toBeGreaterThan(0)
preparation.close()
preparation = undefined
reader.close()
reader = undefined
writer.close()
writerClosed = true
// Reopening has no live stage authority; the old head and absent public receipt still win.
const reopened = new Database(filename)
try {
.run()
writer
.prepare(
"DELETE FROM journal_items WHERE session_id='source-chat' AND item_id='codex:source:T1:139'"
)
.run()
const blocked = writer.prepare('PRAGMA wal_checkpoint(PASSIVE)').get()!
expect(Number(blocked.checkpointed)).toBeLessThan(Number(blocked.log))
for (;;) {
const step = preparation.next()
expect(step.kind).toBe('progress')
if (step.kind === 'progress' && step.sourceReleased) {
break
}
}
expect(reader.isTransaction).toBe(false)
assertPlans()
const released = writer.prepare('PRAGMA wal_checkpoint(PASSIVE)').get()!
expect(Number(released.checkpointed)).toBe(Number(released.log))
if (['temp-write', 'release', 'cleanup'].includes(failure)) {
const primary = new Error(`injected fork ${failure} failure`)
const exec = reader.exec.bind(reader)
const prepare = reader.prepare.bind(reader)
const fault =
failure === 'release'
? vi.spyOn(reader, 'exec').mockImplementation((sql) => {
if (sql === 'RELEASE fork_presentation') {
throw primary
}
exec(sql)
})
: vi.spyOn(reader, 'prepare').mockImplementation((sql) => {
const statement = prepare(sql)
if (/^(?:INSERT|UPDATE)/.test(sql)) {
vi.spyOn(statement, 'run').mockImplementationOnce(() => {
throw primary
})
}
return statement
})
const cleanup =
failure === 'cleanup'
? vi.spyOn(reader, 'exec').mockImplementation((sql) => {
if (sql.startsWith('DROP TABLE')) {
throw new Error('secondary cleanup failure')
}
exec(sql)
})
: undefined
try {
let observed: unknown
try {
preparation!.next()
} catch (error) {
observed = error
}
expect(observed).toBe(primary)
if (failure === 'cleanup') {
expect(() => reader!.prepare('SELECT 1').get()).toThrow()
} else {
expect(reader.isTransaction).toBe(false)
}
expect(() => preparation!.next()).toThrow('fork preparation is closed')
expect(
writer.prepare("SELECT 1 FROM agent_session_records WHERE session_id='child'").get()
).toBeUndefined()
} finally {
fault.mockRestore()
cleanup?.mockRestore()
if (failure === 'cleanup') {
preparation = undefined
reader = undefined
}
}
return
}
const assertPresentationPlans = captureJournalForkPresentationPlans(reader)
const output: JournalReplacementItem[] = []
let indivisiblePage = false
for (;;) {
const step = preparation.next()
expect(reader.isTransaction).toBe(false)
if (step.kind === 'complete') {
break
}
if (step.kind === 'page') {
expect(step.items.length).toBeLessThanOrEqual(64)
const actualBytes = step.items.reduce(
(bytes, item) => bytes + Buffer.byteLength(JSON.stringify(item)),
0
)
expect(step.bytes).toBeGreaterThanOrEqual(actualBytes)
if (step.bytes > 1024 * 1024) {
expect(step.items).toHaveLength(1)
indivisiblePage = true
}
output.push(...step.items)
}
}
assertPresentationPlans()
expect(indivisiblePage).toBe(true)
expect(
reopened.prepare("SELECT epoch FROM journal_chat_state WHERE session_id='child'").get()!
.epoch
).toBe('empty')
output
.filter(
({ identity }) =>
identity.provider === 'legacy' && identity.recordId.startsWith('continued-outside:')
)
.map(({ identity }) => identity)
).toEqual([
{
provider: 'legacy',
agent: 'codex',
sessionId: 'source',
recordId: 'continued-outside:T1'
}
])
expect(
reopened.prepare("SELECT 1 FROM agent_session_records WHERE session_id='child'").get()
).toBeUndefined()
output.filter(
({ identity }) => 'ordinal' in identity && [69, 130, 131].includes(identity.ordinal)
)
).toEqual([])
expect(output).toHaveLength(138)
expect(output[0]!.identity).toEqual({
provider: 'codex',
threadId: 'child',
turnId: 'T1',
ordinal: 0
})
expect(
reopened
output.find(({ identity }) => 'ordinal' in identity && identity.ordinal === 1)?.body
).toEqual({
kind: 'message',
role: 'assistant',
blocks: [{ type: 'text', text: frozenText }]
})
expect(
output.find(({ identity }) => 'ordinal' in identity && identity.ordinal === 2)?.body
).toMatchObject({
blocks: [
{ type: 'text' },
{
type: 'tool-result',
output: '雪🙂 codex:source:T1:1',
fullOutputItemId: 'codex:child:T1:1'
},
{ type: 'tool-result', fullOutputItemId: 'codex:source:missing:1' }
]
})
expect(
output.find(
({ identity }) => identity.provider === 'orca' && identity.clientMessageId === 'stop:fork'
)?.body
).toEqual({ kind: 'status', text: 'Cancellation requested.' })
expect(output).toEqual(
prepareForkSnapshot({
snapshot: {
sessionId: 'source-chat',
cursor: { epoch: 'epoch', sequence: 142 },
submissions,
items: items.map<AgentJournalRenderItem>((item) =>
item.itemId === stopId
? { ...item, body: { kind: 'status', text: 'Cancellation requested.' } }
: item
)
},
nextItemId: null,
sourceThreadId: 'source',
childThreadId: 'child'
})
)
writer
.prepare(
"INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) VALUES('workspace','child','empty',0,1)"
)
.run()
const stages = new JournalForkStages(writer, directory)
const batch = new JournalSqlBatch(writer, {
beforeWrite: async () => {
if (reader!.isTransaction) {
throw new Error('checkpoint admission is waiting for capture')
}
},
afterCommit: () => {}
})
const run = async (operation: Parameters<typeof stages.apply>[0]) => {
const checkpoint = stages.checkpoint()
const committed = await batch.run([operation], (op) => stages.apply(op, 150))
const result = committed.outcomes[0]!
if (!result.committed) {
stages.rollback(checkpoint)
throw result.error
}
stages.acknowledge()
return result.value
}
const begun = await run({
kind: 'fork-stage-begin',
input: {
target: { sessionId: 'child', epoch: 'empty', ownerToken: 'child-owner' },
identity: {
sessionId: 'child',
workspaceId: 'workspace',
hostId: 'local',
agent: 'codex',
providerHandle: codexProviderHandle('child')
},
fence: 1,
operationId: 'create',
fingerprint: 'fork-intent',
source: {
sessionId: 'source-chat',
throughItemId: 'codex:source:T1:140',
capability: 'checkpoint',
retained: boundary.retained,
location: record.location,
submissionsToken: null
}
}
})
if (!begun || !('token' in begun)) {
throw new Error('no private stage')
}
await run({
kind: 'fork-stage-page',
token: begun.token,
ordinal: 0,
items: output.slice(0, 64)
})
expect(
writer
.prepare(
"SELECT COUNT(*) AS count FROM journal_items item JOIN journal_chat_state head USING(session_id,epoch) WHERE item.session_id='child'"
"SELECT COUNT(*) AS count FROM journal_items WHERE session_id='child' AND epoch<>'empty'"
)
.get()!.count
).toBe(0)
} finally {
reopened.close()
}
} finally {
preparation?.close()
reader?.close()
if (!writerClosed) {
).toBeGreaterThan(0)
preparation.close()
preparation = undefined
reader.close()
reader = undefined
writer.close()
writerClosed = true
// Reopening has no live stage authority; the old head and absent public receipt still win.
const reopened = new Database(filename)
try {
expect(
reopened.prepare("SELECT epoch FROM journal_chat_state WHERE session_id='child'").get()!
.epoch
).toBe('empty')
expect(
reopened.prepare("SELECT 1 FROM agent_session_records WHERE session_id='child'").get()
).toBeUndefined()
expect(
reopened
.prepare(
"SELECT COUNT(*) AS count FROM journal_items item JOIN journal_chat_state head USING(session_id,epoch) WHERE item.session_id='child'"
)
.get()!.count
).toBe(0)
} finally {
reopened.close()
}
} finally {
preparation?.close()
reader?.close()
if (!writerClosed) {
writer.close()
}
await rm(directory, { recursive: true, force: true })
}
await rm(directory, { recursive: true, force: true })
}
})
)
@@ -44,7 +44,7 @@ export class JournalSqlForkCopy {
await setImmediate()
}
} catch (error) {
await copy.close()
await copy.close().catch(() => undefined)
throw error
}
}
@@ -94,8 +94,8 @@ export class JournalSqlForkCopy {
}
}
} catch (error) {
await this.close()
await writer.maintain({ kind: 'fork-abort', token })
await this.close().catch(() => undefined)
await writer.maintain({ kind: 'fork-abort', token }).catch(() => undefined)
this.requestSweep()
throw error
}
@@ -120,6 +120,19 @@ port.on('message', ({ id, operation }: JournalSqlReaderRequest) => {
break
}
} catch (error) {
if (fork) {
try {
fork.preparation.close()
} catch {
reply({ type: 'failed', error: journalSqlFailure(error) })
try {
db?.close()
} catch {}
db = undefined
port.close()
return
}
}
reply({ type: 'failed', id, error: journalSqlFailure(error) })
}
})