diff --git a/src/main/native-chat/agent-session-journal/journal-domain-item-display.ts b/src/main/native-chat/agent-session-journal/journal-domain-item-display.ts index 83f74e49147..c44c86cc732 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-item-display.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-item-display.ts @@ -80,18 +80,22 @@ export function journalDomainDisplayTurnState( export function readJournalDomainDisplayedRow( db: Database.Database, target: JournalDomainItemTarget, - row: SqliteRow + row: SqliteRow, + stopStates?: ReadonlyMap ): { item: AgentJournalRenderItem; payload: string } { const source: AgentJournalRenderItem = JSON.parse(String(row.payload_json)) const states = row.wire_version === AGENT_SESSION_ITEM_WIRE_VERSION ? undefined - : readJournalDomainStopStates(db, target, [String(row.item_id)]) + : (stopStates ?? readJournalDomainStopStates(db, target, [String(row.item_id)])) const item = resolveJournalDomainDisplay( source, row.wire_version, row.wire_json, states && journalDomainDisplayTurnState(source, states) ) - return { item, payload: item === source ? String(row.payload_json) : JSON.stringify(item) } + return { + item, + payload: item === source ? String(row.payload_json) : JSON.stringify(item) + } } diff --git a/src/main/native-chat/agent-session-journal/journal-domain-metadata-plan.test-fixture.ts b/src/main/native-chat/agent-session-journal/journal-domain-metadata-plan.test-fixture.ts index 89c5f9d29b0..b2772b518d5 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-metadata-plan.test-fixture.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-metadata-plan.test-fixture.ts @@ -50,3 +50,42 @@ export function assertJournalMetadataPlan(db: Database.Database, sql: string): v } } } + +export function captureJournalForkPresentationPlans(db: Database.Database): () => void { + const statements = vi.spyOn(db, 'prepare') + return () => { + const queries = [...new Set(statements.mock.calls.map(([sql]) => sql))] + statements.mockRestore() + const root = db + .prepare("SELECT rootpage FROM temp.sqlite_master WHERE name='fork_bodies'") + .get()!.rootpage + const payload = db + .prepare('PRAGMA temp.table_info(fork_bodies)') + .all() + .find((row) => row.name === 'payload')!.cid + expect(queries.some((sql) => sql.includes('body.lifecycle'))).toBe(true) + for (const sql of queries) { + const code = db.prepare(`EXPLAIN ${sql}`).all() + expect( + code.filter((row) => ['OpenRead', 'OpenWrite'].includes(String(row.opcode)) && row.p3 === 0) + ).toEqual([]) + expect( + db + .prepare(`EXPLAIN QUERY PLAN ${sql}`) + .all() + .map((row) => String(row.detail)) + .join('\n') + ).not.toContain('TEMP B-TREE') + if (sql.includes('body.lifecycle')) { + const cursors = new Set( + code + .filter((row) => row.opcode === 'OpenRead' && row.p2 === root && row.p3 === 1) + .map((row) => row.p1) + ) + for (const column of code.filter((row) => row.opcode === 'Column' && cursors.has(row.p1))) { + expect(Number(column.p2)).toBeLessThan(Number(payload)) + } + } + } + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-fork-capture.ts b/src/main/native-chat/agent-session-journal/journal-fork-capture.ts new file mode 100644 index 00000000000..cc96ab0ef2c --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-fork-capture.ts @@ -0,0 +1,102 @@ +import type Database from '../../sqlite/sync-database' +import { + agentJournalItemKey, + parseAgentJournalItemKey +} from '../../../shared/agent-session/journal/agent-session-journal-item-key' +import { readAgentJournalTurn } from '../../../shared/agent-session/lifecycle/agent-session-turn-record' +import { + forkDependencies, + omitForkPresentation +} from '../agent-session-wire/acquisition/structured-agent-session-fork-snapshot' +import { + readJournalDomainDisplayedRow, + readJournalDomainStopStates, + type JournalDomainItemTarget +} from './journal-domain-item-display' +import type { JournalForkPreparationInput } from './journal-fork-preparation' +import { journalForkRowBytes } from './journal-fork-workspace' + +/** One dependency item and scalar admission metadata survive a bounded capture step. */ +export class JournalForkCapture { + private after = 0 + private edges?: Generator + + constructor( + private readonly db: Database.Database, + private readonly target: JournalDomainItemTarget, + private readonly input: JournalForkPreparationInput, + private readonly end: number + ) {} + + next(budget: () => boolean, remaining: () => number, account: (bytes: number) => void): boolean { + const drainEdges = () => { + while (this.edges && budget()) { + const edge = this.edges.next() + if (edge.done) { + this.edges = undefined + } else { + this.db.prepare('INSERT OR IGNORE INTO temp.fork_edges VALUES(?,?)').run(...edge.value) + account(journalForkRowBytes(edge.value[0]) + journalForkRowBytes(edge.value[1])) + } + } + } + drainEdges() + if (!budget()) { + return false + } + const prefix = [this.target.sessionId, this.target.epoch] as const + const selection = `FROM temp.fork_headers header + WHERE header.rank>? AND (header.rank String(row.item_id)) + const states = readJournalDomainStopStates(this.db, this.target, ids) + let exhausted = true + for (const row of this.db + .prepare(`SELECT header.rank,header.item_id,header.provider_key, + item.wire_version,item.wire_json,item.payload_json + FROM temp.fork_headers header LEFT JOIN journal_items item + ON item.session_id=?3 AND item.epoch=?4 AND item.item_id=header.item_id + WHERE header.rank>?1 AND (header.rank + private captureState?: JournalForkCapture constructor( private readonly db: Database.Database, @@ -49,8 +47,8 @@ export class JournalForkPreparation { if (db.isTransaction) { throw new Error('fork preparation requires an exclusively reserved reader') } - createJournalForkWorkspace(db) try { + createJournalForkWorkspace(db) db.exec('BEGIN') this.sourceOpen = true const validated = validateJournalForkSource(db, input.source) @@ -63,7 +61,7 @@ export class JournalForkPreparation { .get(input.source.sessionId, this.epoch)!.floor ) } catch (error) { - this.close() + this.abort() throw error } } @@ -73,18 +71,37 @@ export class JournalForkPreparation { } next(): JournalForkPreparationResult { - if (this.presentation) { - return this.presentation.next() + if (this.phase === 'closed') { + throw new Error('fork preparation is closed') } + try { + if (this.presentation) { + if (this.sourceOpen || this.db.isTransaction) { + throw new Error('fork presentation requires a released source snapshot') + } + this.db.exec('SAVEPOINT fork_presentation') + const result = this.presentation.next() + this.db.exec('RELEASE fork_presentation') + return result + } + return this.capture() + } catch (error) { + this.abort() + throw error + } + } + + private capture(): JournalForkPreparationResult { const start = performance.now() let units = 0 let bytes = 0 const budget = () => units < MAX_ROWS && bytes < MAX_BYTES && (units === 0 || performance.now() - start < 8) - const account = (row: SqliteRow | string) => { + const accountBytes = (size: number) => { units++ - bytes += Buffer.byteLength(typeof row === 'string' ? row : JSON.stringify(row)) + bytes += size } + const account = (row: SqliteRow | string) => accountBytes(journalForkRowBytes(row)) const prefix = [this.input.source.sessionId, this.epoch] as const while (budget()) { switch (this.phase) { @@ -222,75 +239,55 @@ export class JournalForkPreparation { break } case 'capture': { - if (this.edges) { - const edge = this.edges.next() - if (edge.done) { - this.edges = undefined - } else { - this.db - .prepare('INSERT OR IGNORE INTO temp.fork_edges VALUES(?,?)') - .run(...edge.value) - account(JSON.stringify(edge.value)) - } - continue - } - const row = this.db - .prepare(`SELECT header.rank,header.item_id,header.provider_key,item.wire_version,item.wire_json,item.payload_json - FROM temp.fork_headers header CROSS JOIN journal_items item - WHERE header.rank>? AND (header.rank MAX_ROWS - units, accountBytes)) { // No domain writer acknowledgement is needed to release this WAL snapshot. this.db.exec('COMMIT') this.sourceOpen = false + this.captureState = undefined this.presentation = new JournalForkPresentation(this.db, this.input) this.after = 0 return { kind: 'progress', sourceReleased: true } } - const { item, payload } = readJournalDomainDisplayedRow( - this.db, - { sessionId: prefix[0], epoch: prefix[1] }, - row - ) - const identity = parseAgentJournalItemKey(String(row.provider_key)) - if (!identity) { - throw new Error('agent_session_fork:missing-identity') - } - const mapped = agentJournalItemKey( - identity.provider === 'codex' && - identity.threadId === this.input.sourceThreadId && - this.input.childThreadId - ? { ...identity, threadId: this.input.childThreadId } - : identity - ) - this.db - .prepare('INSERT INTO temp.fork_bodies(rank,item_id,payload,mapped) VALUES(?,?,?,?)') - .run(row.rank, item.itemId, payload, mapped) - this.edges = forkDependencies(item) - if (omitForkPresentation(item.body)) { - this.omit(item.itemId) - } - this.after = Number(row.rank) - account(row) break } + case 'closed': + throw new Error('fork preparation is closed') } } return { kind: 'progress', sourceReleased: !this.sourceOpen } } close(): void { - if (this.sourceOpen && this.db.isTransaction) { - this.db.exec('ROLLBACK') + try { + if (this.db.isTransaction) { + this.db.exec('ROLLBACK') + } + this.db.exec(`DROP TABLE IF EXISTS temp.fork_headers; DROP TABLE IF EXISTS temp.fork_aliases; + DROP TABLE IF EXISTS temp.fork_kept; DROP TABLE IF EXISTS temp.fork_bodies; DROP TABLE IF EXISTS temp.fork_edges; + DROP TABLE IF EXISTS temp.fork_omitted; DROP TABLE IF EXISTS temp.fork_work`) + } catch (error) { + try { + this.db.close() + } catch {} + throw error + } finally { + this.sourceOpen = false + this.captureState = undefined + this.presentation = undefined + this.phase = 'closed' } - this.sourceOpen = false - this.edges = undefined - this.db.exec(`DROP TABLE IF EXISTS temp.fork_headers; DROP TABLE IF EXISTS temp.fork_aliases; - DROP TABLE IF EXISTS temp.fork_kept; DROP TABLE IF EXISTS temp.fork_bodies; DROP TABLE IF EXISTS temp.fork_edges; - DROP TABLE IF EXISTS temp.fork_omitted; DROP TABLE IF EXISTS temp.fork_work`) - this.presentation = undefined + } + + private abort(): void { + try { + this.close() + } catch {} } private omit(id: string): void { diff --git a/src/main/native-chat/agent-session-journal/journal-fork-presentation.ts b/src/main/native-chat/agent-session-journal/journal-fork-presentation.ts index 4e7fe25096a..86d86b1ae65 100644 --- a/src/main/native-chat/agent-session-journal/journal-fork-presentation.ts +++ b/src/main/native-chat/agent-session-journal/journal-fork-presentation.ts @@ -1,9 +1,9 @@ +import { journalForkRowBytes } from './journal-fork-workspace' import type Database from '../../sqlite/sync-database' import type { SqliteRow } from '../../sqlite/sqlite-statement' import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types' import { agentJournalLinkageFields } from '../../../shared/agent-session/journal/agent-session-journal-producer' import { parseAgentJournalItemKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key' -import { readAgentJournalTurn } from '../../../shared/agent-session/lifecycle/agent-session-turn-record' import { mapBodyLinks } from '../agent-session-wire/acquisition/structured-agent-session-fork-snapshot' import { continuedOutsideRecordId } from '../agent-session-wire/acquisition/structured-agent-session-history-catch-up' import { @@ -41,7 +41,7 @@ export class JournalForkPresentation { units < 64 && bytes < MAX_BYTES && (units === 0 || performance.now() - start < 8) const account = (row: SqliteRow | string) => { units++ - bytes += Buffer.byteLength(typeof row === 'string' ? row : JSON.stringify(row)) + bytes += journalForkRowBytes(row) } while (budget()) { switch (this.phase) { @@ -86,55 +86,72 @@ export class JournalForkPresentation { break } case 'notices': { - const row = this.retainedRow(this.after) - if (!row) { - this.phase = 'output' - this.after = 0 - continue - } - const item: AgentJournalRenderItem = JSON.parse(String(row.payload)) - const key = this.db - .prepare('SELECT provider_key,member FROM temp.fork_headers WHERE rank=?') - .get(row.rank)! - const identity = parseAgentJournalItemKey(String(key.provider_key))! - const scoped = (...parts: string[]) => JSON.stringify(parts) - const noticePrefix = continuedOutsideRecordId('') - if (identity.provider === 'legacy' && identity.recordId.startsWith(noticePrefix)) { - const anchor = identity.recordId.slice(noticePrefix.length) - const notice = - identity.agent === 'codex' - ? scoped('codex', anchor) - : identity.agent === 'claude' - ? scoped('row', anchor) - : scoped(identity.agent, identity.sessionId, providerTimelineKeyPart(anchor)) - this.db - .prepare('UPDATE temp.fork_bodies SET notice=? WHERE rank=?') - .run(notice, row.rank) - } else if (!readAgentJournalTurn(item.body)) { - const work = (anchor: string) => + const anchors = new Map() + let exhausted = true + for (const row of this.db + .prepare(`SELECT body.rank,body.item_id,body.lifecycle,body.turn_scope_id, + header.provider_key,header.member,root.provider_key AS root_provider_key,root.turn_id AS root_turn_id + FROM temp.fork_bodies body JOIN temp.fork_headers header ON header.rank=body.rank + LEFT JOIN temp.fork_headers root ON root.item_id=body.turn_scope_id + WHERE body.rank>? AND NOT EXISTS(SELECT 1 FROM temp.fork_omitted WHERE id=body.item_id) + ORDER BY body.rank LIMIT ?`) + .iterate(this.after, 64 - units)) { + const identity = parseAgentJournalItemKey(String(row.provider_key))! + const scoped = (...parts: string[]) => JSON.stringify(parts) + const noticePrefix = continuedOutsideRecordId('') + if (identity.provider === 'legacy' && identity.recordId.startsWith(noticePrefix)) { + const anchor = identity.recordId.slice(noticePrefix.length) + const notice = + identity.agent === 'codex' + ? scoped('codex', anchor) + : identity.agent === 'claude' + ? scoped('row', anchor) + : scoped(identity.agent, identity.sessionId, providerTimelineKeyPart(anchor)) this.db - .prepare( - 'INSERT INTO temp.fork_work VALUES(?,?) ON CONFLICT(anchor) DO UPDATE SET last_rank=excluded.last_rank' - ) - .run(anchor, row.rank) - work(scoped('row', item.itemId)) - work(scoped('row', String(key.provider_key))) - if (this.input.sourceThreadId && key.member !== null) { - work(scoped('codex', String(key.member))) - } else if (item.turnScope?.kind === 'turn') { - const root = this.db - .prepare('SELECT provider_key,turn_id FROM temp.fork_headers WHERE item_id=?') - .get(item.turnScope.turnItemId) - const rootIdentity = root && parseAgentJournalItemKey(String(root.provider_key)) - const keyPart = - root?.turn_id == null ? null : providerTimelineTurnKeyPart(String(root.turn_id)) - if (rootIdentity?.provider === 'legacy' && keyPart !== null) { - work(scoped(rootIdentity.agent, rootIdentity.sessionId, keyPart)) + .prepare('UPDATE temp.fork_bodies SET notice=? WHERE rank=?') + .run(notice, row.rank) + bytes += Buffer.byteLength(notice) + } else if (!row.lifecycle) { + const work = (anchor: string) => { + anchors.set(anchor, Number(row.rank)) + bytes += Buffer.byteLength(anchor) + } + work(scoped('row', String(row.item_id))) + work(scoped('row', String(row.provider_key))) + if (this.input.sourceThreadId && row.member !== null) { + work(scoped('codex', String(row.member))) + } else if (row.turn_scope_id !== null) { + const rootIdentity = + row.root_provider_key === null + ? null + : parseAgentJournalItemKey(String(row.root_provider_key)) + const keyPart = + row.root_turn_id === null + ? null + : providerTimelineTurnKeyPart(String(row.root_turn_id)) + if (rootIdentity?.provider === 'legacy' && keyPart !== null) { + work(scoped(rootIdentity.agent, rootIdentity.sessionId, keyPart)) + } } } + this.after = Number(row.rank) + account(row) + if (!budget()) { + exhausted = false + break + } + } + if (anchors.size) { + this.db + .prepare(`INSERT INTO temp.fork_work(anchor,last_rank) + SELECT json_extract(value,'$[0]'),json_extract(value,'$[1]') FROM json_each(?) WHERE 1 + ON CONFLICT(anchor) DO UPDATE SET last_rank=MAX(fork_work.last_rank,excluded.last_rank)`) + .run(JSON.stringify([...anchors])) + } + if (exhausted) { + this.phase = 'output' + this.after = 0 } - this.after = Number(row.rank) - account(row) break } case 'output': { @@ -148,15 +165,21 @@ export class JournalForkPresentation { break } const item: AgentJournalRenderItem = JSON.parse(String(row.payload)) - const rewrite = (id: string) => - String( + let rewritten = false + const rewrite = (id: string) => { + const mapped = String( this.db.prepare('SELECT mapped FROM temp.fork_bodies WHERE item_id=?').get(id) ?.mapped ?? id ) + rewritten ||= mapped !== id + return mapped + } + const body = mapBodyLinks(item.body, rewrite) + const bodyUnchanged = !rewritten const identity = parseAgentJournalItemKey(String(row.mapped))! const output: JournalReplacementItem = { identity, - body: mapBodyLinks(item.body, rewrite), + body, observedAt: item.observedAt, ownerFence: 0, ...agentJournalLinkageFields(item), @@ -173,7 +196,11 @@ export class JournalForkPresentation { rank: Number(row.rank), mapped: String(row.mapped), item: output, - bytes: Buffer.byteLength(JSON.stringify(output)) + bytes: bodyUnchanged + ? Number(row.body_bytes) + + Buffer.byteLength(JSON.stringify({ ...output, body: null })) - + 4 + : Buffer.byteLength(JSON.stringify(output)) } } const output = this.pendingOutput @@ -204,7 +231,7 @@ export class JournalForkPresentation { } private retainedRow(after: number, notices = false): SqliteRow | undefined { return this.db - .prepare(`SELECT body.rank,body.item_id,body.payload,body.mapped FROM temp.fork_bodies body + .prepare(`SELECT body.rank,body.item_id,body.payload,body.mapped,body.body_bytes FROM temp.fork_bodies body WHERE rank>? AND NOT EXISTS(SELECT 1 FROM temp.fork_omitted WHERE id=body.item_id) ${notices ? 'AND (notice IS NULL OR EXISTS(SELECT 1 FROM temp.fork_work WHERE anchor=body.notice AND last_rank>body.rank))' : ''} ORDER BY rank LIMIT 1`) diff --git a/src/main/native-chat/agent-session-journal/journal-fork-workspace.ts b/src/main/native-chat/agent-session-journal/journal-fork-workspace.ts index 6601fa91c10..1b66dd825c8 100644 --- a/src/main/native-chat/agent-session-journal/journal-fork-workspace.ts +++ b/src/main/native-chat/agent-session-journal/journal-fork-workspace.ts @@ -21,13 +21,25 @@ export function createJournalForkWorkspace(db: Database.Database): void { CREATE INDEX temp.fork_headers_member ON fork_headers(member,rank); CREATE TEMP TABLE fork_aliases(item_id TEXT PRIMARY KEY,provider_key TEXT NOT NULL); CREATE TEMP TABLE fork_kept(member TEXT PRIMARY KEY); - CREATE TEMP TABLE fork_bodies(rank INTEGER PRIMARY KEY,item_id TEXT UNIQUE,payload TEXT NOT NULL,mapped TEXT NOT NULL,notice TEXT); + CREATE TEMP TABLE fork_bodies(rank INTEGER PRIMARY KEY,item_id TEXT UNIQUE,mapped TEXT NOT NULL,notice TEXT, + body_bytes INTEGER NOT NULL,lifecycle INTEGER NOT NULL,turn_scope_id TEXT,payload TEXT NOT NULL); CREATE TEMP TABLE fork_edges(parent TEXT NOT NULL,child TEXT NOT NULL,PRIMARY KEY(parent,child)) WITHOUT ROWID; CREATE TEMP TABLE fork_omitted(id TEXT UNIQUE,processed INTEGER NOT NULL DEFAULT 0); CREATE INDEX temp.fork_omitted_pending ON fork_omitted(processed); CREATE TEMP TABLE fork_work(anchor TEXT PRIMARY KEY,last_rank INTEGER NOT NULL);`) } +/** UTF-8 bound strings and numeric scalars, without serializing rows containing source JSON. */ +export function journalForkRowBytes(row: SqliteRow | string): number { + return typeof row === 'string' + ? Buffer.byteLength(row) + : Object.values(row).reduce( + (bytes, value) => + bytes + (typeof value === 'string' ? Buffer.byteLength(value) : value === null ? 0 : 8), + 0 + ) +} + export function journalForkHeaderMember(row: SqliteRow, sourceThreadId?: string): string | null { const identity = parseAgentJournalItemKey(String(row.provider_key)) let member: string | null = null diff --git a/src/main/native-chat/agent-session-journal/journal-sql-fork-copy.test.ts b/src/main/native-chat/agent-session-journal/journal-sql-fork-copy.test.ts index 7f6ab1c4229..e70420cb6b2 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-fork-copy.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-fork-copy.test.ts @@ -2,7 +2,7 @@ import type { JournalReplacementItem } from './context-epochs/journal-epoch-repl import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { expect, it } from 'vitest' +import { expect, it, vi } from 'vitest' import Database from '../../sqlite/sync-database' import { agentSessionRecordFixture } from '../agent-session-record-test-fixture' import { encodeAgentSessionRecord } from '../../../shared/agent-session/lifecycle/agent-session-record-stored-form' @@ -21,239 +21,177 @@ import { JournalSqlScope } from './journal-sql-scope' import { JournalSqlBatch } from './journal-sql-batch' import { seedJournalForkSource } from './journal-fork-source.test-fixture' import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key' -import { captureJournalMetadataPlans } from './journal-domain-metadata-plan.test-fixture' +import { + captureJournalMetadataPlans, + captureJournalForkPresentationPlans +} from './journal-domain-metadata-plan.test-fixture' -it('releases a WAL-pressured capture before staging and leaves no visible fork after a pre-install crash', async () => { - const directory = await mkdtemp(join(tmpdir(), 'orca-sql-fork-')) - const filename = join(directory, 'copy.sqlite') - const writer = new Database(filename) - let writerClosed = false - writer.pragma('journal_mode=WAL') - writer.pragma('wal_autocheckpoint=0') - messageFixture(writer) - let reader: Database.Database | undefined - let preparation: JournalForkPreparation | undefined - try { - const record = agentSessionRecordFixture({ - sessionId: 'source-chat', - provider: 'codex', - providerHandleChain: [ +it.each(['current', 'stale', 'mixed', 'temp-write', 'release', 'cleanup'])( + 'releases a WAL-pressured capture and keeps an interrupted private fork invisible (%s)', + async (failure) => { + const directory = await mkdtemp(join(tmpdir(), 'orca-sql-fork-')) + const filename = join(directory, 'copy.sqlite') + const writer = new Database(filename) + let writerClosed = false + writer.pragma('journal_mode=WAL') + writer.pragma('wal_autocheckpoint=0') + messageFixture(writer) + let reader: Database.Database | undefined + let preparation: JournalForkPreparation | undefined + try { + const record = agentSessionRecordFixture({ + sessionId: 'source-chat', + provider: 'codex', + providerHandleChain: [ + { + linkId: 'head', + origin: 'created', + mintedAtFence: 1, + observedAt: 1, + handle: codexProviderHandle('source') + } + ] + }) + writer + .prepare('INSERT INTO agent_session_records(session_id,record_json) VALUES(?,?)') + .run(record.sessionId, JSON.stringify(encodeAgentSessionRecord(record))) + const items: AgentJournalRenderItem[] = [ { - linkId: 'head', - origin: 'created', - mintedAtFence: 1, + itemId: agentJournalSubmissionKey('first-send'), + revision: 1, + sequence: 1, observedAt: 1, - handle: codexProviderHandle('source') + body: { + kind: 'message', + role: 'user', + blocks: [{ type: 'text', text: 'Keep the frozen answer' }] + } + }, + { + itemId: 'legacy:codex:source:root', + revision: 1, + sequence: 2, + observedAt: 2, + body: { + kind: 'turn', + turnId: 'T1', + userItemId: agentJournalSubmissionKey('first-send'), + state: 'interrupted' + } } ] - }) - writer - .prepare('INSERT INTO agent_session_records(session_id,record_json) VALUES(?,?)') - .run(record.sessionId, JSON.stringify(encodeAgentSessionRecord(record))) - const items: AgentJournalRenderItem[] = [ - { - itemId: agentJournalSubmissionKey('first-send'), - revision: 1, - sequence: 1, - observedAt: 1, - body: { - kind: 'message', - role: 'user', - blocks: [{ type: 'text', text: 'Keep the frozen answer' }] + const frozenText = '雪🙂 codex:source:T1:140 "quoted" \\ end\n'.repeat(32768) + const stopId = agentJournalSubmissionKey('stop:fork') + for (let ordinal = 1; ordinal <= 140; ordinal++) { + items.push({ + itemId: ordinal === 3 ? stopId : `codex:source:T1:${ordinal}`, + revision: 1, + sequence: ordinal + 2, + observedAt: ordinal + 2, + turnScope: { kind: 'turn', turnItemId: 'legacy:codex:source:root' }, + body: + ordinal === 3 + ? JSON.parse( + '{"kind":"status","text":"Saved cancellation failure.","failure":{"kind":"cancelUnconfirmed"}}' + ) + : { + kind: 'message', + role: 'assistant', + ...(ordinal === 69 ? { state: 'running' as const } : {}), + blocks: [ + { + type: 'text', + text: ordinal === 1 ? frozenText : `frozen-${ordinal} ${'x'.repeat(8192)}`, + ...([69, 130, 131].includes(ordinal) + ? { + usageLimitTurnItemId: `codex:source:T1:${ordinal === 130 ? 69 : 130}` + } + : {}) + }, + ...(ordinal === 2 + ? [ + { + type: 'tool-result' as const, + output: '雪🙂 codex:source:T1:1', + fullOutputItemId: 'codex:source:T1:1' + }, + { + type: 'tool-result' as const, + output: 'Missing reference stays frozen', + fullOutputItemId: 'codex:source:missing:1' + } + ] + : []) + ] + } + }) + } + for (const [ordinal, id] of [ + [4, 'legacy:codex:source:continued-outside%3AT1'], + [5, 'legacy:claude:source:continued-outside%3Acodex%3Asource%3AT1%3A130'] + ] as const) { + const item = items[ordinal + 1]! + item.itemId = id + item.body = { kind: 'status', text: 'Continued elsewhere' } + } + const submissions: AgentJournalSubmission[] = [ + { + clientMessageId: 'first-send', + fence: 1, + submittedAt: 1, + submittedSequence: 1, + payloadFingerprint: 'accepted-first-send', + dispatchState: 'accepted', + providerItemId: 'codex:source:T1:0', + acceptedSequence: 1, + reason: null, + resolvedAt: 1 } - }, - { - itemId: 'legacy:codex:source:root', - revision: 1, - sequence: 2, - observedAt: 2, - body: { - kind: 'turn', - turnId: 'T1', - userItemId: agentJournalSubmissionKey('first-send'), - state: 'completed' - } - } - ] - for (let ordinal = 1; ordinal <= 140; ordinal++) { - items.push({ - itemId: `codex:source:T1:${ordinal}`, - revision: 1, - sequence: ordinal + 2, - observedAt: ordinal + 2, - turnScope: { kind: 'turn', turnItemId: 'legacy:codex:source:root' }, - body: { - kind: 'message', - role: 'assistant', - ...(ordinal === 69 ? { state: 'running' as const } : {}), - blocks: [ - { - type: 'text', - text: `frozen-${ordinal} ${'x'.repeat(8192)}`, - ...([69, 130, 131].includes(ordinal) - ? { - usageLimitTurnItemId: `codex:source:T1:${ordinal === 130 ? 69 : 130}` - } - : {}) - } - ] - } - }) - } - const submissions: AgentJournalSubmission[] = [ - { - clientMessageId: 'first-send', - fence: 1, - submittedAt: 1, - submittedSequence: 1, - payloadFingerprint: 'accepted-first-send', - dispatchState: 'accepted', - providerItemId: 'codex:source:T1:0', - acceptedSequence: 1, - reason: null, - resolvedAt: 1 - } - ] - seedJournalForkSource( - writer, - { - sessionId: 'source-chat', - cursor: { epoch: 'epoch', sequence: 142 }, - items, - submissions - }, - record - ) - writer.pragma('wal_checkpoint(TRUNCATE)') - reader = new Database(filename) - const source = readJournalDomainForkBoundary( - new JournalSqlScope(writer, 'source-chat', 'epoch'), - 'codex:source:T1:140', - undefined, - false - ) - const boundary = structuredAgentSessionForkBoundary( - source.snapshot, - record, - 'codex:source:T1:140' - )! - expect(boundary).not.toBeNull() - const assertPlans = captureJournalMetadataPlans( - reader, - 'SELECT item_id,sequence,sequence_index,ordinal,turn_item_id,' - ) - preparation = new JournalForkPreparation(reader, { - source: { - sessionId: 'source-chat', - throughItemId: 'codex:source:T1:140', - capability: 'checkpoint', - retained: boundary.retained, - location: record.location, - submissionsToken: null - }, - sourceThreadId: 'source', - childThreadId: 'child' - }) - expect(preparation.next()).toEqual({ kind: 'progress', sourceReleased: false }) - writer - .prepare( - "UPDATE journal_items SET payload_json=json_set(payload_json,'$.body.blocks[0].text','new parent body') WHERE session_id='source-chat' AND item_id='codex:source:T1:140'" - ) - .run() - writer - .prepare( - "DELETE FROM journal_items WHERE session_id='source-chat' AND item_id='codex:source:T1:139'" - ) - .run() - const blocked = writer.prepare('PRAGMA wal_checkpoint(PASSIVE)').get()! - expect(Number(blocked.checkpointed)).toBeLessThan(Number(blocked.log)) - for (;;) { - const step = preparation.next() - expect(step.kind).toBe('progress') - if (step.kind === 'progress' && step.sourceReleased) { - break - } - } - expect(reader.isTransaction).toBe(false) - assertPlans() - const released = writer.prepare('PRAGMA wal_checkpoint(PASSIVE)').get()! - expect(Number(released.checkpointed)).toBe(Number(released.log)) - const output: JournalReplacementItem[] = [] - for (;;) { - const step = preparation.next() - if (step.kind === 'complete') { - break - } - if (step.kind === 'page') { - expect(step.items.length).toBeLessThanOrEqual(64) - expect(step.bytes).toBeLessThanOrEqual(1024 * 1024) - output.push(...step.items) - } - } - expect( - output.filter( - ({ identity }) => 'ordinal' in identity && [69, 130, 131].includes(identity.ordinal) - ) - ).toEqual([]) - expect(output).toHaveLength(139) - expect(output[0]!.identity).toEqual({ - provider: 'codex', - threadId: 'child', - turnId: 'T1', - ordinal: 0 - }) - expect(output).toEqual( - prepareForkSnapshot({ - snapshot: { + ] + seedJournalForkSource( + writer, + { sessionId: 'source-chat', cursor: { epoch: 'epoch', sequence: 142 }, - submissions, - items + items, + submissions }, - nextItemId: null, - sourceThreadId: 'source', - childThreadId: 'child' - }) - ) - writer - .prepare( - "INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) VALUES('workspace','child','empty',0,1)" + record ) - .run() - const stages = new JournalForkStages(writer, directory) - const batch = new JournalSqlBatch(writer, { - beforeWrite: async () => { - if (reader!.isTransaction) { - throw new Error('checkpoint admission is waiting for capture') - } - }, - afterCommit: () => {} - }) - const run = async (operation: Parameters[0]) => { - const checkpoint = stages.checkpoint() - const committed = await batch.run([operation], (op) => stages.apply(op, 150)) - const result = committed.outcomes[0]! - if (!result.committed) { - stages.rollback(checkpoint) - throw result.error + const stop = items.find((item) => item.itemId === stopId)! + writer + .prepare('UPDATE journal_items SET original_stop_note_json=?,wire_json=? WHERE item_id=?') + .run( + JSON.stringify(stop.body), + JSON.stringify({ ...stop, body: { kind: 'status', text: 'Cancellation requested.' } }), + stopId + ) + if (failure === 'stale' || failure === 'mixed') { + writer + .prepare( + `UPDATE journal_items SET wire_version=NULL,wire_json=NULL WHERE ${failure === 'stale' ? '1' : 'ordinal%2=0'}` + ) + .run() } - stages.acknowledge() - return result.value - } - const begun = await run({ - kind: 'fork-stage-begin', - input: { - target: { sessionId: 'child', epoch: 'empty', ownerToken: 'child-owner' }, - identity: { - sessionId: 'child', - workspaceId: 'workspace', - hostId: 'local', - agent: 'codex', - providerHandle: codexProviderHandle('child') - }, - fence: 1, - operationId: 'create', - fingerprint: 'fork-intent', + writer.pragma('wal_checkpoint(TRUNCATE)') + reader = new Database(filename, { readonly: true, fileMustExist: true }) + const source = readJournalDomainForkBoundary( + new JournalSqlScope(writer, 'source-chat', 'epoch'), + 'codex:source:T1:140', + undefined, + false + ) + const boundary = structuredAgentSessionForkBoundary( + source.snapshot, + record, + 'codex:source:T1:140' + )! + expect(boundary).not.toBeNull() + const assertPlans = captureJournalMetadataPlans( + reader, + 'SELECT item_id,sequence,sequence_index,ordinal,turn_item_id,' + ) + preparation = new JournalForkPreparation(reader, { source: { sessionId: 'source-chat', throughItemId: 'codex:source:T1:140', @@ -261,57 +199,283 @@ it('releases a WAL-pressured capture before staging and leaves no visible fork a retained: boundary.retained, location: record.location, submissionsToken: null - } - } - }) - if (!begun || !('token' in begun)) { - throw new Error('no private stage') - } - await run({ - kind: 'fork-stage-page', - token: begun.token, - ordinal: 0, - items: output.slice(0, 64) - }) - expect( + }, + sourceThreadId: 'source', + childThreadId: 'child' + }) + expect(preparation.next()).toEqual({ kind: 'progress', sourceReleased: false }) writer .prepare( - "SELECT COUNT(*) AS count FROM journal_items WHERE session_id='child' AND epoch<>'empty'" + "UPDATE journal_items SET payload_json=json_set(payload_json,'$.body.blocks[0].text','new parent body') WHERE session_id='source-chat' AND item_id='codex:source:T1:140'" ) - .get()!.count - ).toBeGreaterThan(0) - preparation.close() - preparation = undefined - reader.close() - reader = undefined - writer.close() - writerClosed = true - // Reopening has no live stage authority; the old head and absent public receipt still win. - const reopened = new Database(filename) - try { + .run() + writer + .prepare( + "DELETE FROM journal_items WHERE session_id='source-chat' AND item_id='codex:source:T1:139'" + ) + .run() + const blocked = writer.prepare('PRAGMA wal_checkpoint(PASSIVE)').get()! + expect(Number(blocked.checkpointed)).toBeLessThan(Number(blocked.log)) + for (;;) { + const step = preparation.next() + expect(step.kind).toBe('progress') + if (step.kind === 'progress' && step.sourceReleased) { + break + } + } + expect(reader.isTransaction).toBe(false) + assertPlans() + const released = writer.prepare('PRAGMA wal_checkpoint(PASSIVE)').get()! + expect(Number(released.checkpointed)).toBe(Number(released.log)) + if (['temp-write', 'release', 'cleanup'].includes(failure)) { + const primary = new Error(`injected fork ${failure} failure`) + const exec = reader.exec.bind(reader) + const prepare = reader.prepare.bind(reader) + const fault = + failure === 'release' + ? vi.spyOn(reader, 'exec').mockImplementation((sql) => { + if (sql === 'RELEASE fork_presentation') { + throw primary + } + exec(sql) + }) + : vi.spyOn(reader, 'prepare').mockImplementation((sql) => { + const statement = prepare(sql) + if (/^(?:INSERT|UPDATE)/.test(sql)) { + vi.spyOn(statement, 'run').mockImplementationOnce(() => { + throw primary + }) + } + return statement + }) + const cleanup = + failure === 'cleanup' + ? vi.spyOn(reader, 'exec').mockImplementation((sql) => { + if (sql.startsWith('DROP TABLE')) { + throw new Error('secondary cleanup failure') + } + exec(sql) + }) + : undefined + try { + let observed: unknown + try { + preparation!.next() + } catch (error) { + observed = error + } + expect(observed).toBe(primary) + if (failure === 'cleanup') { + expect(() => reader!.prepare('SELECT 1').get()).toThrow() + } else { + expect(reader.isTransaction).toBe(false) + } + expect(() => preparation!.next()).toThrow('fork preparation is closed') + expect( + writer.prepare("SELECT 1 FROM agent_session_records WHERE session_id='child'").get() + ).toBeUndefined() + } finally { + fault.mockRestore() + cleanup?.mockRestore() + if (failure === 'cleanup') { + preparation = undefined + reader = undefined + } + } + return + } + const assertPresentationPlans = captureJournalForkPresentationPlans(reader) + const output: JournalReplacementItem[] = [] + let indivisiblePage = false + for (;;) { + const step = preparation.next() + expect(reader.isTransaction).toBe(false) + if (step.kind === 'complete') { + break + } + if (step.kind === 'page') { + expect(step.items.length).toBeLessThanOrEqual(64) + const actualBytes = step.items.reduce( + (bytes, item) => bytes + Buffer.byteLength(JSON.stringify(item)), + 0 + ) + expect(step.bytes).toBeGreaterThanOrEqual(actualBytes) + if (step.bytes > 1024 * 1024) { + expect(step.items).toHaveLength(1) + indivisiblePage = true + } + output.push(...step.items) + } + } + assertPresentationPlans() + expect(indivisiblePage).toBe(true) expect( - reopened.prepare("SELECT epoch FROM journal_chat_state WHERE session_id='child'").get()! - .epoch - ).toBe('empty') + output + .filter( + ({ identity }) => + identity.provider === 'legacy' && identity.recordId.startsWith('continued-outside:') + ) + .map(({ identity }) => identity) + ).toEqual([ + { + provider: 'legacy', + agent: 'codex', + sessionId: 'source', + recordId: 'continued-outside:T1' + } + ]) expect( - reopened.prepare("SELECT 1 FROM agent_session_records WHERE session_id='child'").get() - ).toBeUndefined() + output.filter( + ({ identity }) => 'ordinal' in identity && [69, 130, 131].includes(identity.ordinal) + ) + ).toEqual([]) + expect(output).toHaveLength(138) + expect(output[0]!.identity).toEqual({ + provider: 'codex', + threadId: 'child', + turnId: 'T1', + ordinal: 0 + }) expect( - reopened + output.find(({ identity }) => 'ordinal' in identity && identity.ordinal === 1)?.body + ).toEqual({ + kind: 'message', + role: 'assistant', + blocks: [{ type: 'text', text: frozenText }] + }) + expect( + output.find(({ identity }) => 'ordinal' in identity && identity.ordinal === 2)?.body + ).toMatchObject({ + blocks: [ + { type: 'text' }, + { + type: 'tool-result', + output: '雪🙂 codex:source:T1:1', + fullOutputItemId: 'codex:child:T1:1' + }, + { type: 'tool-result', fullOutputItemId: 'codex:source:missing:1' } + ] + }) + expect( + output.find( + ({ identity }) => identity.provider === 'orca' && identity.clientMessageId === 'stop:fork' + )?.body + ).toEqual({ kind: 'status', text: 'Cancellation requested.' }) + expect(output).toEqual( + prepareForkSnapshot({ + snapshot: { + sessionId: 'source-chat', + cursor: { epoch: 'epoch', sequence: 142 }, + submissions, + items: items.map((item) => + item.itemId === stopId + ? { ...item, body: { kind: 'status', text: 'Cancellation requested.' } } + : item + ) + }, + nextItemId: null, + sourceThreadId: 'source', + childThreadId: 'child' + }) + ) + writer + .prepare( + "INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) VALUES('workspace','child','empty',0,1)" + ) + .run() + const stages = new JournalForkStages(writer, directory) + const batch = new JournalSqlBatch(writer, { + beforeWrite: async () => { + if (reader!.isTransaction) { + throw new Error('checkpoint admission is waiting for capture') + } + }, + afterCommit: () => {} + }) + const run = async (operation: Parameters[0]) => { + const checkpoint = stages.checkpoint() + const committed = await batch.run([operation], (op) => stages.apply(op, 150)) + const result = committed.outcomes[0]! + if (!result.committed) { + stages.rollback(checkpoint) + throw result.error + } + stages.acknowledge() + return result.value + } + const begun = await run({ + kind: 'fork-stage-begin', + input: { + target: { sessionId: 'child', epoch: 'empty', ownerToken: 'child-owner' }, + identity: { + sessionId: 'child', + workspaceId: 'workspace', + hostId: 'local', + agent: 'codex', + providerHandle: codexProviderHandle('child') + }, + fence: 1, + operationId: 'create', + fingerprint: 'fork-intent', + source: { + sessionId: 'source-chat', + throughItemId: 'codex:source:T1:140', + capability: 'checkpoint', + retained: boundary.retained, + location: record.location, + submissionsToken: null + } + } + }) + if (!begun || !('token' in begun)) { + throw new Error('no private stage') + } + await run({ + kind: 'fork-stage-page', + token: begun.token, + ordinal: 0, + items: output.slice(0, 64) + }) + expect( + writer .prepare( - "SELECT COUNT(*) AS count FROM journal_items item JOIN journal_chat_state head USING(session_id,epoch) WHERE item.session_id='child'" + "SELECT COUNT(*) AS count FROM journal_items WHERE session_id='child' AND epoch<>'empty'" ) .get()!.count - ).toBe(0) - } finally { - reopened.close() - } - } finally { - preparation?.close() - reader?.close() - if (!writerClosed) { + ).toBeGreaterThan(0) + preparation.close() + preparation = undefined + reader.close() + reader = undefined writer.close() + writerClosed = true + // Reopening has no live stage authority; the old head and absent public receipt still win. + const reopened = new Database(filename) + try { + expect( + reopened.prepare("SELECT epoch FROM journal_chat_state WHERE session_id='child'").get()! + .epoch + ).toBe('empty') + expect( + reopened.prepare("SELECT 1 FROM agent_session_records WHERE session_id='child'").get() + ).toBeUndefined() + expect( + reopened + .prepare( + "SELECT COUNT(*) AS count FROM journal_items item JOIN journal_chat_state head USING(session_id,epoch) WHERE item.session_id='child'" + ) + .get()!.count + ).toBe(0) + } finally { + reopened.close() + } + } finally { + preparation?.close() + reader?.close() + if (!writerClosed) { + writer.close() + } + await rm(directory, { recursive: true, force: true }) } - await rm(directory, { recursive: true, force: true }) } -}) +) diff --git a/src/main/native-chat/agent-session-journal/journal-sql-fork-copy.ts b/src/main/native-chat/agent-session-journal/journal-sql-fork-copy.ts index 0d9d5e09b36..f3df70c1756 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-fork-copy.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-fork-copy.ts @@ -44,7 +44,7 @@ export class JournalSqlForkCopy { await setImmediate() } } catch (error) { - await copy.close() + await copy.close().catch(() => undefined) throw error } } @@ -94,8 +94,8 @@ export class JournalSqlForkCopy { } } } catch (error) { - await this.close() - await writer.maintain({ kind: 'fork-abort', token }) + await this.close().catch(() => undefined) + await writer.maintain({ kind: 'fork-abort', token }).catch(() => undefined) this.requestSweep() throw error } diff --git a/src/main/native-chat/agent-session-journal/journal-sql-reader-entry.ts b/src/main/native-chat/agent-session-journal/journal-sql-reader-entry.ts index bbd11b0b5ee..4c7adac77f4 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-reader-entry.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-reader-entry.ts @@ -120,6 +120,19 @@ port.on('message', ({ id, operation }: JournalSqlReaderRequest) => { break } } catch (error) { + if (fork) { + try { + fork.preparation.close() + } catch { + reply({ type: 'failed', error: journalSqlFailure(error) }) + try { + db?.close() + } catch {} + db = undefined + port.close() + return + } + } reply({ type: 'failed', id, error: journalSqlFailure(error) }) } })