From 5dae0b92d54f3fcf407b4348d8a49f7cb1289a5a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:22:30 -0800 Subject: [PATCH] refactor: use one chat durability mode and ready batch selector --- ...ctured-journal-settlement-rollback.test.ts | 32 ++-- .../context-epochs/journal-epoch-rollover.ts | 1 - .../journal-checkpoint-worker-source.ts | 2 +- .../database/journal-database.ts | 4 +- .../database/journal-host-database.ts | 33 +--- .../journal-sql-batch.test.ts | 141 ++++++------------ .../journal-sql-batch.ts | 24 +-- .../journal-sql-command.test.ts | 82 ++-------- .../journal-sql-command.ts | 37 ----- .../journal-sql-input-command.test-fixture.ts | 4 +- .../journal-sql-maintenance.ts | 5 - .../journal-sql-message-commit.ts | 97 +----------- .../journal-sql-message.test.ts | 11 +- .../journal-sql-message.ts | 23 +-- .../journal-sql-open.test.ts | 4 +- .../journal-sql-owner.ts | 6 +- .../journal-sql-protocol.ts | 5 +- .../journal-sql-ready-batch.ts | 116 +++++++++----- .../journal-sql-reopen-oracle.test.ts | 9 +- .../journal-sql-worker-entry.ts | 68 ++++----- .../journal-writer-session.ts | 2 +- .../journal-store-collaborators.ts | 6 +- ...revision.ts => journal-output-revision.ts} | 42 ------ .../writing/journal-row-writer.ts | 11 -- ...streamed-revision-contract-test-fixture.ts | 78 ---------- 25 files changed, 205 insertions(+), 638 deletions(-) rename src/main/native-chat/agent-session-journal/writing/{journal-streamed-text-revision.ts => journal-output-revision.ts} (61%) delete mode 100644 src/main/native-chat/agent-session-journal/writing/journal-streamed-revision-contract-test-fixture.ts diff --git a/src/main/codex/structured/journal/codex-structured-journal-settlement-rollback.test.ts b/src/main/codex/structured/journal/codex-structured-journal-settlement-rollback.test.ts index b0d19818fb5..31e81be813a 100644 --- a/src/main/codex/structured/journal/codex-structured-journal-settlement-rollback.test.ts +++ b/src/main/codex/structured/journal/codex-structured-journal-settlement-rollback.test.ts @@ -83,24 +83,20 @@ it('rolls back the entire Codex exit settlement when a later row fails', async ( const diskBefore = liveTestJournalRows(database.db, SESSION) const sawEarlierInserts: number[] = [] const transaction = database.transaction.bind(database) - vi.spyOn(database, 'transaction').mockImplementation((run, streamed, committed) => - transaction( - (db) => { - try { - return run(db) - } catch (error) { - sawEarlierInserts.push(liveTestJournalRows(db, SESSION).length - diskBefore.length) - expect(journal.snapshot()).toEqual(before) - expect(commits).not.toHaveBeenCalled() - expect( - published.every((snapshot) => JSON.stringify(snapshot) === JSON.stringify(before)) - ).toBe(true) - throw error - } - }, - streamed, - committed - ) + vi.spyOn(database, 'transaction').mockImplementation((run, committed) => + transaction((db) => { + try { + return run(db) + } catch (error) { + sawEarlierInserts.push(liveTestJournalRows(db, SESSION).length - diskBefore.length) + expect(journal.snapshot()).toEqual(before) + expect(commits).not.toHaveBeenCalled() + expect( + published.every((snapshot) => JSON.stringify(snapshot) === JSON.stringify(before)) + ).toBe(true) + throw error + } + }, committed) ) database.db.exec(`CREATE TEMP TRIGGER reject_codex_later BEFORE INSERT ON journal_rows WHEN NEW.session_id = '${SESSION}' AND NEW.seq > ${before.cursor.sequence + 1} diff --git a/src/main/native-chat/agent-session-journal/context-epochs/journal-epoch-rollover.ts b/src/main/native-chat/agent-session-journal/context-epochs/journal-epoch-rollover.ts index d82b04f6227..bb89cc0dcb5 100644 --- a/src/main/native-chat/agent-session-journal/context-epochs/journal-epoch-rollover.ts +++ b/src/main/native-chat/agent-session-journal/context-epochs/journal-epoch-rollover.ts @@ -74,7 +74,6 @@ export function publishNewEpoch(input: { publishJournalSessionEpoch(db, input.identity, input.epoch) return publishJournalOutcomeEpoch(db, sessionId, state.epoch, rows) }, - false, (publication) => { // Adopt, then finish the nonthrowing queue bookkeeping, before the fallible publication. const state = publication.adopt() diff --git a/src/main/native-chat/agent-session-journal/database/journal-checkpoint-worker-source.ts b/src/main/native-chat/agent-session-journal/database/journal-checkpoint-worker-source.ts index 013aad286b7..75fa7b192e6 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-checkpoint-worker-source.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-checkpoint-worker-source.ts @@ -10,7 +10,7 @@ try { if (db.prepare('PRAGMA journal_mode').get().journal_mode !== 'wal') { throw new Error('chat journal is not in WAL mode'); } - db.exec('PRAGMA busy_timeout = 0; PRAGMA synchronous = FULL; PRAGMA checkpoint_fullfsync = ON;'); + db.exec('PRAGMA busy_timeout = 0; PRAGMA synchronous = NORMAL; PRAGMA checkpoint_fullfsync = ON;'); db.exec('PRAGMA journal_size_limit = ' + workerData.sizeLimit); const frames = db.prepare('PRAGMA wal_checkpoint(NOOP)'); const checkpoint = db.prepare('PRAGMA wal_checkpoint(PASSIVE)'); diff --git a/src/main/native-chat/agent-session-journal/database/journal-database.ts b/src/main/native-chat/agent-session-journal/database/journal-database.ts index 70631f1c1c6..e0680831a67 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-database.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-database.ts @@ -16,7 +16,7 @@ import { ensureAgentSessionAttachmentClaimTables } from '../../agent-session-att export const JOURNAL_BUSY_TIMEOUT_MS = 5000 export const JOURNAL_SIZE_LIMIT_BYTES = 32 * 1024 * 1024 -const JOURNAL_SYNCHRONOUS = 'FULL' +const JOURNAL_SYNCHRONOUS = 'NORMAL' export type OpenJournalDatabase = { db: Database.Database @@ -97,7 +97,7 @@ function configureJournalPragmas(db: Database.Database, stored: number): void { db.pragma('journal_mode = WAL') db.pragma(`busy_timeout = ${JOURNAL_BUSY_TIMEOUT_MS}`) db.pragma('foreign_keys = ON') - // FULL fsyncs submissions before dispatch; checkpoint_fullfsync also flushes macOS drive caches. + // WAL checkpoints flush committed pages and macOS drive caches. db.pragma(`synchronous = ${JOURNAL_SYNCHRONOUS}`) db.pragma('checkpoint_fullfsync = ON') db.pragma(`journal_size_limit = ${JOURNAL_SIZE_LIMIT_BYTES}`) diff --git a/src/main/native-chat/agent-session-journal/database/journal-host-database.ts b/src/main/native-chat/agent-session-journal/database/journal-host-database.ts index b77c96a6f57..b10ce11b04a 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-host-database.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-host-database.ts @@ -72,35 +72,13 @@ export class JournalHostDatabase { return readJournalSessionEpoch(this.db, sessionId) } - /** Streamed text may lose only its trailing suffix on OS crash; the next FULL commit makes it durable. */ - transaction( - run: (db: Database.Database) => T, - streamed = false, - committed?: (result: T) => void - ): T { + transaction(run: (db: Database.Database) => T, committed?: (result: T) => void): T { const db = this.db this.transactionNumber += 1 - if (!streamed) { - const result = runJournalTransaction(db, run, () => { - this.stranded = true - }) - committed?.(result) - this.checkpoint?.afterCommit() - return result - } - // Synchronous level changes require no transaction; stranded recovery restores FULL first. - db.pragma('synchronous = NORMAL') - let result: T - try { - result = runJournalTransaction(db, run, () => { - this.stranded = true - }) - committed?.(result) - } finally { - if (!db.isTransaction) { - db.pragma('synchronous = FULL') - } - } + const result = runJournalTransaction(db, run, () => { + this.stranded = true + }) + committed?.(result) this.checkpoint?.afterCommit() return result } @@ -139,7 +117,6 @@ export class JournalHostDatabase { throw journalOpenRefusalError(error) } } - connection.pragma('synchronous = FULL') this.stranded = false } } diff --git a/src/main/native-chat/agent-session-journal/journal-sql-batch.test.ts b/src/main/native-chat/agent-session-journal/journal-sql-batch.test.ts index 86bf23f2171..14506f36cce 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-batch.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-batch.test.ts @@ -1,7 +1,9 @@ import { afterEach, expect, it, vi } from 'vitest' import Database from '../../sqlite/sync-database' import { JournalSqlBatch, JournalSqlBatchError } from './journal-sql-batch' -import { takeJournalSqlReadyBatch } from './journal-sql-ready-batch' +import { prepareJournalSqlRequest, takeJournalSqlReadyBatch } from './journal-sql-ready-batch' +import { prepareJournalSqlMessages } from './journal-sql-message-commit' +import { message } from './journal-sql-message.test-fixture' import type { JournalSqlRequest } from './journal-sql-protocol' const databases: Database.Database[] = [] @@ -22,21 +24,14 @@ function fixture() { it('rolls back a failed command while later commands see preceding successful writes', async () => { const { db, batch } = fixture() const failure = new Error('command failed') - const answer = await batch.run( - [ - { command: 1, normal: true }, - { command: 2, normal: true }, - { command: 3, normal: false } - ], - (id) => { - const previous = db.prepare('SELECT id FROM writes ORDER BY id').all() - db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written') - if (id === 2) { - throw failure - } - return previous + const answer = await batch.run([1, 2, 3], (id) => { + const previous = db.prepare('SELECT id FROM writes ORDER BY id').all() + db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written') + if (id === 2) { + throw failure } - ) + return previous + }) expect(answer).toEqual({ outcomes: [ { committed: true, value: [] }, @@ -51,38 +46,19 @@ it('rolls back a failed command while later commands see preceding successful wr expect(db.isTransaction).toBe(false) }) -it.each([ - { normal: [true, true], synchronous: 1 }, - { normal: [true, false], synchronous: 2 } -])('uses batch durability $synchronous and restores FULL before the next batch', async (input) => { +it('leaves newly admitted work for a new drain', async () => { const { db, batch } = fixture() - const answer = await batch.run( - input.normal.map((normal, id) => ({ command: id, normal })), - (id) => { - db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written') - return db.pragma('synchronous', { simple: true }) - } - ) - expect(answer.outcomes).toEqual([ - { committed: true, value: input.synchronous }, - { committed: true, value: input.synchronous } - ]) - expect(db.pragma('synchronous', { simple: true })).toBe(2) -}) - -it('leaves newly admitted work for a new drain with its own durability', async () => { - const { db, batch } = fixture() - const admitted = [{ command: 1, normal: true }] + const admitted = [1] const first = await batch.run(admitted, (id) => { db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'stream') - admitted.push({ command: 2, normal: false }) + admitted.push(2) return id }) expect(first.outcomes).toEqual([{ committed: true, value: 1 }]) expect(db.prepare('SELECT id, value FROM writes').all()).toEqual([{ id: 1, value: 'stream' }]) const send = await batch.run(admitted.slice(1), (id) => { db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'send') - return db.pragma('synchronous', { simple: true }) + return id }) expect(send.outcomes).toEqual([{ committed: true, value: 2 }]) expect(db.prepare('SELECT id, value FROM writes ORDER BY id').all()).toEqual([ @@ -91,43 +67,39 @@ it('leaves newly admitted work for a new drain with its own durability', async ( ]) }) -it('commits a control receipt before later streamed work with its own durability', async () => { +it('keeps bounded snapshot prefixes and controls in enqueue order', async () => { const { db, batch } = fixture() - const stream = (id: number): JournalSqlRequest => ({ - id, - command: { - type: 'write', - command: { - target: { sessionId: 'session', epoch: 'epoch', ownerToken: 'owner' }, - operation: { - kind: 'item', - identity: { provider: 'orca', clientMessageId: 'output' }, - body: { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'output' }] }, - options: { fence: 1, turnScope: { kind: 'thread' } } - } - } - } - }) + const stream = (id: number, steps: number, text = 'output'): JournalSqlRequest => + prepareJournalSqlRequest(id, { + type: 'messages', + job: prepareJournalSqlMessages({ + target: { sessionId: `session${id}`, epoch: 'epoch' }, + steps: Array.from({ length: steps }, () => message(text)), + savedAt: 100 + }) + }) const queue: JournalSqlRequest[] = [ - stream(1), - { id: 2, command: { type: 'effects', effects: [] } }, - stream(3) + stream(1, 32), + stream(2, 32), + stream(3, 1), + { id: 4, command: { type: 'effects', effects: [] } }, + stream(5, 1, 'x'.repeat(700 * 1024)), + stream(6, 1, 'x'.repeat(700 * 1024)) ] const commit = async () => - batch.run( - takeJournalSqlReadyBatch(queue, () => true).map(({ request, normal }) => ({ - command: request, - normal - })), - (request) => { - db.prepare('INSERT INTO writes VALUES (?, ?)').run(request.id, 'written') - return { id: request.id, durability: db.pragma('synchronous', { simple: true }) } - } - ) - expect((await commit()).outcomes).toEqual([{ committed: true, value: { id: 1, durability: 1 } }]) - expect((await commit()).outcomes).toEqual([{ committed: true, value: { id: 2, durability: 2 } }]) + batch.run(takeJournalSqlReadyBatch(queue), (request) => { + db.prepare('INSERT INTO writes VALUES (?, ?)').run(request.id, 'written') + return request.id + }) + expect((await commit()).outcomes).toEqual([ + { committed: true, value: 1 }, + { committed: true, value: 2 } + ]) expect(db.prepare('SELECT id FROM writes ORDER BY id').all()).toEqual([{ id: 1 }, { id: 2 }]) - expect((await commit()).outcomes).toEqual([{ committed: true, value: { id: 3, durability: 1 } }]) + expect((await commit()).outcomes).toEqual([{ committed: true, value: 3 }]) + expect((await commit()).outcomes).toEqual([{ committed: true, value: 4 }]) + expect((await commit()).outcomes).toEqual([{ committed: true, value: 5 }]) + expect((await commit()).outcomes).toEqual([{ committed: true, value: 6 }]) }) it.each([false, true])( @@ -146,7 +118,7 @@ it.each([false, true])( }) let failure: unknown try { - await batch.run([{ command: 1, normal: false }], (id) => { + await batch.run([1], (id) => { db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written') return id }) @@ -156,34 +128,13 @@ it.each([false, true])( expect(failure).toBeInstanceOf(JournalSqlBatchError) expect(failure).toMatchObject({ outcome: 'indeterminate' }) expect(db.prepare('SELECT id FROM writes').all()).toEqual(committed ? [{ id: 1 }] : []) - await expect(batch.run([{ command: 2, normal: false }], (id) => id)).rejects.toThrow( + await expect(batch.run([2], (id) => id)).rejects.toThrow( expect.objectContaining({ outcome: 'not-dispatched' }) ) expect(db.prepare('SELECT id FROM writes').all()).toEqual(committed ? [{ id: 1 }] : []) } ) -it('acknowledges a committed stream even when restoring FULL fails afterwards', async () => { - const { db, batch } = fixture() - const pragma = db.pragma.bind(db) - let committed = false - const failure = new Error('FULL setting failed') - vi.spyOn(db, 'pragma').mockImplementation((sql, options) => { - if (sql === 'synchronous = FULL' && committed) { - throw failure - } - return pragma(sql, options) - }) - const answer = await batch.run([{ command: 1, normal: true }], (id) => { - db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written') - committed = true - return id - }) - expect(answer).toEqual({ outcomes: [{ committed: true, value: 1 }], postcommit: failure }) - expect(db.prepare('SELECT id FROM writes').all()).toEqual([{ id: 1 }]) - expect(batch.fault).toBe(failure) -}) - it('does not claim rollback when both command and outer rollback fail', async () => { const { db, batch } = fixture() const exec = db.exec.bind(db) @@ -195,13 +146,13 @@ it('does not claim rollback when both command and outer rollback fail', async () exec(sql) }) await expect( - batch.run([{ command: 1, normal: false }], (id) => { + batch.run([1], (id) => { db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'uncommitted') throw new Error('command failed') }) ).rejects.toThrow(expect.objectContaining({ outcome: 'indeterminate', rollback })) expect(db.isTransaction).toBe(true) - await expect(batch.run([{ command: 2, normal: false }], (id) => id)).rejects.toThrow( + await expect(batch.run([2], (id) => id)).rejects.toThrow( expect.objectContaining({ outcome: 'not-dispatched' }) ) exec('ROLLBACK') diff --git a/src/main/native-chat/agent-session-journal/journal-sql-batch.ts b/src/main/native-chat/agent-session-journal/journal-sql-batch.ts index ceda7b469fa..93f113d7e79 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-batch.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-batch.ts @@ -6,7 +6,6 @@ export type JournalSqlBatchOutcome = export type JournalSqlBatchResult = { outcomes: JournalSqlBatchOutcome[] - postcommit?: unknown } export class JournalSqlBatchError extends Error { @@ -33,7 +32,7 @@ export class JournalSqlBatch { /** Commands may await only microtasks: the worker owns the connection while one is open. */ async run( - commands: readonly { command: Command; normal: boolean }[], + commands: readonly Command[], apply: (command: Command) => Value | Promise ): Promise> { if (this.failure) { @@ -44,20 +43,13 @@ export class JournalSqlBatch { return { outcomes } } const drain = commands.slice() - const normal = drain.every((entry) => entry.normal) let began = false let committing = false let failed: JournalSqlBatchError | undefined - let postcommit: unknown try { - // Re-prove FULL before every drain, including after a failed PRAGMA from an earlier drain. - this.db.pragma('synchronous = FULL') - if (normal) { - this.db.pragma('synchronous = NORMAL') - } this.db.exec('BEGIN IMMEDIATE') began = true - for (const { command } of drain) { + for (const command of drain) { this.db.exec('SAVEPOINT journal_command') try { const value = await apply(command) @@ -88,20 +80,10 @@ export class JournalSqlBatch { if (committing || rollback) { this.failure = failed } - } finally { - if (normal && !this.db.isTransaction) { - try { - this.db.pragma('synchronous = FULL') - } catch (error) { - // The COMMIT answer remains true; the owner reports this fault after its acknowledgement. - postcommit = error - this.failure ??= error - } - } } if (failed) { throw failed } - return { outcomes, ...(postcommit ? { postcommit } : {}) } + return { outcomes } } } diff --git a/src/main/native-chat/agent-session-journal/journal-sql-command.test.ts b/src/main/native-chat/agent-session-journal/journal-sql-command.test.ts index 0b2034fd41a..3e34c0dae1c 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-command.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-command.test.ts @@ -71,13 +71,10 @@ it('commits a step prefix without leaking the rolled-back session or admitting i const answer = ( await batch.run( [ - { - command: command({ - kind: 'steps', - steps: [step('first'), step('refused'), step('suffix')] - }), - normal: false - } + command({ + kind: 'steps', + steps: [step('first'), step('refused'), step('suffix')] + }) ], (entry) => engine.apply(entry, 100) ) @@ -161,10 +158,7 @@ it('assigns revisions after preceding successes and rolls back failed command pr ] } const results = await batch.run( - [accepted, duplicate, command(item('second')), command(item('stale', 0))].map((entry) => ({ - command: entry, - normal: engine.streamed(entry) - })), + [accepted, duplicate, command(item('second')), command(item('stale', 0))], (entry) => engine.apply(entry, 100) ) expect(results.outcomes.map((result) => result.committed)).toEqual([true, false, true, false]) @@ -240,11 +234,8 @@ it('keeps exact saved records across command shells and publishes metadata witho memory = persistJournalOutcomeRows(oracle, memory, source, rows, 100, false, (state) => journalInputOrder(state).cards.ensure(state) ).adopt() - const result = ( - await batch.run([{ command: command(operation), normal: false }], (entry) => - engine.apply(entry, 100) - ) - ).outcomes[0]! + const result = (await batch.run([command(operation)], (entry) => engine.apply(entry, 100))) + .outcomes[0]! expect(result.committed).toBe(true) } const records = (connection: Database.Database) => @@ -254,9 +245,7 @@ it('keeps exact saved records across command shells and publishes metadata witho .all() expect(records(db)).toEqual(records(oracle)) const final = ( - await batch.run([{ command: command(item('after deletion')), normal: false }], (entry) => - engine.apply(entry, 100) - ) + await batch.run([command(item('after deletion'))], (entry) => engine.apply(entry, 100)) ).outcomes[0]! expect(final).toMatchObject({ committed: true, @@ -265,55 +254,6 @@ it('keeps exact saved records across command shells and publishes metadata witho expect(JSON.stringify(final)).not.toContain('after deletion') }) -it('chooses NORMAL only for proven text revisions and retains FULL for settlements and effects', async () => { - const { db, engine, batch } = fixture() - expect(engine.streamed(command(item('first')))).toBe(false) - await batch.run([{ command: command(item('first')), normal: false }], (entry) => - engine.apply(entry, 100) - ) - const revision = command(item('next')) - const textStep: JournalWriterStep = { - kind: 'resolved-item', - reservedBytes: 4096, - resolution: { - kind: 'text-snapshot', - identity, - body: { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'streamed' }] } - }, - options: { fence: 1, turnScope: { kind: 'thread' } } - } - const steps = command({ kind: 'steps', steps: [textStep] }) - db.pragma('query_only = ON') - try { - expect(engine.streamed(revision)).toBe(true) - expect(engine.streamed(steps)).toBe(true) - expect( - engine.streamed( - command({ - kind: 'steps', - steps: [{ ...textStep, options: { ...textStep.options, executionOwner: 2 } }] - }) - ) - ).toBe(false) - } finally { - db.pragma('query_only = OFF') - } - expect( - engine.streamed({ - ...revision, - effects: [{ kind: 'queue-delete', sessionId: 'session', messageIds: [] }] - }) - ).toBe(false) - expect(engine.streamed(command({ kind: 'queue-resume', fence: 1 }))).toBe(false) - let synchronous = -1 - await batch.run([{ command: steps, normal: engine.streamed(steps) }], (entry) => { - synchronous = Number(db.pragma('synchronous', { simple: true })) - return engine.apply(entry, 100) - }) - expect(synchronous).toBe(1) - expect(Number(db.pragma('synchronous', { simple: true }))).toBe(2) -}) - it('refuses stale ownership, changed epochs and log coverage even for an empty plan', async () => { const { db, engine, batch } = fixture() const noOp = command({ @@ -322,8 +262,7 @@ it('refuses stale ownership, changed epochs and log coverage even for an empty p onlyIfUnsettled: true }) const run = async (entry = noOp) => - (await batch.run([{ command: entry, normal: false }], (value) => engine.apply(value, 100))) - .outcomes[0]! + (await batch.run([entry], (value) => engine.apply(value, 100))).outcomes[0]! expect( await run({ ...noOp, @@ -354,8 +293,7 @@ it('refuses stale ownership, changed epochs and log coverage even for an empty p it('replaces the epoch inside one writer command, moving the logical cursor epoch', async () => { const { engine, batch } = fixture() const run = async (entry: JournalWriterCommand) => - (await batch.run([{ command: entry, normal: false }], (value) => engine.apply(value, 100))) - .outcomes[0]! + (await batch.run([entry], (value) => engine.apply(value, 100))).outcomes[0]! const before = await run(command(item('before'))) const replaced = await run( command({ diff --git a/src/main/native-chat/agent-session-journal/journal-sql-command.ts b/src/main/native-chat/agent-session-journal/journal-sql-command.ts index 58cee1a244f..69e79786c6b 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-command.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-command.ts @@ -5,11 +5,8 @@ import type { JournalWriterCommit } from './journal-writer-changes' import { applyJournalWriterEffect, type JournalWriterEffectResult } from './journal-writer-effects' import { JournalWriterSession } from './journal-writer-session' import { runJournalWriterOperation } from './journal-writer-operations' -import { planJournalWriterRows } from './journal-writer-plan' import type { JournalOperationReceipt } from './writing/journal-row-writer' import { runJournalSqlSteps } from './journal-sql-steps' -import { isJournalStreamedTextRevision } from './writing/journal-streamed-text-revision' -import { planJournalWriterStep } from './writing/journal-writer-step-plan' import { claimJournalSqlProjection, journalLogNeedsFounding, @@ -22,40 +19,6 @@ export class JournalSqlCommand { private readonly stateDirectory: string ) {} - /** Conservative preflight only; all authority and revision assignment are repeated in apply(). */ - streamed(command: JournalWriterCommand): boolean { - if (command.effects) { - return false - } - try { - const { state } = this.session(command, 0) - const { operation } = command - if (operation.kind === 'steps') { - return operation.steps.every((step) => { - if (step.kind === 'settlement' || step.kind === 'creation-fact') { - return false - } - const planned = planJournalWriterStep(state, step, () => {}) - return ( - planned === null || - planned.every((build) => - isJournalStreamedTextRevision(state, build(state.lastSequence + 1, 0)) - ) - ) - }) - } - if (operation.kind !== 'item') { - return false - } - return planJournalWriterRows(state, operation).every((build) => - isJournalStreamedTextRevision(state, build(state.lastSequence + 1, 0)) - ) - } catch { - // A stale/invalid command fails its own SAVEPOINT; it must not downgrade the batch. - return false - } - } - async apply(command: JournalWriterCommand, savedAt: number): Promise { if (!this.db.isTransaction) { throw new Error('journal SQL command requires its transaction') diff --git a/src/main/native-chat/agent-session-journal/journal-sql-input-command.test-fixture.ts b/src/main/native-chat/agent-session-journal/journal-sql-input-command.test-fixture.ts index 3f62a2fd18e..a92d94977fb 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-input-command.test-fixture.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-input-command.test-fixture.ts @@ -145,9 +145,7 @@ export async function sides() { facts: { identity, ...facts }, savedAt: NOW } - const [outcome] = ( - await batch.run([{ command, normal: false }], (entry) => engine.apply(entry, NOW)) - ).outcomes + const [outcome] = (await batch.run([command], (entry) => engine.apply(entry, NOW))).outcomes if (!outcome!.committed) { // Exactly what reaches the host through the worker port. throw restoreJournalSqlFailure(journalSqlFailure(outcome!.error)) diff --git a/src/main/native-chat/agent-session-journal/journal-sql-maintenance.ts b/src/main/native-chat/agent-session-journal/journal-sql-maintenance.ts index 96dbd247cbd..a9bb3fed38f 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-maintenance.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-maintenance.ts @@ -39,11 +39,6 @@ export type JournalSqlMaintenanceResult = | boolean | null -/** Each call is one writer batch; only the final publication needs FULL durability. */ -export function journalSqlMaintenanceIsNormal(operation: JournalSqlMaintenance): boolean { - return operation.kind !== 'conversion-finish' && operation.kind !== 'projection-state' -} - export function applyJournalSqlMaintenance( db: Database.Database, operation: JournalSqlMaintenance diff --git a/src/main/native-chat/agent-session-journal/journal-sql-message-commit.ts b/src/main/native-chat/agent-session-journal/journal-sql-message-commit.ts index 0a3bd212f90..885f50e4b6a 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-message-commit.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-message-commit.ts @@ -1,12 +1,7 @@ import type Database from '../../sqlite/sync-database' -import type { - JournalSqlBatch, - JournalSqlBatchOutcome, - JournalSqlBatchResult -} from './journal-sql-batch' +import type { JournalSqlBatch, JournalSqlBatchResult } from './journal-sql-batch' import { applyJournalSqlMessage, - JournalMessageNeedsFull, type JournalSqlMessage, type JournalSqlMessageCommit } from './journal-sql-message' @@ -20,7 +15,6 @@ type MessageCommand = { export type JournalSqlMessageJob = { command: MessageCommand bytes: number - normal: boolean } export type JournalSqlMessagesCommit = Omit & { @@ -32,47 +26,10 @@ export type JournalSqlMessagesCommit = Omit export function prepareJournalSqlMessages(command: MessageCommand): JournalSqlMessageJob { return { command, - bytes: Buffer.byteLength(JSON.stringify(command), 'utf8'), - normal: command.steps.length > 0 && command.steps.every((step) => step.kind === 'text-snapshot') + bytes: Buffer.byteLength(JSON.stringify(command), 'utf8') } } -/** Select one ready prefix; controls and repeated chats keep their serialization boundary. */ -export function journalSqlMessageBatchSize( - ready: readonly T[], - jobFor: (entry: T) => JournalSqlMessageJob | undefined, - at = 0 -): number { - const sessions = new Set() - let bytes = 0 - let events = 0 - let count = 0 - for (let next = at; next < ready.length; next++) { - const job = jobFor(ready[next]!) - if (!job) { - break - } - const size = Math.max(1, job.command.steps.length) - if ( - count && - (!job.normal || - sessions.has(job.command.target.sessionId) || - bytes + job.bytes > 1024 * 1024 || - events + size > 64) - ) { - break - } - count++ - bytes += job.bytes - events += size - sessions.add(job.command.target.sessionId) - if (!job.normal || bytes >= 1024 * 1024 || events >= 64) { - break - } - } - return count -} - /** Called by the existing writer admission lane; publication completes before the next drain. */ export async function commitJournalSqlMessages( db: Database.Database, @@ -80,48 +37,13 @@ export async function commitJournalSqlMessages( jobs: readonly JournalSqlMessageJob[], publish: (commit: JournalSqlMessagesCommit, index: number) => void ): Promise> { - const outcomes: JournalSqlBatchOutcome[] = [] - let at = 0 - while (at < jobs.length) { - const ready = jobs.slice(at, at + journalSqlMessageBatchSize(jobs, (job) => job, at)) - const result = await batch.run( - ready.map((job) => ({ command: job, normal: job.normal })), - (job) => applyMessageSteps(db, job) - ) - for (const [index, outcome] of result.outcomes.entries()) { - if (outcome.committed) { - // Payloads are owned by this just-committed result, never reloaded through a reader. - publish(outcome.value, at + index) - } + const result = await batch.run(jobs, (job) => applyMessageSteps(db, job)) + for (const [index, outcome] of result.outcomes.entries()) { + if (outcome.committed) { + publish(outcome.value, index) } - for (let index = 0; index < result.outcomes.length; index++) { - let outcome = result.outcomes[index]! - if ( - !outcome.committed && - outcome.error instanceof JournalMessageNeedsFull && - !result.postcommit - ) { - const job = { ...ready[index]!, normal: false } - const rerouted = await batch.run([{ command: job, normal: false }], (job) => - applyMessageSteps(db, job) - ) - outcome = rerouted.outcomes[0]! - if (outcome.committed) { - publish(outcome.value, at + index) - } - if (rerouted.postcommit) { - outcomes.push(outcome) - return { outcomes, postcommit: rerouted.postcommit } - } - } - outcomes.push(outcome) - } - if (result.postcommit) { - return { outcomes, postcommit: result.postcommit } - } - at += ready.length } - return { outcomes } + return result } function applyMessageSteps( @@ -146,7 +68,7 @@ function applyMessageSteps( for (const step of steps) { db.exec('SAVEPOINT journal_message_step') try { - const commit = applyJournalSqlMessage(db, target, step, savedAt, job.normal) + const commit = applyJournalSqlMessage(db, target, step, savedAt) db.exec('RELEASE journal_message_step') result.cursor = commit.cursor result.outcomes.push(commit.outcome) @@ -160,9 +82,6 @@ function applyMessageSteps( } } catch (error) { db.exec('ROLLBACK TO journal_message_step; RELEASE journal_message_step') - if (error instanceof JournalMessageNeedsFull) { - throw error - } result.failure = { message: error instanceof Error ? error.message : String(error) } break } diff --git a/src/main/native-chat/agent-session-journal/journal-sql-message.test.ts b/src/main/native-chat/agent-session-journal/journal-sql-message.test.ts index 896b01eb646..a48e4966df5 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-message.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-message.test.ts @@ -16,17 +16,14 @@ afterEach(() => { } }) -it('reroutes a first snapshot to FULL and publishes exact committed revisions in order', async () => { +it('publishes exact committed snapshots and their final body in order', async () => { const { db, batch, commit, item, published } = fixture() - db.exec(`CREATE TEMP TRIGGER durable_first BEFORE INSERT ON journal_items - WHEN (SELECT synchronous FROM pragma_synchronous)<>2 BEGIN SELECT RAISE(ABORT,'creation was not FULL'); END`) expect((await commit(message('first'))).outcomes[0]).toMatchObject({ committed: true, value: { cursor: { epoch: 'epoch', sequence: 11 } } }) - db.exec('DROP TRIGGER durable_first') const result = await commitJournalSqlMessages( db, batch, @@ -42,9 +39,6 @@ it('reroutes a first snapshot to FULL and publishes exact committed revisions in { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'third' }] } ]) expect(item('orca:assistant')).toMatchObject({ revision: 3, sequence: 11, observedAt: 100 }) - db.exec(`CREATE TEMP TRIGGER durable_message_end BEFORE UPDATE ON journal_items - WHEN (SELECT synchronous FROM pragma_synchronous)<>2 AND json_extract(NEW.payload_json,'$.body.completedAt') IS NOT NULL - BEGIN SELECT RAISE(ABORT,'message end was not FULL'); END`) const ended = message('third') ended.body.completedAt = 300 expect((await commit(ended)).outcomes[0]).toMatchObject({ @@ -54,7 +48,6 @@ it('reroutes a first snapshot to FULL and publishes exact committed revisions in items: [{ body: { completedAt: 300 } }] } }) - expect(db.pragma('user_version', { simple: true })).toBe(4) }) it('commits ready revisions from different chats atomically before publishing either payload', async () => { @@ -132,8 +125,6 @@ it('accepts an optimistic echo once, preserving its local body and moved placeme `INSERT INTO journal_card_versions(session_id,epoch,message_id,version_sequence,version_index,current,state,command_turn_item_id,payload_json) VALUES('chat','epoch','card',1,0,1,'returned','orca:reported-command',?)` ).run(JSON.stringify(card)) - db.exec(`CREATE TEMP TRIGGER durable_echo BEFORE UPDATE ON journal_submission_attempts - WHEN (SELECT synchronous FROM pragma_synchronous)<>2 BEGIN SELECT RAISE(ABORT,'acceptance was not FULL'); END`) const input = message('original', 'unused', 'item') input.identity = { provider: 'codex', threadId: 'thread', turnId: 'turn', ordinal: 0 } input.body.role = 'user' diff --git a/src/main/native-chat/agent-session-journal/journal-sql-message.ts b/src/main/native-chat/agent-session-journal/journal-sql-message.ts index a2873ac764c..21a0d0dbd23 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-message.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-message.ts @@ -1,4 +1,3 @@ -import { isDeepStrictEqual } from 'node:util' import type Database from '../../sqlite/sync-database' import { agentJournalItemKey, @@ -28,7 +27,6 @@ import { import { journalRenderItem } from './projection/journal-render-item' import { resolveJournalGuardedItem } from './writing/journal-timeline-resolution' import { journalRowBase } from './writing/journal-row-builders' -import { isJournalTextRevision } from './writing/journal-streamed-text-revision' import { assertJournalFence } from './writing/journal-write-guards' import type { JournalItemRow } from './durable-rows/journal-row-schema' import type { QueuedMessageRow } from './queued-messages/queued-message-table' @@ -60,15 +58,12 @@ export type JournalSqlMessageCommit = { editDeadline?: number | null } -export class JournalMessageNeedsFull extends Error {} - /** Runs only on the owning writer connection, inside its command SAVEPOINT. */ export function applyJournalSqlMessage( db: Database.Database, target: { sessionId: string; epoch: string }, input: JournalSqlMessage, - savedAt: number, - normal: boolean + savedAt: number ): JournalSqlMessageCommit { if (!db.isTransaction) { throw new Error('journal message requires its writer transaction') @@ -186,22 +181,6 @@ export function applyJournalSqlMessage( JOIN journal_items AS item ON item.session_id=card.session_id AND item.epoch=card.epoch AND item.item_id=card.command_turn_item_id WHERE card.session_id=? AND card.epoch=? AND card.current=1 AND card.state='returned'`) .all(...prefix) - if ( - normal && - !( - input.kind === 'text-snapshot' && - !echo && - !declaration && - returned.length === 0 && - isJournalTextRevision(row, previous) && - row.ownerFence === stored?.owner_fence && - (row.executionOwner === undefined || row.executionOwner === stored?.execution_owner) && - isDeepStrictEqual(scope, previous?.turnScope) && - isDeepStrictEqual(agentJournalLinkageFields(row), agentJournalLinkageFields(previous)) - ) - ) { - throw new JournalMessageNeedsFull() - } const result: JournalSqlMessageCommit = { cursor: { epoch, sequence }, outcome: { kind: 'committed', wrote: true }, diff --git a/src/main/native-chat/agent-session-journal/journal-sql-open.test.ts b/src/main/native-chat/agent-session-journal/journal-sql-open.test.ts index e8d0b53bbcc..c68143f1c16 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-open.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-open.test.ts @@ -50,9 +50,7 @@ function fixture() { operation, facts: { identity: { ...identity, sessionId } } } - const [outcome] = ( - await batch.run([{ command, normal: false }], (entry) => engine.apply(entry, 100)) - ).outcomes + const [outcome] = (await batch.run([command], (entry) => engine.apply(entry, 100))).outcomes if (!outcome!.committed) { throw outcome!.error } diff --git a/src/main/native-chat/agent-session-journal/journal-sql-owner.ts b/src/main/native-chat/agent-session-journal/journal-sql-owner.ts index e181dffa1c2..d0deef79b8a 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-owner.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-owner.ts @@ -6,6 +6,7 @@ import { } from '../../worker-threads/worker-thread-entry-path' import type { JournalSqlRequest, JournalSqlResponse } from './journal-sql-protocol' import { restoreJournalSqlFailure } from './journal-sql-failure' +import { prepareJournalSqlRequest } from './journal-sql-ready-batch' import type { JournalWriterCommand } from './journal-writer-command' import type { JournalWriterCommit } from './journal-writer-changes' import type { JournalWriterEffect, JournalWriterEffectResult } from './journal-writer-effects' @@ -130,7 +131,7 @@ export class JournalSqlOwner { !(command.type === 'maintenance' && command.operation.kind === 'projection-state') }) try { - this.worker.postMessage({ id, command } satisfies JournalSqlRequest) + this.worker.postMessage(prepareJournalSqlRequest(id, command)) } catch (error) { this.pending.delete(id) reject(error) @@ -170,9 +171,6 @@ export class JournalSqlOwner { return } pending?.resolve(response) - if ('postcommit' in response && response.postcommit) { - this.fail(restoreJournalSqlFailure(response.postcommit)) - } } private fail(error: Error): void { diff --git a/src/main/native-chat/agent-session-journal/journal-sql-protocol.ts b/src/main/native-chat/agent-session-journal/journal-sql-protocol.ts index f356752a942..019e7e90558 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-protocol.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-protocol.ts @@ -7,6 +7,7 @@ import type { JournalSqlMessageJob, JournalSqlMessagesCommit } from './journal-s export type JournalSqlRequest = { id: number + batch?: { sessionId: string; steps: number; bytes: number } command: | { type: 'write'; command: JournalWriterCommand } | { type: 'messages'; job: JournalSqlMessageJob } @@ -17,20 +18,18 @@ export type JournalSqlRequest = { export type JournalSqlResponse = | { type: 'ready'; readOnly: boolean } - | { type: 'committed'; id: number; commit: JournalWriterCommit; postcommit?: JournalSqlFailure } + | { type: 'committed'; id: number; commit: JournalWriterCommit } | { type: 'messages'; id: number; commit: JournalSqlMessagesCommit } | { type: 'effects' id: number results: JournalWriterEffectResult[] - postcommit?: JournalSqlFailure } | { type: 'done'; id: number } | { type: 'maintenance' id: number result: JournalSqlMaintenanceResult - postcommit?: JournalSqlFailure } | { type: 'failed' diff --git a/src/main/native-chat/agent-session-journal/journal-sql-ready-batch.ts b/src/main/native-chat/agent-session-journal/journal-sql-ready-batch.ts index 16905b68c74..9de1c3c3744 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-ready-batch.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-ready-batch.ts @@ -1,53 +1,87 @@ +import type { AgentJournalMessageItem } from '../../../shared/agent-session/journal/agent-session-journal-types' import type { JournalSqlRequest } from './journal-sql-protocol' -import type { JournalWriterCommand } from './journal-writer-command' -import { journalSqlMaintenanceIsNormal } from './journal-sql-maintenance' -const MAX_EVENTS = 64 +const MAX_STEPS = 64 const MAX_BYTES = 1024 * 1024 -/** Control commands keep their own boundary; an oversized atomic command runs alone. */ -export function takeJournalSqlReadyBatch( - queue: JournalSqlRequest[], - streamed: (command: JournalWriterCommand) => boolean -): { request: JournalSqlRequest; normal: boolean }[] { - const batch: { request: JournalSqlRequest; normal: boolean }[] = [] - let events = 0 +/** Sender work only: classification never opens a saved projection. */ +export function prepareJournalSqlRequest( + id: number, + command: JournalSqlRequest['command'] +): JournalSqlRequest { + let batch: JournalSqlRequest['batch'] + if ( + command.type === 'messages' && + command.job.command.steps.length > 0 && + command.job.command.steps.every( + (step) => step.kind === 'text-snapshot' && streamingText(step.body) + ) + ) { + batch = { + sessionId: command.job.command.target.sessionId, + steps: command.job.command.steps.length, + bytes: command.job.bytes + } + } else if (command.type === 'write' && !command.command.effects) { + const { operation } = command.command + if ( + operation.kind === 'steps' && + operation.steps.length > 0 && + operation.steps.every( + (step) => + step.kind === 'resolved-item' && + step.resolution.kind === 'text-snapshot' && + streamingText(step.resolution.body) + ) + ) { + batch = { + sessionId: command.command.target.sessionId, + steps: operation.steps.length, + bytes: Buffer.byteLength(JSON.stringify(command.command), 'utf8') + } + } + } + return { id, command, ...(batch ? { batch } : {}) } +} + +/** Controls, repeated chats and an oversized atomic command keep their own boundary. */ +export function takeJournalSqlReadyBatch(queue: JournalSqlRequest[]): JournalSqlRequest[] { + const first = queue[0] + if (!first?.batch) { + return queue.splice(0, 1) + } + const sessions = new Set() + let count = 0 + let steps = 0 let bytes = 0 for (const request of queue) { - const { command } = request - const normal = command.type === 'write' && streamed(command.command) - if (!normal) { - if (batch.length === 0) { - batch.push({ - request, - normal: command.type === 'maintenance' && journalSqlMaintenanceIsNormal(command.operation) - }) - } + const next = request.batch + if ( + !next || + request.command.type !== first.command.type || + (count > 0 && + (sessions.has(next.sessionId) || + steps + next.steps > MAX_STEPS || + bytes + next.bytes > MAX_BYTES)) + ) { break } - const count = - command.command.operation.kind === 'steps' - ? Math.max(1, command.command.operation.steps.length) - : 1 - let size: number - try { - size = Buffer.byteLength(JSON.stringify(command.command), 'utf8') - } catch { - if (batch.length === 0) { - batch.push({ request, normal: false }) - } - break - } - if (batch.length > 0 && (events + count > MAX_EVENTS || bytes + size > MAX_BYTES)) { - break - } - batch.push({ request, normal: true }) - events += count - bytes += size - if (events >= MAX_EVENTS || bytes >= MAX_BYTES) { + count++ + steps += next.steps + bytes += next.bytes + sessions.add(next.sessionId) + if (steps >= MAX_STEPS || bytes >= MAX_BYTES) { break } } - queue.splice(0, batch.length) - return batch + return queue.splice(0, count) +} + +function streamingText(body: AgentJournalMessageItem): boolean { + return ( + (body.role === 'assistant' || body.role === 'reasoning') && + body.completedAt === undefined && + body.state === undefined && + body.blocks.every((block) => block.type === 'text') + ) } diff --git a/src/main/native-chat/agent-session-journal/journal-sql-reopen-oracle.test.ts b/src/main/native-chat/agent-session-journal/journal-sql-reopen-oracle.test.ts index 383507fd3be..7ec8b8fbfd6 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-reopen-oracle.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-reopen-oracle.test.ts @@ -155,12 +155,9 @@ it.each([ await new JournalSqlBatch(persisted.db).run( [ { - command: { - target: { sessionId: 'session', epoch: 'epoch', ownerToken: `open-${index}` }, - operation: { kind: 'open' as const }, - facts: { identity } - }, - normal: false + target: { sessionId: 'session', epoch: 'epoch', ownerToken: `open-${index}` }, + operation: { kind: 'open' as const }, + facts: { identity } } ], (entry) => engine.apply(entry, 100) diff --git a/src/main/native-chat/agent-session-journal/journal-sql-worker-entry.ts b/src/main/native-chat/agent-session-journal/journal-sql-worker-entry.ts index ebd413952aa..8fe608fe61c 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-worker-entry.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-worker-entry.ts @@ -14,7 +14,7 @@ import { assertJournalWritable } from './writing/journal-write-guards' import { applyJournalSqlMaintenance } from './journal-sql-maintenance' import { initializeJournalSqlProjection } from './journal-sql-schema-migration' import { migrateJournalDomainSchema } from './journal-domain-schema' -import { commitJournalSqlMessages, journalSqlMessageBatchSize } from './journal-sql-message-commit' +import { commitJournalSqlMessages } from './journal-sql-message-commit' if ( !parentPort || @@ -94,24 +94,19 @@ type MessageRequest = { /** Bound ready work before opening a transaction, then yield after its acknowledgements. */ async function drain(): Promise { - if (queue[0]?.command.type === 'messages') { - const count = journalSqlMessageBatchSize(queue, ({ command }) => - command.type === 'messages' ? command.job : undefined - ) + const pending = takeJournalSqlReadyBatch(queue) + if (pending[0]?.command.type === 'messages') { await drainMessages( - queue - .splice(0, count) - .filter((request): request is MessageRequest => request.command.type === 'messages') + pending.filter((request): request is MessageRequest => request.command.type === 'messages') ) scheduleNextDrain() return } - const pending = takeJournalSqlReadyBatch(queue, (command) => commands?.streamed(command) === true) const writes = pending.filter( - (entry): entry is { request: WriteRequest; normal: boolean } => - entry.request.command.type !== 'close' && entry.request.command.type !== 'messages' + (request): request is WriteRequest => + request.command.type !== 'close' && request.command.type !== 'messages' ) - const close = pending.find(({ request }) => request.command.type === 'close')?.request + const close = pending.find((request) => request.command.type === 'close') const database = db if (writes.length > 0 && database && batch && commands) { const executor = commands @@ -122,32 +117,25 @@ async function drain(): Promise { throw fault } assertJournalWritable(readOnly, 'database') - const result = await batch.run( - writes.map(({ request, normal }) => ({ - command: request, - normal - })), - async (request) => { - const { command, id } = request - if (command.type === 'maintenance') { - return { - type: 'maintenance', - id, - result: applyJournalSqlMaintenance(database, command.operation) - } + const result = await batch.run(writes, async (request) => { + const { command, id } = request + if (command.type === 'maintenance') { + return { + type: 'maintenance', + id, + result: applyJournalSqlMaintenance(database, command.operation) } - return command.type === 'write' - ? { type: 'committed', id, commit: await executor.apply(command.command, Date.now()) } - : { - type: 'effects', - id, - results: command.effects.map((effect) => - applyJournalWriterEffect(database, stateDirectory, effect) - ) - } } - ) - postcommit = result.postcommit ? journalSqlFailure(result.postcommit) : undefined + return command.type === 'write' + ? { type: 'committed', id, commit: await executor.apply(command.command, Date.now()) } + : { + type: 'effects', + id, + results: command.effects.map((effect) => + applyJournalWriterEffect(database, stateDirectory, effect) + ) + } + }) try { checkpoint?.afterCommit() } catch (error) { @@ -159,18 +147,18 @@ async function drain(): Promise { ? outcome.value : { type: 'failed', - id: writes[index]!.request.id, + id: writes[index]!.id, outcome: 'rolled-back', error: journalSqlFailure(outcome.error) } ) if (postcommit) { - fault = result.postcommit ?? postcommit + fault = postcommit } } catch (error) { const outcome = error instanceof JournalSqlBatchError ? error.outcome : 'not-dispatched' fault ??= batch.fault - responses = writes.map(({ request }) => ({ + responses = writes.map((request) => ({ type: 'failed', id: request.id, outcome, @@ -240,7 +228,7 @@ async function drainMessages(requests: readonly MessageRequest[]): Promise acknowledged.add(id) } } - let postcommit = result.postcommit + let postcommit: unknown try { checkpoint?.afterCommit() } catch (error) { diff --git a/src/main/native-chat/agent-session-journal/journal-writer-session.ts b/src/main/native-chat/agent-session-journal/journal-writer-session.ts index accd7f80fc1..130699fe2c5 100644 --- a/src/main/native-chat/agent-session-journal/journal-writer-session.ts +++ b/src/main/native-chat/agent-session-journal/journal-writer-session.ts @@ -77,7 +77,7 @@ export class JournalWriterSession { db, readOnly: false, stateDirectory, - transaction: (run, _streamed, committed) => this.savepoint(run, committed) + transaction: (run, committed) => this.savepoint(run, committed) } const held = queuedMessageEditLeaseHolds(facts.editLeases ?? []) this.collaborators = createJournalStoreCollaborators({ diff --git a/src/main/native-chat/agent-session-journal/store-lifecycle/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/store-lifecycle/journal-store-collaborators.ts index 7ce17d47571..d7bae90d633 100644 --- a/src/main/native-chat/agent-session-journal/store-lifecycle/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/store-lifecycle/journal-store-collaborators.ts @@ -31,10 +31,7 @@ import { import { claimAgentSessionAttachmentsInTransaction } from '../../agent-session-attachments/agent-session-attachment-claims' import { JournalGoalAttachmentInverse } from '../writing/journal-goal-attachment-inverse' import { JournalEpochReplacements } from '../context-epochs/journal-epoch-replacements' -import { - isJournalOutputRevision, - isJournalStreamedTextRevision -} from '../writing/journal-streamed-text-revision' +import { isJournalOutputRevision } from '../writing/journal-output-revision' export type JournalStoreHost = { /** Notify readers of durable draft-only transactions that append no journal row. */ @@ -97,7 +94,6 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal prepare: host.prepare, commit: host.commit, project: (rows) => projectJournalRows(host.state(), rows), - streamedTextRevision: (row) => isJournalStreamedTextRevision(host.state(), row), outputRevision: (row) => isJournalOutputRevision(host.state(), row) }) const inputReceipt = (rows: readonly JournalRow[], receipt?: JournalOperationReceipt) => diff --git a/src/main/native-chat/agent-session-journal/writing/journal-streamed-text-revision.ts b/src/main/native-chat/agent-session-journal/writing/journal-output-revision.ts similarity index 61% rename from src/main/native-chat/agent-session-journal/writing/journal-streamed-text-revision.ts rename to src/main/native-chat/agent-session-journal/writing/journal-output-revision.ts index 917f68790d6..57e747ca995 100644 --- a/src/main/native-chat/agent-session-journal/writing/journal-streamed-text-revision.ts +++ b/src/main/native-chat/agent-session-journal/writing/journal-output-revision.ts @@ -2,48 +2,6 @@ import { isDeepStrictEqual } from 'node:util' import { agentJournalLinkageFields } from '../../../../shared/agent-session/journal/agent-session-journal-producer' import type { JournalReducerState } from '../projection/journal-reducer' import type { JournalRow } from '../durable-rows/journal-row-schema' -import type { AgentJournalRenderItem } from '../../../../shared/agent-session/journal/agent-session-journal-types' - -/** Only streamed assistant text may skip per-commit fsync; FULL turn settlement makes it durable. */ -export function isJournalStreamedTextRevision( - state: JournalReducerState, - row: JournalRow -): boolean { - if (row.kind !== 'item' || row.recovered) { - return false - } - const previous = state.items.get(row.itemId) - return isJournalTextRevision(row, previous) && sameRevisionPlacement(state, row, previous!) -} - -export function isJournalTextRevision( - row: Extract, - previous: AgentJournalRenderItem | undefined -): boolean { - if (row.recovered) { - return false - } - const { body } = row - if ( - body.kind !== 'message' || - body.role !== 'assistant' || - !body.blocks.every((block) => block.type === 'text') || - body.sentAs !== undefined || - body.command !== undefined || - body.from !== undefined || - previous?.body.kind !== 'message' || - previous.body.role !== 'assistant' || - previous.body.sentAs !== undefined || - previous.body.command !== undefined || - previous.body.from !== undefined || - !previous.body.blocks.every((block) => block.type === 'text') - ) { - return false - } - const { blocks: _oldText, ...before } = previous.body - const { blocks: _newText, ...after } = body - return isDeepStrictEqual(before, after) -} /** Output revisions cannot change queue eligibility; lifecycle and unknown changes still wake it. */ export function isJournalOutputRevision(state: JournalReducerState, row: JournalRow): boolean { diff --git a/src/main/native-chat/agent-session-journal/writing/journal-row-writer.ts b/src/main/native-chat/agent-session-journal/writing/journal-row-writer.ts index 9097f3b43c4..fcb394c22db 100644 --- a/src/main/native-chat/agent-session-journal/writing/journal-row-writer.ts +++ b/src/main/native-chat/agent-session-journal/writing/journal-row-writer.ts @@ -102,8 +102,6 @@ export type JournalRowWriterDeps = { finishFold?: (draft: JournalReducerState) => void ) => () => void commit: (rows: readonly JournalRow[], savedAt: number, inputChanged?: boolean) => void - /** Streamed-text-only revisions may skip per-commit fsync; settlement makes them durable. */ - streamedTextRevision?: (row: JournalRow) => boolean outputRevision?: (row: JournalRow) => boolean } @@ -120,7 +118,6 @@ export class JournalRowWriter { const row = build(this.deps.nextSequence(), savedAt) assertJournalFence(row.fence, this.deps.highestFence()) const receipt = typeof prepareReceipt === 'function' ? prepareReceipt([row]) : prepareReceipt - const streamed = !receipt && this.streamed([row]) const inputChanged = Boolean(receipt) || !this.outputOnly([row]) this.deps.database().transaction( (db) => { @@ -131,7 +128,6 @@ export class JournalRowWriter { receipt?.write(db, this.project([row])[0]) return adopt }, - streamed, (adopt) => adopt?.() ) receipt?.committed() @@ -166,7 +162,6 @@ export class JournalRowWriter { assertJournalFence(row.fence, this.deps.highestFence()) } const receipt = typeof prepareReceipt === 'function' ? prepareReceipt(rows) : prepareReceipt - const streamed = !receipt && this.streamed(rows) const inputChanged = Boolean(receipt) || !this.outputOnly(rows) this.deps.database().transaction( (db) => { @@ -179,7 +174,6 @@ export class JournalRowWriter { receipt?.write(db, receiptRow?.(this.project(rows))) return adopt }, - streamed, (adopt) => adopt?.() ) receipt?.committed() @@ -205,9 +199,4 @@ export class JournalRowWriter { private outputOnly(rows: readonly JournalRow[]): boolean { return this.deps.outputRevision !== undefined && rows.every(this.deps.outputRevision) } - - private streamed(rows: readonly JournalRow[]): boolean { - const { streamedTextRevision } = this.deps - return streamedTextRevision !== undefined && rows.every((row) => streamedTextRevision(row)) - } } diff --git a/src/main/native-chat/agent-session-journal/writing/journal-streamed-revision-contract-test-fixture.ts b/src/main/native-chat/agent-session-journal/writing/journal-streamed-revision-contract-test-fixture.ts deleted file mode 100644 index a320ee06325..00000000000 --- a/src/main/native-chat/agent-session-journal/writing/journal-streamed-revision-contract-test-fixture.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { mkdtemp, rm, mkdir, writeFile } from 'node:fs/promises' -import { randomUUID } from 'node:crypto' -import { tmpdir } from 'node:os' -import { join, dirname } from 'node:path' -import { afterEach, beforeEach, vi } from 'vitest' -import type { AgentJournalMessageItem } from '../../../../shared/agent-session/journal/agent-session-journal-types' -import { journalPragmaNumber } from '../database/journal-database' -import { - closeTestJournalHostDatabases, - createTrackedJournalOpener, - openTestJournalHostDatabase -} from '../store-lifecycle/journal-host-database-test-support' -import { formatNativeChatFileReference } from '../../../../shared/agent-image-paste' -import * as attachmentReferences from '../../agent-session-attachments/agent-session-attachment-references' - -export const journals = createTrackedJournalOpener({ legacyInputs: true }) - -let root: string - -beforeEach(async () => { - root = await mkdtemp(join(tmpdir(), 'orca-streamed-revision-')) -}) - -afterEach(async () => { - vi.restoreAllMocks() - await journals.closeAll() - await closeTestJournalHostDatabases() - await rm(root, { recursive: true, force: true }) -}) - -export const text = (role: 'assistant' | 'user', value: string): AgentJournalMessageItem => ({ - kind: 'message', - role, - blocks: [{ type: 'text', text: value }] -}) - -export function openJournal(sessionId = 'stream') { - return journals.open({ - stateDirectory: root, - identity: { - sessionId, - workspaceId: 'folder', - hostId: 'local', - agent: 'grok', - providerHandle: null - } - }) -} - -export function commitLevels() { - const { db } = openTestJournalHostDatabase(root) - const levels: number[] = [] - const exec = db.exec.bind(db) - vi.spyOn(db, 'exec').mockImplementation((sql) => { - if (sql === 'COMMIT') { - levels.push(journalPragmaNumber(db, 'synchronous')) - } - exec(sql) - }) - return levels -} - -export async function uploadedInput() { - const id = randomUUID() - const path = join(attachmentReferences.agentSessionAttachmentStoreRoot(root), id, 'source.txt') - await mkdir(dirname(path), { recursive: true }) - await writeFile(path, 'retained input') - return { id, body: text('user', formatNativeChatFileReference(path)) } -} - -export const fixture = { - get root() { - return root - }, - set root(value: typeof root) { - root = value - } -}