mirror of
https://github.com/stablyai/orca.git
synced 2026-11-19 04:23:57 +00:00
refactor: use one chat durability mode and ready batch selector
This commit is contained in:
+14
-18
@@ -83,24 +83,20 @@ it('rolls back the entire Codex exit settlement when a later row fails', async (
|
||||
const diskBefore = liveTestJournalRows(database.db, SESSION)
|
||||
const sawEarlierInserts: number[] = []
|
||||
const transaction = database.transaction.bind(database)
|
||||
vi.spyOn(database, 'transaction').mockImplementation((run, streamed, committed) =>
|
||||
transaction(
|
||||
(db) => {
|
||||
try {
|
||||
return run(db)
|
||||
} catch (error) {
|
||||
sawEarlierInserts.push(liveTestJournalRows(db, SESSION).length - diskBefore.length)
|
||||
expect(journal.snapshot()).toEqual(before)
|
||||
expect(commits).not.toHaveBeenCalled()
|
||||
expect(
|
||||
published.every((snapshot) => JSON.stringify(snapshot) === JSON.stringify(before))
|
||||
).toBe(true)
|
||||
throw error
|
||||
}
|
||||
},
|
||||
streamed,
|
||||
committed
|
||||
)
|
||||
vi.spyOn(database, 'transaction').mockImplementation((run, committed) =>
|
||||
transaction((db) => {
|
||||
try {
|
||||
return run(db)
|
||||
} catch (error) {
|
||||
sawEarlierInserts.push(liveTestJournalRows(db, SESSION).length - diskBefore.length)
|
||||
expect(journal.snapshot()).toEqual(before)
|
||||
expect(commits).not.toHaveBeenCalled()
|
||||
expect(
|
||||
published.every((snapshot) => JSON.stringify(snapshot) === JSON.stringify(before))
|
||||
).toBe(true)
|
||||
throw error
|
||||
}
|
||||
}, committed)
|
||||
)
|
||||
database.db.exec(`CREATE TEMP TRIGGER reject_codex_later BEFORE INSERT ON journal_rows
|
||||
WHEN NEW.session_id = '${SESSION}' AND NEW.seq > ${before.cursor.sequence + 1}
|
||||
|
||||
@@ -74,7 +74,6 @@ export function publishNewEpoch(input: {
|
||||
publishJournalSessionEpoch(db, input.identity, input.epoch)
|
||||
return publishJournalOutcomeEpoch(db, sessionId, state.epoch, rows)
|
||||
},
|
||||
false,
|
||||
(publication) => {
|
||||
// Adopt, then finish the nonthrowing queue bookkeeping, before the fallible publication.
|
||||
const state = publication.adopt()
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ try {
|
||||
if (db.prepare('PRAGMA journal_mode').get().journal_mode !== 'wal') {
|
||||
throw new Error('chat journal is not in WAL mode');
|
||||
}
|
||||
db.exec('PRAGMA busy_timeout = 0; PRAGMA synchronous = FULL; PRAGMA checkpoint_fullfsync = ON;');
|
||||
db.exec('PRAGMA busy_timeout = 0; PRAGMA synchronous = NORMAL; PRAGMA checkpoint_fullfsync = ON;');
|
||||
db.exec('PRAGMA journal_size_limit = ' + workerData.sizeLimit);
|
||||
const frames = db.prepare('PRAGMA wal_checkpoint(NOOP)');
|
||||
const checkpoint = db.prepare('PRAGMA wal_checkpoint(PASSIVE)');
|
||||
|
||||
@@ -16,7 +16,7 @@ import { ensureAgentSessionAttachmentClaimTables } from '../../agent-session-att
|
||||
|
||||
export const JOURNAL_BUSY_TIMEOUT_MS = 5000
|
||||
export const JOURNAL_SIZE_LIMIT_BYTES = 32 * 1024 * 1024
|
||||
const JOURNAL_SYNCHRONOUS = 'FULL'
|
||||
const JOURNAL_SYNCHRONOUS = 'NORMAL'
|
||||
|
||||
export type OpenJournalDatabase = {
|
||||
db: Database.Database
|
||||
@@ -97,7 +97,7 @@ function configureJournalPragmas(db: Database.Database, stored: number): void {
|
||||
db.pragma('journal_mode = WAL')
|
||||
db.pragma(`busy_timeout = ${JOURNAL_BUSY_TIMEOUT_MS}`)
|
||||
db.pragma('foreign_keys = ON')
|
||||
// FULL fsyncs submissions before dispatch; checkpoint_fullfsync also flushes macOS drive caches.
|
||||
// WAL checkpoints flush committed pages and macOS drive caches.
|
||||
db.pragma(`synchronous = ${JOURNAL_SYNCHRONOUS}`)
|
||||
db.pragma('checkpoint_fullfsync = ON')
|
||||
db.pragma(`journal_size_limit = ${JOURNAL_SIZE_LIMIT_BYTES}`)
|
||||
|
||||
@@ -72,35 +72,13 @@ export class JournalHostDatabase {
|
||||
return readJournalSessionEpoch(this.db, sessionId)
|
||||
}
|
||||
|
||||
/** Streamed text may lose only its trailing suffix on OS crash; the next FULL commit makes it durable. */
|
||||
transaction<T>(
|
||||
run: (db: Database.Database) => T,
|
||||
streamed = false,
|
||||
committed?: (result: T) => void
|
||||
): T {
|
||||
transaction<T>(run: (db: Database.Database) => T, committed?: (result: T) => void): T {
|
||||
const db = this.db
|
||||
this.transactionNumber += 1
|
||||
if (!streamed) {
|
||||
const result = runJournalTransaction(db, run, () => {
|
||||
this.stranded = true
|
||||
})
|
||||
committed?.(result)
|
||||
this.checkpoint?.afterCommit()
|
||||
return result
|
||||
}
|
||||
// Synchronous level changes require no transaction; stranded recovery restores FULL first.
|
||||
db.pragma('synchronous = NORMAL')
|
||||
let result: T
|
||||
try {
|
||||
result = runJournalTransaction(db, run, () => {
|
||||
this.stranded = true
|
||||
})
|
||||
committed?.(result)
|
||||
} finally {
|
||||
if (!db.isTransaction) {
|
||||
db.pragma('synchronous = FULL')
|
||||
}
|
||||
}
|
||||
const result = runJournalTransaction(db, run, () => {
|
||||
this.stranded = true
|
||||
})
|
||||
committed?.(result)
|
||||
this.checkpoint?.afterCommit()
|
||||
return result
|
||||
}
|
||||
@@ -139,7 +117,6 @@ export class JournalHostDatabase {
|
||||
throw journalOpenRefusalError(error)
|
||||
}
|
||||
}
|
||||
connection.pragma('synchronous = FULL')
|
||||
this.stranded = false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import Database from '../../sqlite/sync-database'
|
||||
import { JournalSqlBatch, JournalSqlBatchError } from './journal-sql-batch'
|
||||
import { takeJournalSqlReadyBatch } from './journal-sql-ready-batch'
|
||||
import { prepareJournalSqlRequest, takeJournalSqlReadyBatch } from './journal-sql-ready-batch'
|
||||
import { prepareJournalSqlMessages } from './journal-sql-message-commit'
|
||||
import { message } from './journal-sql-message.test-fixture'
|
||||
import type { JournalSqlRequest } from './journal-sql-protocol'
|
||||
|
||||
const databases: Database.Database[] = []
|
||||
@@ -22,21 +24,14 @@ function fixture() {
|
||||
it('rolls back a failed command while later commands see preceding successful writes', async () => {
|
||||
const { db, batch } = fixture()
|
||||
const failure = new Error('command failed')
|
||||
const answer = await batch.run(
|
||||
[
|
||||
{ command: 1, normal: true },
|
||||
{ command: 2, normal: true },
|
||||
{ command: 3, normal: false }
|
||||
],
|
||||
(id) => {
|
||||
const previous = db.prepare('SELECT id FROM writes ORDER BY id').all()
|
||||
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written')
|
||||
if (id === 2) {
|
||||
throw failure
|
||||
}
|
||||
return previous
|
||||
const answer = await batch.run([1, 2, 3], (id) => {
|
||||
const previous = db.prepare('SELECT id FROM writes ORDER BY id').all()
|
||||
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written')
|
||||
if (id === 2) {
|
||||
throw failure
|
||||
}
|
||||
)
|
||||
return previous
|
||||
})
|
||||
expect(answer).toEqual({
|
||||
outcomes: [
|
||||
{ committed: true, value: [] },
|
||||
@@ -51,38 +46,19 @@ it('rolls back a failed command while later commands see preceding successful wr
|
||||
expect(db.isTransaction).toBe(false)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ normal: [true, true], synchronous: 1 },
|
||||
{ normal: [true, false], synchronous: 2 }
|
||||
])('uses batch durability $synchronous and restores FULL before the next batch', async (input) => {
|
||||
it('leaves newly admitted work for a new drain', async () => {
|
||||
const { db, batch } = fixture()
|
||||
const answer = await batch.run(
|
||||
input.normal.map((normal, id) => ({ command: id, normal })),
|
||||
(id) => {
|
||||
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written')
|
||||
return db.pragma('synchronous', { simple: true })
|
||||
}
|
||||
)
|
||||
expect(answer.outcomes).toEqual([
|
||||
{ committed: true, value: input.synchronous },
|
||||
{ committed: true, value: input.synchronous }
|
||||
])
|
||||
expect(db.pragma('synchronous', { simple: true })).toBe(2)
|
||||
})
|
||||
|
||||
it('leaves newly admitted work for a new drain with its own durability', async () => {
|
||||
const { db, batch } = fixture()
|
||||
const admitted = [{ command: 1, normal: true }]
|
||||
const admitted = [1]
|
||||
const first = await batch.run(admitted, (id) => {
|
||||
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'stream')
|
||||
admitted.push({ command: 2, normal: false })
|
||||
admitted.push(2)
|
||||
return id
|
||||
})
|
||||
expect(first.outcomes).toEqual([{ committed: true, value: 1 }])
|
||||
expect(db.prepare('SELECT id, value FROM writes').all()).toEqual([{ id: 1, value: 'stream' }])
|
||||
const send = await batch.run(admitted.slice(1), (id) => {
|
||||
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'send')
|
||||
return db.pragma('synchronous', { simple: true })
|
||||
return id
|
||||
})
|
||||
expect(send.outcomes).toEqual([{ committed: true, value: 2 }])
|
||||
expect(db.prepare('SELECT id, value FROM writes ORDER BY id').all()).toEqual([
|
||||
@@ -91,43 +67,39 @@ it('leaves newly admitted work for a new drain with its own durability', async (
|
||||
])
|
||||
})
|
||||
|
||||
it('commits a control receipt before later streamed work with its own durability', async () => {
|
||||
it('keeps bounded snapshot prefixes and controls in enqueue order', async () => {
|
||||
const { db, batch } = fixture()
|
||||
const stream = (id: number): JournalSqlRequest => ({
|
||||
id,
|
||||
command: {
|
||||
type: 'write',
|
||||
command: {
|
||||
target: { sessionId: 'session', epoch: 'epoch', ownerToken: 'owner' },
|
||||
operation: {
|
||||
kind: 'item',
|
||||
identity: { provider: 'orca', clientMessageId: 'output' },
|
||||
body: { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'output' }] },
|
||||
options: { fence: 1, turnScope: { kind: 'thread' } }
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
const stream = (id: number, steps: number, text = 'output'): JournalSqlRequest =>
|
||||
prepareJournalSqlRequest(id, {
|
||||
type: 'messages',
|
||||
job: prepareJournalSqlMessages({
|
||||
target: { sessionId: `session${id}`, epoch: 'epoch' },
|
||||
steps: Array.from({ length: steps }, () => message(text)),
|
||||
savedAt: 100
|
||||
})
|
||||
})
|
||||
const queue: JournalSqlRequest[] = [
|
||||
stream(1),
|
||||
{ id: 2, command: { type: 'effects', effects: [] } },
|
||||
stream(3)
|
||||
stream(1, 32),
|
||||
stream(2, 32),
|
||||
stream(3, 1),
|
||||
{ id: 4, command: { type: 'effects', effects: [] } },
|
||||
stream(5, 1, 'x'.repeat(700 * 1024)),
|
||||
stream(6, 1, 'x'.repeat(700 * 1024))
|
||||
]
|
||||
const commit = async () =>
|
||||
batch.run(
|
||||
takeJournalSqlReadyBatch(queue, () => true).map(({ request, normal }) => ({
|
||||
command: request,
|
||||
normal
|
||||
})),
|
||||
(request) => {
|
||||
db.prepare('INSERT INTO writes VALUES (?, ?)').run(request.id, 'written')
|
||||
return { id: request.id, durability: db.pragma('synchronous', { simple: true }) }
|
||||
}
|
||||
)
|
||||
expect((await commit()).outcomes).toEqual([{ committed: true, value: { id: 1, durability: 1 } }])
|
||||
expect((await commit()).outcomes).toEqual([{ committed: true, value: { id: 2, durability: 2 } }])
|
||||
batch.run(takeJournalSqlReadyBatch(queue), (request) => {
|
||||
db.prepare('INSERT INTO writes VALUES (?, ?)').run(request.id, 'written')
|
||||
return request.id
|
||||
})
|
||||
expect((await commit()).outcomes).toEqual([
|
||||
{ committed: true, value: 1 },
|
||||
{ committed: true, value: 2 }
|
||||
])
|
||||
expect(db.prepare('SELECT id FROM writes ORDER BY id').all()).toEqual([{ id: 1 }, { id: 2 }])
|
||||
expect((await commit()).outcomes).toEqual([{ committed: true, value: { id: 3, durability: 1 } }])
|
||||
expect((await commit()).outcomes).toEqual([{ committed: true, value: 3 }])
|
||||
expect((await commit()).outcomes).toEqual([{ committed: true, value: 4 }])
|
||||
expect((await commit()).outcomes).toEqual([{ committed: true, value: 5 }])
|
||||
expect((await commit()).outcomes).toEqual([{ committed: true, value: 6 }])
|
||||
})
|
||||
|
||||
it.each([false, true])(
|
||||
@@ -146,7 +118,7 @@ it.each([false, true])(
|
||||
})
|
||||
let failure: unknown
|
||||
try {
|
||||
await batch.run([{ command: 1, normal: false }], (id) => {
|
||||
await batch.run([1], (id) => {
|
||||
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written')
|
||||
return id
|
||||
})
|
||||
@@ -156,34 +128,13 @@ it.each([false, true])(
|
||||
expect(failure).toBeInstanceOf(JournalSqlBatchError)
|
||||
expect(failure).toMatchObject({ outcome: 'indeterminate' })
|
||||
expect(db.prepare('SELECT id FROM writes').all()).toEqual(committed ? [{ id: 1 }] : [])
|
||||
await expect(batch.run([{ command: 2, normal: false }], (id) => id)).rejects.toThrow(
|
||||
await expect(batch.run([2], (id) => id)).rejects.toThrow(
|
||||
expect.objectContaining({ outcome: 'not-dispatched' })
|
||||
)
|
||||
expect(db.prepare('SELECT id FROM writes').all()).toEqual(committed ? [{ id: 1 }] : [])
|
||||
}
|
||||
)
|
||||
|
||||
it('acknowledges a committed stream even when restoring FULL fails afterwards', async () => {
|
||||
const { db, batch } = fixture()
|
||||
const pragma = db.pragma.bind(db)
|
||||
let committed = false
|
||||
const failure = new Error('FULL setting failed')
|
||||
vi.spyOn(db, 'pragma').mockImplementation((sql, options) => {
|
||||
if (sql === 'synchronous = FULL' && committed) {
|
||||
throw failure
|
||||
}
|
||||
return pragma(sql, options)
|
||||
})
|
||||
const answer = await batch.run([{ command: 1, normal: true }], (id) => {
|
||||
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'written')
|
||||
committed = true
|
||||
return id
|
||||
})
|
||||
expect(answer).toEqual({ outcomes: [{ committed: true, value: 1 }], postcommit: failure })
|
||||
expect(db.prepare('SELECT id FROM writes').all()).toEqual([{ id: 1 }])
|
||||
expect(batch.fault).toBe(failure)
|
||||
})
|
||||
|
||||
it('does not claim rollback when both command and outer rollback fail', async () => {
|
||||
const { db, batch } = fixture()
|
||||
const exec = db.exec.bind(db)
|
||||
@@ -195,13 +146,13 @@ it('does not claim rollback when both command and outer rollback fail', async ()
|
||||
exec(sql)
|
||||
})
|
||||
await expect(
|
||||
batch.run([{ command: 1, normal: false }], (id) => {
|
||||
batch.run([1], (id) => {
|
||||
db.prepare('INSERT INTO writes VALUES (?, ?)').run(id, 'uncommitted')
|
||||
throw new Error('command failed')
|
||||
})
|
||||
).rejects.toThrow(expect.objectContaining({ outcome: 'indeterminate', rollback }))
|
||||
expect(db.isTransaction).toBe(true)
|
||||
await expect(batch.run([{ command: 2, normal: false }], (id) => id)).rejects.toThrow(
|
||||
await expect(batch.run([2], (id) => id)).rejects.toThrow(
|
||||
expect.objectContaining({ outcome: 'not-dispatched' })
|
||||
)
|
||||
exec('ROLLBACK')
|
||||
|
||||
@@ -6,7 +6,6 @@ export type JournalSqlBatchOutcome<Value> =
|
||||
|
||||
export type JournalSqlBatchResult<Value> = {
|
||||
outcomes: JournalSqlBatchOutcome<Value>[]
|
||||
postcommit?: unknown
|
||||
}
|
||||
|
||||
export class JournalSqlBatchError extends Error {
|
||||
@@ -33,7 +32,7 @@ export class JournalSqlBatch {
|
||||
|
||||
/** Commands may await only microtasks: the worker owns the connection while one is open. */
|
||||
async run<Command, Value>(
|
||||
commands: readonly { command: Command; normal: boolean }[],
|
||||
commands: readonly Command[],
|
||||
apply: (command: Command) => Value | Promise<Value>
|
||||
): Promise<JournalSqlBatchResult<Value>> {
|
||||
if (this.failure) {
|
||||
@@ -44,20 +43,13 @@ export class JournalSqlBatch {
|
||||
return { outcomes }
|
||||
}
|
||||
const drain = commands.slice()
|
||||
const normal = drain.every((entry) => entry.normal)
|
||||
let began = false
|
||||
let committing = false
|
||||
let failed: JournalSqlBatchError | undefined
|
||||
let postcommit: unknown
|
||||
try {
|
||||
// Re-prove FULL before every drain, including after a failed PRAGMA from an earlier drain.
|
||||
this.db.pragma('synchronous = FULL')
|
||||
if (normal) {
|
||||
this.db.pragma('synchronous = NORMAL')
|
||||
}
|
||||
this.db.exec('BEGIN IMMEDIATE')
|
||||
began = true
|
||||
for (const { command } of drain) {
|
||||
for (const command of drain) {
|
||||
this.db.exec('SAVEPOINT journal_command')
|
||||
try {
|
||||
const value = await apply(command)
|
||||
@@ -88,20 +80,10 @@ export class JournalSqlBatch {
|
||||
if (committing || rollback) {
|
||||
this.failure = failed
|
||||
}
|
||||
} finally {
|
||||
if (normal && !this.db.isTransaction) {
|
||||
try {
|
||||
this.db.pragma('synchronous = FULL')
|
||||
} catch (error) {
|
||||
// The COMMIT answer remains true; the owner reports this fault after its acknowledgement.
|
||||
postcommit = error
|
||||
this.failure ??= error
|
||||
}
|
||||
}
|
||||
}
|
||||
if (failed) {
|
||||
throw failed
|
||||
}
|
||||
return { outcomes, ...(postcommit ? { postcommit } : {}) }
|
||||
return { outcomes }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,13 +71,10 @@ it('commits a step prefix without leaking the rolled-back session or admitting i
|
||||
const answer = (
|
||||
await batch.run(
|
||||
[
|
||||
{
|
||||
command: command({
|
||||
kind: 'steps',
|
||||
steps: [step('first'), step('refused'), step('suffix')]
|
||||
}),
|
||||
normal: false
|
||||
}
|
||||
command({
|
||||
kind: 'steps',
|
||||
steps: [step('first'), step('refused'), step('suffix')]
|
||||
})
|
||||
],
|
||||
(entry) => engine.apply(entry, 100)
|
||||
)
|
||||
@@ -161,10 +158,7 @@ it('assigns revisions after preceding successes and rolls back failed command pr
|
||||
]
|
||||
}
|
||||
const results = await batch.run(
|
||||
[accepted, duplicate, command(item('second')), command(item('stale', 0))].map((entry) => ({
|
||||
command: entry,
|
||||
normal: engine.streamed(entry)
|
||||
})),
|
||||
[accepted, duplicate, command(item('second')), command(item('stale', 0))],
|
||||
(entry) => engine.apply(entry, 100)
|
||||
)
|
||||
expect(results.outcomes.map((result) => result.committed)).toEqual([true, false, true, false])
|
||||
@@ -240,11 +234,8 @@ it('keeps exact saved records across command shells and publishes metadata witho
|
||||
memory = persistJournalOutcomeRows(oracle, memory, source, rows, 100, false, (state) =>
|
||||
journalInputOrder(state).cards.ensure(state)
|
||||
).adopt()
|
||||
const result = (
|
||||
await batch.run([{ command: command(operation), normal: false }], (entry) =>
|
||||
engine.apply(entry, 100)
|
||||
)
|
||||
).outcomes[0]!
|
||||
const result = (await batch.run([command(operation)], (entry) => engine.apply(entry, 100)))
|
||||
.outcomes[0]!
|
||||
expect(result.committed).toBe(true)
|
||||
}
|
||||
const records = (connection: Database.Database) =>
|
||||
@@ -254,9 +245,7 @@ it('keeps exact saved records across command shells and publishes metadata witho
|
||||
.all()
|
||||
expect(records(db)).toEqual(records(oracle))
|
||||
const final = (
|
||||
await batch.run([{ command: command(item('after deletion')), normal: false }], (entry) =>
|
||||
engine.apply(entry, 100)
|
||||
)
|
||||
await batch.run([command(item('after deletion'))], (entry) => engine.apply(entry, 100))
|
||||
).outcomes[0]!
|
||||
expect(final).toMatchObject({
|
||||
committed: true,
|
||||
@@ -265,55 +254,6 @@ it('keeps exact saved records across command shells and publishes metadata witho
|
||||
expect(JSON.stringify(final)).not.toContain('after deletion')
|
||||
})
|
||||
|
||||
it('chooses NORMAL only for proven text revisions and retains FULL for settlements and effects', async () => {
|
||||
const { db, engine, batch } = fixture()
|
||||
expect(engine.streamed(command(item('first')))).toBe(false)
|
||||
await batch.run([{ command: command(item('first')), normal: false }], (entry) =>
|
||||
engine.apply(entry, 100)
|
||||
)
|
||||
const revision = command(item('next'))
|
||||
const textStep: JournalWriterStep = {
|
||||
kind: 'resolved-item',
|
||||
reservedBytes: 4096,
|
||||
resolution: {
|
||||
kind: 'text-snapshot',
|
||||
identity,
|
||||
body: { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'streamed' }] }
|
||||
},
|
||||
options: { fence: 1, turnScope: { kind: 'thread' } }
|
||||
}
|
||||
const steps = command({ kind: 'steps', steps: [textStep] })
|
||||
db.pragma('query_only = ON')
|
||||
try {
|
||||
expect(engine.streamed(revision)).toBe(true)
|
||||
expect(engine.streamed(steps)).toBe(true)
|
||||
expect(
|
||||
engine.streamed(
|
||||
command({
|
||||
kind: 'steps',
|
||||
steps: [{ ...textStep, options: { ...textStep.options, executionOwner: 2 } }]
|
||||
})
|
||||
)
|
||||
).toBe(false)
|
||||
} finally {
|
||||
db.pragma('query_only = OFF')
|
||||
}
|
||||
expect(
|
||||
engine.streamed({
|
||||
...revision,
|
||||
effects: [{ kind: 'queue-delete', sessionId: 'session', messageIds: [] }]
|
||||
})
|
||||
).toBe(false)
|
||||
expect(engine.streamed(command({ kind: 'queue-resume', fence: 1 }))).toBe(false)
|
||||
let synchronous = -1
|
||||
await batch.run([{ command: steps, normal: engine.streamed(steps) }], (entry) => {
|
||||
synchronous = Number(db.pragma('synchronous', { simple: true }))
|
||||
return engine.apply(entry, 100)
|
||||
})
|
||||
expect(synchronous).toBe(1)
|
||||
expect(Number(db.pragma('synchronous', { simple: true }))).toBe(2)
|
||||
})
|
||||
|
||||
it('refuses stale ownership, changed epochs and log coverage even for an empty plan', async () => {
|
||||
const { db, engine, batch } = fixture()
|
||||
const noOp = command({
|
||||
@@ -322,8 +262,7 @@ it('refuses stale ownership, changed epochs and log coverage even for an empty p
|
||||
onlyIfUnsettled: true
|
||||
})
|
||||
const run = async (entry = noOp) =>
|
||||
(await batch.run([{ command: entry, normal: false }], (value) => engine.apply(value, 100)))
|
||||
.outcomes[0]!
|
||||
(await batch.run([entry], (value) => engine.apply(value, 100))).outcomes[0]!
|
||||
expect(
|
||||
await run({
|
||||
...noOp,
|
||||
@@ -354,8 +293,7 @@ it('refuses stale ownership, changed epochs and log coverage even for an empty p
|
||||
it('replaces the epoch inside one writer command, moving the logical cursor epoch', async () => {
|
||||
const { engine, batch } = fixture()
|
||||
const run = async (entry: JournalWriterCommand) =>
|
||||
(await batch.run([{ command: entry, normal: false }], (value) => engine.apply(value, 100)))
|
||||
.outcomes[0]!
|
||||
(await batch.run([entry], (value) => engine.apply(value, 100))).outcomes[0]!
|
||||
const before = await run(command(item('before')))
|
||||
const replaced = await run(
|
||||
command({
|
||||
|
||||
@@ -5,11 +5,8 @@ import type { JournalWriterCommit } from './journal-writer-changes'
|
||||
import { applyJournalWriterEffect, type JournalWriterEffectResult } from './journal-writer-effects'
|
||||
import { JournalWriterSession } from './journal-writer-session'
|
||||
import { runJournalWriterOperation } from './journal-writer-operations'
|
||||
import { planJournalWriterRows } from './journal-writer-plan'
|
||||
import type { JournalOperationReceipt } from './writing/journal-row-writer'
|
||||
import { runJournalSqlSteps } from './journal-sql-steps'
|
||||
import { isJournalStreamedTextRevision } from './writing/journal-streamed-text-revision'
|
||||
import { planJournalWriterStep } from './writing/journal-writer-step-plan'
|
||||
import {
|
||||
claimJournalSqlProjection,
|
||||
journalLogNeedsFounding,
|
||||
@@ -22,40 +19,6 @@ export class JournalSqlCommand {
|
||||
private readonly stateDirectory: string
|
||||
) {}
|
||||
|
||||
/** Conservative preflight only; all authority and revision assignment are repeated in apply(). */
|
||||
streamed(command: JournalWriterCommand): boolean {
|
||||
if (command.effects) {
|
||||
return false
|
||||
}
|
||||
try {
|
||||
const { state } = this.session(command, 0)
|
||||
const { operation } = command
|
||||
if (operation.kind === 'steps') {
|
||||
return operation.steps.every((step) => {
|
||||
if (step.kind === 'settlement' || step.kind === 'creation-fact') {
|
||||
return false
|
||||
}
|
||||
const planned = planJournalWriterStep(state, step, () => {})
|
||||
return (
|
||||
planned === null ||
|
||||
planned.every((build) =>
|
||||
isJournalStreamedTextRevision(state, build(state.lastSequence + 1, 0))
|
||||
)
|
||||
)
|
||||
})
|
||||
}
|
||||
if (operation.kind !== 'item') {
|
||||
return false
|
||||
}
|
||||
return planJournalWriterRows(state, operation).every((build) =>
|
||||
isJournalStreamedTextRevision(state, build(state.lastSequence + 1, 0))
|
||||
)
|
||||
} catch {
|
||||
// A stale/invalid command fails its own SAVEPOINT; it must not downgrade the batch.
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async apply(command: JournalWriterCommand, savedAt: number): Promise<JournalWriterCommit> {
|
||||
if (!this.db.isTransaction) {
|
||||
throw new Error('journal SQL command requires its transaction')
|
||||
|
||||
+1
-3
@@ -145,9 +145,7 @@ export async function sides() {
|
||||
facts: { identity, ...facts },
|
||||
savedAt: NOW
|
||||
}
|
||||
const [outcome] = (
|
||||
await batch.run([{ command, normal: false }], (entry) => engine.apply(entry, NOW))
|
||||
).outcomes
|
||||
const [outcome] = (await batch.run([command], (entry) => engine.apply(entry, NOW))).outcomes
|
||||
if (!outcome!.committed) {
|
||||
// Exactly what reaches the host through the worker port.
|
||||
throw restoreJournalSqlFailure(journalSqlFailure(outcome!.error))
|
||||
|
||||
@@ -39,11 +39,6 @@ export type JournalSqlMaintenanceResult =
|
||||
| boolean
|
||||
| null
|
||||
|
||||
/** Each call is one writer batch; only the final publication needs FULL durability. */
|
||||
export function journalSqlMaintenanceIsNormal(operation: JournalSqlMaintenance): boolean {
|
||||
return operation.kind !== 'conversion-finish' && operation.kind !== 'projection-state'
|
||||
}
|
||||
|
||||
export function applyJournalSqlMaintenance(
|
||||
db: Database.Database,
|
||||
operation: JournalSqlMaintenance
|
||||
|
||||
@@ -1,12 +1,7 @@
|
||||
import type Database from '../../sqlite/sync-database'
|
||||
import type {
|
||||
JournalSqlBatch,
|
||||
JournalSqlBatchOutcome,
|
||||
JournalSqlBatchResult
|
||||
} from './journal-sql-batch'
|
||||
import type { JournalSqlBatch, JournalSqlBatchResult } from './journal-sql-batch'
|
||||
import {
|
||||
applyJournalSqlMessage,
|
||||
JournalMessageNeedsFull,
|
||||
type JournalSqlMessage,
|
||||
type JournalSqlMessageCommit
|
||||
} from './journal-sql-message'
|
||||
@@ -20,7 +15,6 @@ type MessageCommand = {
|
||||
export type JournalSqlMessageJob = {
|
||||
command: MessageCommand
|
||||
bytes: number
|
||||
normal: boolean
|
||||
}
|
||||
|
||||
export type JournalSqlMessagesCommit = Omit<JournalSqlMessageCommit, 'outcome'> & {
|
||||
@@ -32,47 +26,10 @@ export type JournalSqlMessagesCommit = Omit<JournalSqlMessageCommit, 'outcome'>
|
||||
export function prepareJournalSqlMessages(command: MessageCommand): JournalSqlMessageJob {
|
||||
return {
|
||||
command,
|
||||
bytes: Buffer.byteLength(JSON.stringify(command), 'utf8'),
|
||||
normal: command.steps.length > 0 && command.steps.every((step) => step.kind === 'text-snapshot')
|
||||
bytes: Buffer.byteLength(JSON.stringify(command), 'utf8')
|
||||
}
|
||||
}
|
||||
|
||||
/** Select one ready prefix; controls and repeated chats keep their serialization boundary. */
|
||||
export function journalSqlMessageBatchSize<T>(
|
||||
ready: readonly T[],
|
||||
jobFor: (entry: T) => JournalSqlMessageJob | undefined,
|
||||
at = 0
|
||||
): number {
|
||||
const sessions = new Set<string>()
|
||||
let bytes = 0
|
||||
let events = 0
|
||||
let count = 0
|
||||
for (let next = at; next < ready.length; next++) {
|
||||
const job = jobFor(ready[next]!)
|
||||
if (!job) {
|
||||
break
|
||||
}
|
||||
const size = Math.max(1, job.command.steps.length)
|
||||
if (
|
||||
count &&
|
||||
(!job.normal ||
|
||||
sessions.has(job.command.target.sessionId) ||
|
||||
bytes + job.bytes > 1024 * 1024 ||
|
||||
events + size > 64)
|
||||
) {
|
||||
break
|
||||
}
|
||||
count++
|
||||
bytes += job.bytes
|
||||
events += size
|
||||
sessions.add(job.command.target.sessionId)
|
||||
if (!job.normal || bytes >= 1024 * 1024 || events >= 64) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
/** Called by the existing writer admission lane; publication completes before the next drain. */
|
||||
export async function commitJournalSqlMessages(
|
||||
db: Database.Database,
|
||||
@@ -80,48 +37,13 @@ export async function commitJournalSqlMessages(
|
||||
jobs: readonly JournalSqlMessageJob[],
|
||||
publish: (commit: JournalSqlMessagesCommit, index: number) => void
|
||||
): Promise<JournalSqlBatchResult<JournalSqlMessagesCommit>> {
|
||||
const outcomes: JournalSqlBatchOutcome<JournalSqlMessagesCommit>[] = []
|
||||
let at = 0
|
||||
while (at < jobs.length) {
|
||||
const ready = jobs.slice(at, at + journalSqlMessageBatchSize(jobs, (job) => job, at))
|
||||
const result = await batch.run(
|
||||
ready.map((job) => ({ command: job, normal: job.normal })),
|
||||
(job) => applyMessageSteps(db, job)
|
||||
)
|
||||
for (const [index, outcome] of result.outcomes.entries()) {
|
||||
if (outcome.committed) {
|
||||
// Payloads are owned by this just-committed result, never reloaded through a reader.
|
||||
publish(outcome.value, at + index)
|
||||
}
|
||||
const result = await batch.run(jobs, (job) => applyMessageSteps(db, job))
|
||||
for (const [index, outcome] of result.outcomes.entries()) {
|
||||
if (outcome.committed) {
|
||||
publish(outcome.value, index)
|
||||
}
|
||||
for (let index = 0; index < result.outcomes.length; index++) {
|
||||
let outcome = result.outcomes[index]!
|
||||
if (
|
||||
!outcome.committed &&
|
||||
outcome.error instanceof JournalMessageNeedsFull &&
|
||||
!result.postcommit
|
||||
) {
|
||||
const job = { ...ready[index]!, normal: false }
|
||||
const rerouted = await batch.run([{ command: job, normal: false }], (job) =>
|
||||
applyMessageSteps(db, job)
|
||||
)
|
||||
outcome = rerouted.outcomes[0]!
|
||||
if (outcome.committed) {
|
||||
publish(outcome.value, at + index)
|
||||
}
|
||||
if (rerouted.postcommit) {
|
||||
outcomes.push(outcome)
|
||||
return { outcomes, postcommit: rerouted.postcommit }
|
||||
}
|
||||
}
|
||||
outcomes.push(outcome)
|
||||
}
|
||||
if (result.postcommit) {
|
||||
return { outcomes, postcommit: result.postcommit }
|
||||
}
|
||||
at += ready.length
|
||||
}
|
||||
return { outcomes }
|
||||
return result
|
||||
}
|
||||
|
||||
function applyMessageSteps(
|
||||
@@ -146,7 +68,7 @@ function applyMessageSteps(
|
||||
for (const step of steps) {
|
||||
db.exec('SAVEPOINT journal_message_step')
|
||||
try {
|
||||
const commit = applyJournalSqlMessage(db, target, step, savedAt, job.normal)
|
||||
const commit = applyJournalSqlMessage(db, target, step, savedAt)
|
||||
db.exec('RELEASE journal_message_step')
|
||||
result.cursor = commit.cursor
|
||||
result.outcomes.push(commit.outcome)
|
||||
@@ -160,9 +82,6 @@ function applyMessageSteps(
|
||||
}
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK TO journal_message_step; RELEASE journal_message_step')
|
||||
if (error instanceof JournalMessageNeedsFull) {
|
||||
throw error
|
||||
}
|
||||
result.failure = { message: error instanceof Error ? error.message : String(error) }
|
||||
break
|
||||
}
|
||||
|
||||
@@ -16,17 +16,14 @@ afterEach(() => {
|
||||
}
|
||||
})
|
||||
|
||||
it('reroutes a first snapshot to FULL and publishes exact committed revisions in order', async () => {
|
||||
it('publishes exact committed snapshots and their final body in order', async () => {
|
||||
const { db, batch, commit, item, published } = fixture()
|
||||
db.exec(`CREATE TEMP TRIGGER durable_first BEFORE INSERT ON journal_items
|
||||
WHEN (SELECT synchronous FROM pragma_synchronous)<>2 BEGIN SELECT RAISE(ABORT,'creation was not FULL'); END`)
|
||||
expect((await commit(message('first'))).outcomes[0]).toMatchObject({
|
||||
committed: true,
|
||||
value: {
|
||||
cursor: { epoch: 'epoch', sequence: 11 }
|
||||
}
|
||||
})
|
||||
db.exec('DROP TRIGGER durable_first')
|
||||
const result = await commitJournalSqlMessages(
|
||||
db,
|
||||
batch,
|
||||
@@ -42,9 +39,6 @@ it('reroutes a first snapshot to FULL and publishes exact committed revisions in
|
||||
{ kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'third' }] }
|
||||
])
|
||||
expect(item('orca:assistant')).toMatchObject({ revision: 3, sequence: 11, observedAt: 100 })
|
||||
db.exec(`CREATE TEMP TRIGGER durable_message_end BEFORE UPDATE ON journal_items
|
||||
WHEN (SELECT synchronous FROM pragma_synchronous)<>2 AND json_extract(NEW.payload_json,'$.body.completedAt') IS NOT NULL
|
||||
BEGIN SELECT RAISE(ABORT,'message end was not FULL'); END`)
|
||||
const ended = message('third')
|
||||
ended.body.completedAt = 300
|
||||
expect((await commit(ended)).outcomes[0]).toMatchObject({
|
||||
@@ -54,7 +48,6 @@ it('reroutes a first snapshot to FULL and publishes exact committed revisions in
|
||||
items: [{ body: { completedAt: 300 } }]
|
||||
}
|
||||
})
|
||||
expect(db.pragma('user_version', { simple: true })).toBe(4)
|
||||
})
|
||||
|
||||
it('commits ready revisions from different chats atomically before publishing either payload', async () => {
|
||||
@@ -132,8 +125,6 @@ it('accepts an optimistic echo once, preserving its local body and moved placeme
|
||||
`INSERT INTO journal_card_versions(session_id,epoch,message_id,version_sequence,version_index,current,state,command_turn_item_id,payload_json)
|
||||
VALUES('chat','epoch','card',1,0,1,'returned','orca:reported-command',?)`
|
||||
).run(JSON.stringify(card))
|
||||
db.exec(`CREATE TEMP TRIGGER durable_echo BEFORE UPDATE ON journal_submission_attempts
|
||||
WHEN (SELECT synchronous FROM pragma_synchronous)<>2 BEGIN SELECT RAISE(ABORT,'acceptance was not FULL'); END`)
|
||||
const input = message('original', 'unused', 'item')
|
||||
input.identity = { provider: 'codex', threadId: 'thread', turnId: 'turn', ordinal: 0 }
|
||||
input.body.role = 'user'
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { isDeepStrictEqual } from 'node:util'
|
||||
import type Database from '../../sqlite/sync-database'
|
||||
import {
|
||||
agentJournalItemKey,
|
||||
@@ -28,7 +27,6 @@ import {
|
||||
import { journalRenderItem } from './projection/journal-render-item'
|
||||
import { resolveJournalGuardedItem } from './writing/journal-timeline-resolution'
|
||||
import { journalRowBase } from './writing/journal-row-builders'
|
||||
import { isJournalTextRevision } from './writing/journal-streamed-text-revision'
|
||||
import { assertJournalFence } from './writing/journal-write-guards'
|
||||
import type { JournalItemRow } from './durable-rows/journal-row-schema'
|
||||
import type { QueuedMessageRow } from './queued-messages/queued-message-table'
|
||||
@@ -60,15 +58,12 @@ export type JournalSqlMessageCommit = {
|
||||
editDeadline?: number | null
|
||||
}
|
||||
|
||||
export class JournalMessageNeedsFull extends Error {}
|
||||
|
||||
/** Runs only on the owning writer connection, inside its command SAVEPOINT. */
|
||||
export function applyJournalSqlMessage(
|
||||
db: Database.Database,
|
||||
target: { sessionId: string; epoch: string },
|
||||
input: JournalSqlMessage,
|
||||
savedAt: number,
|
||||
normal: boolean
|
||||
savedAt: number
|
||||
): JournalSqlMessageCommit {
|
||||
if (!db.isTransaction) {
|
||||
throw new Error('journal message requires its writer transaction')
|
||||
@@ -186,22 +181,6 @@ export function applyJournalSqlMessage(
|
||||
JOIN journal_items AS item ON item.session_id=card.session_id AND item.epoch=card.epoch AND item.item_id=card.command_turn_item_id
|
||||
WHERE card.session_id=? AND card.epoch=? AND card.current=1 AND card.state='returned'`)
|
||||
.all(...prefix)
|
||||
if (
|
||||
normal &&
|
||||
!(
|
||||
input.kind === 'text-snapshot' &&
|
||||
!echo &&
|
||||
!declaration &&
|
||||
returned.length === 0 &&
|
||||
isJournalTextRevision(row, previous) &&
|
||||
row.ownerFence === stored?.owner_fence &&
|
||||
(row.executionOwner === undefined || row.executionOwner === stored?.execution_owner) &&
|
||||
isDeepStrictEqual(scope, previous?.turnScope) &&
|
||||
isDeepStrictEqual(agentJournalLinkageFields(row), agentJournalLinkageFields(previous))
|
||||
)
|
||||
) {
|
||||
throw new JournalMessageNeedsFull()
|
||||
}
|
||||
const result: JournalSqlMessageCommit = {
|
||||
cursor: { epoch, sequence },
|
||||
outcome: { kind: 'committed', wrote: true },
|
||||
|
||||
@@ -50,9 +50,7 @@ function fixture() {
|
||||
operation,
|
||||
facts: { identity: { ...identity, sessionId } }
|
||||
}
|
||||
const [outcome] = (
|
||||
await batch.run([{ command, normal: false }], (entry) => engine.apply(entry, 100))
|
||||
).outcomes
|
||||
const [outcome] = (await batch.run([command], (entry) => engine.apply(entry, 100))).outcomes
|
||||
if (!outcome!.committed) {
|
||||
throw outcome!.error
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
} from '../../worker-threads/worker-thread-entry-path'
|
||||
import type { JournalSqlRequest, JournalSqlResponse } from './journal-sql-protocol'
|
||||
import { restoreJournalSqlFailure } from './journal-sql-failure'
|
||||
import { prepareJournalSqlRequest } from './journal-sql-ready-batch'
|
||||
import type { JournalWriterCommand } from './journal-writer-command'
|
||||
import type { JournalWriterCommit } from './journal-writer-changes'
|
||||
import type { JournalWriterEffect, JournalWriterEffectResult } from './journal-writer-effects'
|
||||
@@ -130,7 +131,7 @@ export class JournalSqlOwner {
|
||||
!(command.type === 'maintenance' && command.operation.kind === 'projection-state')
|
||||
})
|
||||
try {
|
||||
this.worker.postMessage({ id, command } satisfies JournalSqlRequest)
|
||||
this.worker.postMessage(prepareJournalSqlRequest(id, command))
|
||||
} catch (error) {
|
||||
this.pending.delete(id)
|
||||
reject(error)
|
||||
@@ -170,9 +171,6 @@ export class JournalSqlOwner {
|
||||
return
|
||||
}
|
||||
pending?.resolve(response)
|
||||
if ('postcommit' in response && response.postcommit) {
|
||||
this.fail(restoreJournalSqlFailure(response.postcommit))
|
||||
}
|
||||
}
|
||||
|
||||
private fail(error: Error): void {
|
||||
|
||||
@@ -7,6 +7,7 @@ import type { JournalSqlMessageJob, JournalSqlMessagesCommit } from './journal-s
|
||||
|
||||
export type JournalSqlRequest = {
|
||||
id: number
|
||||
batch?: { sessionId: string; steps: number; bytes: number }
|
||||
command:
|
||||
| { type: 'write'; command: JournalWriterCommand }
|
||||
| { type: 'messages'; job: JournalSqlMessageJob }
|
||||
@@ -17,20 +18,18 @@ export type JournalSqlRequest = {
|
||||
|
||||
export type JournalSqlResponse =
|
||||
| { type: 'ready'; readOnly: boolean }
|
||||
| { type: 'committed'; id: number; commit: JournalWriterCommit; postcommit?: JournalSqlFailure }
|
||||
| { type: 'committed'; id: number; commit: JournalWriterCommit }
|
||||
| { type: 'messages'; id: number; commit: JournalSqlMessagesCommit }
|
||||
| {
|
||||
type: 'effects'
|
||||
id: number
|
||||
results: JournalWriterEffectResult[]
|
||||
postcommit?: JournalSqlFailure
|
||||
}
|
||||
| { type: 'done'; id: number }
|
||||
| {
|
||||
type: 'maintenance'
|
||||
id: number
|
||||
result: JournalSqlMaintenanceResult
|
||||
postcommit?: JournalSqlFailure
|
||||
}
|
||||
| {
|
||||
type: 'failed'
|
||||
|
||||
@@ -1,53 +1,87 @@
|
||||
import type { AgentJournalMessageItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
import type { JournalSqlRequest } from './journal-sql-protocol'
|
||||
import type { JournalWriterCommand } from './journal-writer-command'
|
||||
import { journalSqlMaintenanceIsNormal } from './journal-sql-maintenance'
|
||||
|
||||
const MAX_EVENTS = 64
|
||||
const MAX_STEPS = 64
|
||||
const MAX_BYTES = 1024 * 1024
|
||||
|
||||
/** Control commands keep their own boundary; an oversized atomic command runs alone. */
|
||||
export function takeJournalSqlReadyBatch(
|
||||
queue: JournalSqlRequest[],
|
||||
streamed: (command: JournalWriterCommand) => boolean
|
||||
): { request: JournalSqlRequest; normal: boolean }[] {
|
||||
const batch: { request: JournalSqlRequest; normal: boolean }[] = []
|
||||
let events = 0
|
||||
/** Sender work only: classification never opens a saved projection. */
|
||||
export function prepareJournalSqlRequest(
|
||||
id: number,
|
||||
command: JournalSqlRequest['command']
|
||||
): JournalSqlRequest {
|
||||
let batch: JournalSqlRequest['batch']
|
||||
if (
|
||||
command.type === 'messages' &&
|
||||
command.job.command.steps.length > 0 &&
|
||||
command.job.command.steps.every(
|
||||
(step) => step.kind === 'text-snapshot' && streamingText(step.body)
|
||||
)
|
||||
) {
|
||||
batch = {
|
||||
sessionId: command.job.command.target.sessionId,
|
||||
steps: command.job.command.steps.length,
|
||||
bytes: command.job.bytes
|
||||
}
|
||||
} else if (command.type === 'write' && !command.command.effects) {
|
||||
const { operation } = command.command
|
||||
if (
|
||||
operation.kind === 'steps' &&
|
||||
operation.steps.length > 0 &&
|
||||
operation.steps.every(
|
||||
(step) =>
|
||||
step.kind === 'resolved-item' &&
|
||||
step.resolution.kind === 'text-snapshot' &&
|
||||
streamingText(step.resolution.body)
|
||||
)
|
||||
) {
|
||||
batch = {
|
||||
sessionId: command.command.target.sessionId,
|
||||
steps: operation.steps.length,
|
||||
bytes: Buffer.byteLength(JSON.stringify(command.command), 'utf8')
|
||||
}
|
||||
}
|
||||
}
|
||||
return { id, command, ...(batch ? { batch } : {}) }
|
||||
}
|
||||
|
||||
/** Controls, repeated chats and an oversized atomic command keep their own boundary. */
|
||||
export function takeJournalSqlReadyBatch(queue: JournalSqlRequest[]): JournalSqlRequest[] {
|
||||
const first = queue[0]
|
||||
if (!first?.batch) {
|
||||
return queue.splice(0, 1)
|
||||
}
|
||||
const sessions = new Set<string>()
|
||||
let count = 0
|
||||
let steps = 0
|
||||
let bytes = 0
|
||||
for (const request of queue) {
|
||||
const { command } = request
|
||||
const normal = command.type === 'write' && streamed(command.command)
|
||||
if (!normal) {
|
||||
if (batch.length === 0) {
|
||||
batch.push({
|
||||
request,
|
||||
normal: command.type === 'maintenance' && journalSqlMaintenanceIsNormal(command.operation)
|
||||
})
|
||||
}
|
||||
const next = request.batch
|
||||
if (
|
||||
!next ||
|
||||
request.command.type !== first.command.type ||
|
||||
(count > 0 &&
|
||||
(sessions.has(next.sessionId) ||
|
||||
steps + next.steps > MAX_STEPS ||
|
||||
bytes + next.bytes > MAX_BYTES))
|
||||
) {
|
||||
break
|
||||
}
|
||||
const count =
|
||||
command.command.operation.kind === 'steps'
|
||||
? Math.max(1, command.command.operation.steps.length)
|
||||
: 1
|
||||
let size: number
|
||||
try {
|
||||
size = Buffer.byteLength(JSON.stringify(command.command), 'utf8')
|
||||
} catch {
|
||||
if (batch.length === 0) {
|
||||
batch.push({ request, normal: false })
|
||||
}
|
||||
break
|
||||
}
|
||||
if (batch.length > 0 && (events + count > MAX_EVENTS || bytes + size > MAX_BYTES)) {
|
||||
break
|
||||
}
|
||||
batch.push({ request, normal: true })
|
||||
events += count
|
||||
bytes += size
|
||||
if (events >= MAX_EVENTS || bytes >= MAX_BYTES) {
|
||||
count++
|
||||
steps += next.steps
|
||||
bytes += next.bytes
|
||||
sessions.add(next.sessionId)
|
||||
if (steps >= MAX_STEPS || bytes >= MAX_BYTES) {
|
||||
break
|
||||
}
|
||||
}
|
||||
queue.splice(0, batch.length)
|
||||
return batch
|
||||
return queue.splice(0, count)
|
||||
}
|
||||
|
||||
function streamingText(body: AgentJournalMessageItem): boolean {
|
||||
return (
|
||||
(body.role === 'assistant' || body.role === 'reasoning') &&
|
||||
body.completedAt === undefined &&
|
||||
body.state === undefined &&
|
||||
body.blocks.every((block) => block.type === 'text')
|
||||
)
|
||||
}
|
||||
|
||||
@@ -155,12 +155,9 @@ it.each([
|
||||
await new JournalSqlBatch(persisted.db).run(
|
||||
[
|
||||
{
|
||||
command: {
|
||||
target: { sessionId: 'session', epoch: 'epoch', ownerToken: `open-${index}` },
|
||||
operation: { kind: 'open' as const },
|
||||
facts: { identity }
|
||||
},
|
||||
normal: false
|
||||
target: { sessionId: 'session', epoch: 'epoch', ownerToken: `open-${index}` },
|
||||
operation: { kind: 'open' as const },
|
||||
facts: { identity }
|
||||
}
|
||||
],
|
||||
(entry) => engine.apply(entry, 100)
|
||||
|
||||
@@ -14,7 +14,7 @@ import { assertJournalWritable } from './writing/journal-write-guards'
|
||||
import { applyJournalSqlMaintenance } from './journal-sql-maintenance'
|
||||
import { initializeJournalSqlProjection } from './journal-sql-schema-migration'
|
||||
import { migrateJournalDomainSchema } from './journal-domain-schema'
|
||||
import { commitJournalSqlMessages, journalSqlMessageBatchSize } from './journal-sql-message-commit'
|
||||
import { commitJournalSqlMessages } from './journal-sql-message-commit'
|
||||
|
||||
if (
|
||||
!parentPort ||
|
||||
@@ -94,24 +94,19 @@ type MessageRequest = {
|
||||
|
||||
/** Bound ready work before opening a transaction, then yield after its acknowledgements. */
|
||||
async function drain(): Promise<void> {
|
||||
if (queue[0]?.command.type === 'messages') {
|
||||
const count = journalSqlMessageBatchSize(queue, ({ command }) =>
|
||||
command.type === 'messages' ? command.job : undefined
|
||||
)
|
||||
const pending = takeJournalSqlReadyBatch(queue)
|
||||
if (pending[0]?.command.type === 'messages') {
|
||||
await drainMessages(
|
||||
queue
|
||||
.splice(0, count)
|
||||
.filter((request): request is MessageRequest => request.command.type === 'messages')
|
||||
pending.filter((request): request is MessageRequest => request.command.type === 'messages')
|
||||
)
|
||||
scheduleNextDrain()
|
||||
return
|
||||
}
|
||||
const pending = takeJournalSqlReadyBatch(queue, (command) => commands?.streamed(command) === true)
|
||||
const writes = pending.filter(
|
||||
(entry): entry is { request: WriteRequest; normal: boolean } =>
|
||||
entry.request.command.type !== 'close' && entry.request.command.type !== 'messages'
|
||||
(request): request is WriteRequest =>
|
||||
request.command.type !== 'close' && request.command.type !== 'messages'
|
||||
)
|
||||
const close = pending.find(({ request }) => request.command.type === 'close')?.request
|
||||
const close = pending.find((request) => request.command.type === 'close')
|
||||
const database = db
|
||||
if (writes.length > 0 && database && batch && commands) {
|
||||
const executor = commands
|
||||
@@ -122,32 +117,25 @@ async function drain(): Promise<void> {
|
||||
throw fault
|
||||
}
|
||||
assertJournalWritable(readOnly, 'database')
|
||||
const result = await batch.run<WriteRequest, CommitResponse>(
|
||||
writes.map(({ request, normal }) => ({
|
||||
command: request,
|
||||
normal
|
||||
})),
|
||||
async (request) => {
|
||||
const { command, id } = request
|
||||
if (command.type === 'maintenance') {
|
||||
return {
|
||||
type: 'maintenance',
|
||||
id,
|
||||
result: applyJournalSqlMaintenance(database, command.operation)
|
||||
}
|
||||
const result = await batch.run<WriteRequest, CommitResponse>(writes, async (request) => {
|
||||
const { command, id } = request
|
||||
if (command.type === 'maintenance') {
|
||||
return {
|
||||
type: 'maintenance',
|
||||
id,
|
||||
result: applyJournalSqlMaintenance(database, command.operation)
|
||||
}
|
||||
return command.type === 'write'
|
||||
? { type: 'committed', id, commit: await executor.apply(command.command, Date.now()) }
|
||||
: {
|
||||
type: 'effects',
|
||||
id,
|
||||
results: command.effects.map((effect) =>
|
||||
applyJournalWriterEffect(database, stateDirectory, effect)
|
||||
)
|
||||
}
|
||||
}
|
||||
)
|
||||
postcommit = result.postcommit ? journalSqlFailure(result.postcommit) : undefined
|
||||
return command.type === 'write'
|
||||
? { type: 'committed', id, commit: await executor.apply(command.command, Date.now()) }
|
||||
: {
|
||||
type: 'effects',
|
||||
id,
|
||||
results: command.effects.map((effect) =>
|
||||
applyJournalWriterEffect(database, stateDirectory, effect)
|
||||
)
|
||||
}
|
||||
})
|
||||
try {
|
||||
checkpoint?.afterCommit()
|
||||
} catch (error) {
|
||||
@@ -159,18 +147,18 @@ async function drain(): Promise<void> {
|
||||
? outcome.value
|
||||
: {
|
||||
type: 'failed',
|
||||
id: writes[index]!.request.id,
|
||||
id: writes[index]!.id,
|
||||
outcome: 'rolled-back',
|
||||
error: journalSqlFailure(outcome.error)
|
||||
}
|
||||
)
|
||||
if (postcommit) {
|
||||
fault = result.postcommit ?? postcommit
|
||||
fault = postcommit
|
||||
}
|
||||
} catch (error) {
|
||||
const outcome = error instanceof JournalSqlBatchError ? error.outcome : 'not-dispatched'
|
||||
fault ??= batch.fault
|
||||
responses = writes.map(({ request }) => ({
|
||||
responses = writes.map((request) => ({
|
||||
type: 'failed',
|
||||
id: request.id,
|
||||
outcome,
|
||||
@@ -240,7 +228,7 @@ async function drainMessages(requests: readonly MessageRequest[]): Promise<void>
|
||||
acknowledged.add(id)
|
||||
}
|
||||
}
|
||||
let postcommit = result.postcommit
|
||||
let postcommit: unknown
|
||||
try {
|
||||
checkpoint?.afterCommit()
|
||||
} catch (error) {
|
||||
|
||||
@@ -77,7 +77,7 @@ export class JournalWriterSession {
|
||||
db,
|
||||
readOnly: false,
|
||||
stateDirectory,
|
||||
transaction: (run, _streamed, committed) => this.savepoint(run, committed)
|
||||
transaction: (run, committed) => this.savepoint(run, committed)
|
||||
}
|
||||
const held = queuedMessageEditLeaseHolds(facts.editLeases ?? [])
|
||||
this.collaborators = createJournalStoreCollaborators({
|
||||
|
||||
+1
-5
@@ -31,10 +31,7 @@ import {
|
||||
import { claimAgentSessionAttachmentsInTransaction } from '../../agent-session-attachments/agent-session-attachment-claims'
|
||||
import { JournalGoalAttachmentInverse } from '../writing/journal-goal-attachment-inverse'
|
||||
import { JournalEpochReplacements } from '../context-epochs/journal-epoch-replacements'
|
||||
import {
|
||||
isJournalOutputRevision,
|
||||
isJournalStreamedTextRevision
|
||||
} from '../writing/journal-streamed-text-revision'
|
||||
import { isJournalOutputRevision } from '../writing/journal-output-revision'
|
||||
|
||||
export type JournalStoreHost = {
|
||||
/** Notify readers of durable draft-only transactions that append no journal row. */
|
||||
@@ -97,7 +94,6 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal
|
||||
prepare: host.prepare,
|
||||
commit: host.commit,
|
||||
project: (rows) => projectJournalRows(host.state(), rows),
|
||||
streamedTextRevision: (row) => isJournalStreamedTextRevision(host.state(), row),
|
||||
outputRevision: (row) => isJournalOutputRevision(host.state(), row)
|
||||
})
|
||||
const inputReceipt = (rows: readonly JournalRow[], receipt?: JournalOperationReceipt) =>
|
||||
|
||||
-42
@@ -2,48 +2,6 @@ import { isDeepStrictEqual } from 'node:util'
|
||||
import { agentJournalLinkageFields } from '../../../../shared/agent-session/journal/agent-session-journal-producer'
|
||||
import type { JournalReducerState } from '../projection/journal-reducer'
|
||||
import type { JournalRow } from '../durable-rows/journal-row-schema'
|
||||
import type { AgentJournalRenderItem } from '../../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
|
||||
/** Only streamed assistant text may skip per-commit fsync; FULL turn settlement makes it durable. */
|
||||
export function isJournalStreamedTextRevision(
|
||||
state: JournalReducerState,
|
||||
row: JournalRow
|
||||
): boolean {
|
||||
if (row.kind !== 'item' || row.recovered) {
|
||||
return false
|
||||
}
|
||||
const previous = state.items.get(row.itemId)
|
||||
return isJournalTextRevision(row, previous) && sameRevisionPlacement(state, row, previous!)
|
||||
}
|
||||
|
||||
export function isJournalTextRevision(
|
||||
row: Extract<JournalRow, { kind: 'item' }>,
|
||||
previous: AgentJournalRenderItem | undefined
|
||||
): boolean {
|
||||
if (row.recovered) {
|
||||
return false
|
||||
}
|
||||
const { body } = row
|
||||
if (
|
||||
body.kind !== 'message' ||
|
||||
body.role !== 'assistant' ||
|
||||
!body.blocks.every((block) => block.type === 'text') ||
|
||||
body.sentAs !== undefined ||
|
||||
body.command !== undefined ||
|
||||
body.from !== undefined ||
|
||||
previous?.body.kind !== 'message' ||
|
||||
previous.body.role !== 'assistant' ||
|
||||
previous.body.sentAs !== undefined ||
|
||||
previous.body.command !== undefined ||
|
||||
previous.body.from !== undefined ||
|
||||
!previous.body.blocks.every((block) => block.type === 'text')
|
||||
) {
|
||||
return false
|
||||
}
|
||||
const { blocks: _oldText, ...before } = previous.body
|
||||
const { blocks: _newText, ...after } = body
|
||||
return isDeepStrictEqual(before, after)
|
||||
}
|
||||
|
||||
/** Output revisions cannot change queue eligibility; lifecycle and unknown changes still wake it. */
|
||||
export function isJournalOutputRevision(state: JournalReducerState, row: JournalRow): boolean {
|
||||
@@ -102,8 +102,6 @@ export type JournalRowWriterDeps = {
|
||||
finishFold?: (draft: JournalReducerState) => void
|
||||
) => () => void
|
||||
commit: (rows: readonly JournalRow[], savedAt: number, inputChanged?: boolean) => void
|
||||
/** Streamed-text-only revisions may skip per-commit fsync; settlement makes them durable. */
|
||||
streamedTextRevision?: (row: JournalRow) => boolean
|
||||
outputRevision?: (row: JournalRow) => boolean
|
||||
}
|
||||
|
||||
@@ -120,7 +118,6 @@ export class JournalRowWriter {
|
||||
const row = build(this.deps.nextSequence(), savedAt)
|
||||
assertJournalFence(row.fence, this.deps.highestFence())
|
||||
const receipt = typeof prepareReceipt === 'function' ? prepareReceipt([row]) : prepareReceipt
|
||||
const streamed = !receipt && this.streamed([row])
|
||||
const inputChanged = Boolean(receipt) || !this.outputOnly([row])
|
||||
this.deps.database().transaction(
|
||||
(db) => {
|
||||
@@ -131,7 +128,6 @@ export class JournalRowWriter {
|
||||
receipt?.write(db, this.project([row])[0])
|
||||
return adopt
|
||||
},
|
||||
streamed,
|
||||
(adopt) => adopt?.()
|
||||
)
|
||||
receipt?.committed()
|
||||
@@ -166,7 +162,6 @@ export class JournalRowWriter {
|
||||
assertJournalFence(row.fence, this.deps.highestFence())
|
||||
}
|
||||
const receipt = typeof prepareReceipt === 'function' ? prepareReceipt(rows) : prepareReceipt
|
||||
const streamed = !receipt && this.streamed(rows)
|
||||
const inputChanged = Boolean(receipt) || !this.outputOnly(rows)
|
||||
this.deps.database().transaction(
|
||||
(db) => {
|
||||
@@ -179,7 +174,6 @@ export class JournalRowWriter {
|
||||
receipt?.write(db, receiptRow?.(this.project(rows)))
|
||||
return adopt
|
||||
},
|
||||
streamed,
|
||||
(adopt) => adopt?.()
|
||||
)
|
||||
receipt?.committed()
|
||||
@@ -205,9 +199,4 @@ export class JournalRowWriter {
|
||||
private outputOnly(rows: readonly JournalRow[]): boolean {
|
||||
return this.deps.outputRevision !== undefined && rows.every(this.deps.outputRevision)
|
||||
}
|
||||
|
||||
private streamed(rows: readonly JournalRow[]): boolean {
|
||||
const { streamedTextRevision } = this.deps
|
||||
return streamedTextRevision !== undefined && rows.every((row) => streamedTextRevision(row))
|
||||
}
|
||||
}
|
||||
|
||||
-78
@@ -1,78 +0,0 @@
|
||||
import { mkdtemp, rm, mkdir, writeFile } from 'node:fs/promises'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, dirname } from 'node:path'
|
||||
import { afterEach, beforeEach, vi } from 'vitest'
|
||||
import type { AgentJournalMessageItem } from '../../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
import { journalPragmaNumber } from '../database/journal-database'
|
||||
import {
|
||||
closeTestJournalHostDatabases,
|
||||
createTrackedJournalOpener,
|
||||
openTestJournalHostDatabase
|
||||
} from '../store-lifecycle/journal-host-database-test-support'
|
||||
import { formatNativeChatFileReference } from '../../../../shared/agent-image-paste'
|
||||
import * as attachmentReferences from '../../agent-session-attachments/agent-session-attachment-references'
|
||||
|
||||
export const journals = createTrackedJournalOpener({ legacyInputs: true })
|
||||
|
||||
let root: string
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), 'orca-streamed-revision-'))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks()
|
||||
await journals.closeAll()
|
||||
await closeTestJournalHostDatabases()
|
||||
await rm(root, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
export const text = (role: 'assistant' | 'user', value: string): AgentJournalMessageItem => ({
|
||||
kind: 'message',
|
||||
role,
|
||||
blocks: [{ type: 'text', text: value }]
|
||||
})
|
||||
|
||||
export function openJournal(sessionId = 'stream') {
|
||||
return journals.open({
|
||||
stateDirectory: root,
|
||||
identity: {
|
||||
sessionId,
|
||||
workspaceId: 'folder',
|
||||
hostId: 'local',
|
||||
agent: 'grok',
|
||||
providerHandle: null
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export function commitLevels() {
|
||||
const { db } = openTestJournalHostDatabase(root)
|
||||
const levels: number[] = []
|
||||
const exec = db.exec.bind(db)
|
||||
vi.spyOn(db, 'exec').mockImplementation((sql) => {
|
||||
if (sql === 'COMMIT') {
|
||||
levels.push(journalPragmaNumber(db, 'synchronous'))
|
||||
}
|
||||
exec(sql)
|
||||
})
|
||||
return levels
|
||||
}
|
||||
|
||||
export async function uploadedInput() {
|
||||
const id = randomUUID()
|
||||
const path = join(attachmentReferences.agentSessionAttachmentStoreRoot(root), id, 'source.txt')
|
||||
await mkdir(dirname(path), { recursive: true })
|
||||
await writeFile(path, 'retained input')
|
||||
return { id, body: text('user', formatNativeChatFileReference(path)) }
|
||||
}
|
||||
|
||||
export const fixture = {
|
||||
get root() {
|
||||
return root
|
||||
},
|
||||
set root(value: typeof root) {
|
||||
root = value
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user