fix: bound retired journal cleanup work per chat

This commit is contained in:
Brennan Benson
2026-11-08 07:44:07 -08:00
parent 010bc1aaf8
commit 32d6482322
4 changed files with 419 additions and 43 deletions
@@ -1,10 +1,22 @@
import { afterEach, expect, it } from 'vitest'
import { afterEach, expect, it, vi } from 'vitest'
import Database from '../../sqlite/sync-database'
import { messageFixture } from './journal-sql-message.test-fixture'
import { applyJournalSqlMaintenance, type JournalSqlMaintenance } from './journal-sql-maintenance'
import { pruneJournalDomainRetiredPage } from './journal-domain-maintenance'
import { readJournalSqlDatabase } from './journal-sql-database-read'
import { JournalSqlBackgroundMaintenance } from './journal-sql-background-maintenance'
import { JournalSqlActivity } from './journal-sql-activity'
import type { SqliteBindings } from '../../sqlite/sqlite-statement'
const databases: Database.Database[] = []
afterEach(() => {
const maintainers: JournalSqlBackgroundMaintenance[] = []
afterEach(async () => {
for (const maintenance of maintainers.splice(0)) {
maintenance.stop()
await maintenance.settled()
}
vi.restoreAllMocks()
vi.useRealTimers()
for (const db of databases.splice(0)) {
db.close()
}
@@ -15,13 +27,20 @@ function fixture() {
databases.push(db)
db.pragma('auto_vacuum=INCREMENTAL')
messageFixture(db)
db.exec(`CREATE TABLE IF NOT EXISTS journal_item_subagent_names (
session_id TEXT NOT NULL, epoch TEXT NOT NULL, subagent_id TEXT NOT NULL, item_id TEXT NOT NULL,
PRIMARY KEY(session_id,epoch,subagent_id,item_id)
) WITHOUT ROWID`)
return db
}
function maintain(db: Database.Database, operation: JournalSqlMaintenance) {
db.exec('BEGIN IMMEDIATE')
try {
const result = applyJournalSqlMaintenance(db, operation)
const result =
operation.kind === 'domain-retired-prune'
? pruneJournalDomainRetiredPage(db, operation)
: applyJournalSqlMaintenance(db, operation)
db.exec('COMMIT')
return result
} catch (error) {
@@ -45,6 +64,7 @@ it('prunes proven retired epochs with atomic rollback while preserving incomplet
...prefix
)
db.prepare(`INSERT INTO journal_item_tombstones VALUES(?,?,'deleted',1)`).run(...prefix)
db.prepare(`INSERT INTO journal_item_subagent_names VALUES(?,?,'agent','body')`).run(...prefix)
db.prepare(`INSERT INTO journal_item_aliases VALUES(?,?,'provider','body',0,NULL)`).run(
...prefix
)
@@ -66,7 +86,7 @@ it('prunes proven retired epochs with atomic rollback while preserving incomplet
db.exec(`INSERT INTO agent_session_records VALUES('chat','saved record');
INSERT INTO agent_session_tabs VALUES('tab','chat',1)`)
db.exec('BEGIN IMMEDIATE')
applyJournalSqlMaintenance(db, { kind: 'domain-retired-prune' })
pruneJournalDomainRetiredPage(db, { sessionId: 'chat', epoch: 'epoch' })
applyJournalSqlMaintenance(db, { kind: 'compaction-complete', now: 200 })
db.exec('ROLLBACK')
expect(
@@ -80,7 +100,7 @@ it('prunes proven retired epochs with atomic rollback while preserving incomplet
})
let more = true
while (more) {
const result = maintain(db, { kind: 'domain-retired-prune' })
const result = maintain(db, { kind: 'domain-retired-prune', sessionId: 'chat', epoch: 'epoch' })
if (!result || typeof result !== 'object' || !('more' in result)) {
throw new Error('expected page')
}
@@ -96,6 +116,13 @@ it('prunes proven retired epochs with atomic rollback while preserving incomplet
expect(
db.prepare("SELECT payload_json FROM journal_send_receipts WHERE session_id='chat'").all()
).toEqual([{ payload_json: 'accepted receipt' }])
expect(
db
.prepare(
"SELECT epoch,subagent_id,item_id FROM journal_item_subagent_names WHERE session_id='chat'"
)
.all()
).toEqual([{ epoch: 'epoch', subagent_id: 'agent', item_id: 'body' }])
expect(
db.prepare("SELECT payload_json FROM journal_submission_attempts WHERE session_id='chat'").all()
).toEqual([{ payload_json: 'accepted body' }])
@@ -129,3 +156,254 @@ it('returns a free page inside the writer transaction and restores it on rollbac
maintain(db, { kind: 'page-reclaim' })
expect(pages()).toBeLessThan(before)
})
it('bounds a ready-head pass at its captured high key and skips changed or removed heads', () => {
const db = fixture()
const insert =
db.prepare(`INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence)
VALUES(?,'epoch',0,0)`)
for (let index = 0; index < 20; index++) {
insert.run(`session-${String(index).padStart(2, '0')}`)
}
db.exec(`INSERT INTO journal_item_tombstones VALUES('chat','before','old',1),
('chat','epoch','current',1),('session-00','before','old',1)`)
const page = readJournalSqlDatabase(db, { kind: 'retired-heads', after: null, through: null })
expect(page).toMatchObject({ kind: 'retired-heads', through: 'session-19' })
if (page.kind !== 'retired-heads') {
throw new Error('expected ready heads')
}
expect(page.heads.map((head) => head.sessionId)).toEqual([
'chat',
'session-00',
'session-01',
'session-02',
'session-03',
'session-04',
'session-05',
'session-06',
'session-07',
'session-08',
'session-09',
'session-10',
'session-11',
'session-12',
'session-13',
'session-14'
])
expect(db.isTransaction).toBe(false)
insert.run('session-20')
db.exec(`UPDATE journal_chat_state SET epoch='replacement' WHERE session_id='chat';
DELETE FROM journal_chat_state WHERE session_id IN ('session-00','session-19')`)
for (const target of page.heads.slice(0, 2)) {
expect(maintain(db, { kind: 'domain-retired-prune', ...target })).toEqual({
kind: 'domain-retired',
deleted: 0,
more: false
})
}
expect(
db
.prepare('SELECT session_id,epoch FROM journal_item_tombstones ORDER BY session_id,epoch')
.all()
).toEqual([
{ session_id: 'chat', epoch: 'before' },
{ session_id: 'chat', epoch: 'epoch' },
{ session_id: 'session-00', epoch: 'before' }
])
expect(
readJournalSqlDatabase(db, {
kind: 'retired-heads',
after: 'session-14',
through: page.through
})
).toEqual({
kind: 'retired-heads',
through: 'session-19',
heads: [
{ sessionId: 'session-15', epoch: 'epoch' },
{ sessionId: 'session-16', epoch: 'epoch' },
{ sessionId: 'session-17', epoch: 'epoch' },
{ sessionId: 'session-18', epoch: 'epoch' }
]
})
expect(
readJournalSqlDatabase(db, {
kind: 'retired-heads',
after: 'session-18',
through: page.through
})
).toEqual({ kind: 'retired-heads', through: 'session-19', heads: [] })
expect(
readJournalSqlDatabase(db, { kind: 'retired-heads', after: null, through: null })
).toMatchObject({ kind: 'retired-heads', through: 'session-20' })
})
it.each([100, 1_000])(
'seeks one selected chat around %i current rows and unrelated heads without scanning them',
(size) => {
const db = fixture()
db.exec('BEGIN IMMEDIATE')
const head =
db.prepare(`INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence)
VALUES(?,'epoch',0,0)`)
const current = db.prepare(`INSERT INTO journal_item_tombstones VALUES(?,'epoch',?,1)`)
for (let index = 0; index < size; index++) {
const sessionId = `profile-${index}`
head.run(sessionId)
current.run(sessionId, 'current')
current.run('chat', `current-${index}`)
}
db.exec(`INSERT INTO journal_item_tombstones VALUES('profile-0','before','keep-other-chat',1)`)
const retired = db.prepare(`INSERT INTO journal_item_tombstones VALUES('chat',?,?,1)`)
for (let index = 0; index < 25; index++) {
retired.run('before', `old-${index}`)
retired.run('later', `old-${index}`)
}
db.exec(`INSERT INTO journal_item_subagent_names VALUES
('chat','before','agent-a','old'),('chat','later','agent-b','old'),
('chat','later','agent-c','old');
INSERT INTO journal_events VALUES('chat','later',1,1,'{}')`)
db.exec('COMMIT')
const prepare = db.prepare.bind(db)
const queries = new Map<string, SqliteBindings>()
const instrumented = new WeakSet<Database.Statement>()
vi.spyOn(db, 'prepare').mockImplementation((sql) => {
const statement = prepare(sql)
if (!instrumented.has(statement)) {
instrumented.add(statement)
const run = statement.run.bind(statement)
vi.spyOn(statement, 'run').mockImplementation((...parameters: SqliteBindings) => {
queries.set(sql, parameters)
return run(...parameters)
})
const get = statement.get.bind(statement)
vi.spyOn(statement, 'get').mockImplementation((...parameters: SqliteBindings) => {
queries.set(sql, parameters)
return get(...parameters)
})
}
return statement
})
expect(
maintain(db, { kind: 'domain-retired-prune', sessionId: 'chat', epoch: 'epoch' })
).toEqual({ kind: 'domain-retired', deleted: 50, more: true })
expect(
maintain(db, { kind: 'domain-retired-prune', sessionId: 'chat', epoch: 'epoch' })
).toEqual({ kind: 'domain-retired', deleted: 4, more: false })
vi.restoreAllMocks()
// Query plans establish the range contract independently of how many heads were seeded.
const plans = [...queries].flatMap(([sql, parameters]) =>
prepare(`EXPLAIN QUERY PLAN ${sql}`)
.all(...parameters)
.map((row) => String(row.detail))
)
expect(plans).toContain(
'SEARCH journal_chat_state USING INDEX sqlite_autoindex_journal_chat_state_1 (session_id=?)'
)
expect(plans).toContain('SEARCH journal_events USING PRIMARY KEY (session_id=? AND epoch>?)')
expect(plans.some((detail) => /^SCAN (journal_|ready|retired)/.test(detail))).toBe(false)
expect(
plans
.filter((detail) => detail.startsWith('SEARCH '))
.every(
(detail) =>
detail.startsWith('SEARCH journal_chat_state ') ||
/\(session_id=\? AND epoch[<>=]\?/.test(detail)
)
).toBe(true)
expect(
db
.prepare(
"SELECT COUNT(*) AS count FROM journal_item_tombstones WHERE session_id='chat' AND epoch='epoch'"
)
.get()
).toEqual({ count: size })
expect(
db
.prepare(
"SELECT epoch,item_id FROM journal_item_tombstones WHERE session_id='profile-0' ORDER BY epoch"
)
.all()
).toEqual([
{ epoch: 'before', item_id: 'keep-other-chat' },
{ epoch: 'epoch', item_id: 'current' }
])
expect(
db.prepare("SELECT epoch FROM journal_item_subagent_names WHERE session_id='chat'").all()
).toEqual([])
}
)
it('finishes a retired pass during activity while an already-started conversion is paused on idle', async () => {
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })
vi.setSystemTime(100)
const db = fixture()
const activity = new JournalSqlActivity()
const paused = Promise.withResolvers<void>()
const passFinished = Promise.withResolvers<void>()
let converted = false
let awaitingPass = false
const errors: unknown[] = []
const maintenance = new JournalSqlBackgroundMaintenance(
{ maintain: async (operation) => maintain(db, operation) },
{
database: async (operation) => {
if (operation.kind === 'sessions') {
return { kind: 'sessions', sessionIds: operation.after === null ? ['unready'] : [] }
}
const result = readJournalSqlDatabase(db, operation)
if (awaitingPass && result.kind === 'retired-heads' && result.heads.length === 0) {
passFinished.resolve()
}
return result
}
},
{
ensure: async (_sessionId, foreground) => {
if (foreground) {
throw new Error('background conversion was promoted')
}
activity.setSessionActive('streaming', true)
paused.resolve()
await activity.waitForIdle(new AbortController().signal)
converted = true
return null
}
},
activity,
(error) => errors.push(error)
)
maintainers.push(maintenance)
const started = maintenance.start()
await paused.promise
db.exec('BEGIN IMMEDIATE')
const head =
db.prepare(`INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence)
VALUES(?,'epoch',0,0)`)
const retired = db.prepare(`INSERT INTO journal_item_tombstones VALUES(?,'before',?,1)`)
for (let chat = 0; chat < 20; chat++) {
const sessionId = `retired-${String(chat).padStart(2, '0')}`
head.run(sessionId)
for (let row = 0; row < 53; row++) {
retired.run(sessionId, `old-${row}`)
}
db.prepare(`INSERT INTO journal_item_tombstones VALUES(?,'epoch','current',1)`).run(sessionId)
}
db.exec('COMMIT')
awaitingPass = true
await vi.advanceTimersByTimeAsync(60_000)
await passFinished.promise
expect(converted).toBe(false)
expect(errors).toEqual([])
expect(
db.prepare("SELECT COUNT(*) AS count FROM journal_item_tombstones WHERE epoch='before'").get()
).toEqual({ count: 0 })
expect(
db.prepare("SELECT COUNT(*) AS count FROM journal_item_tombstones WHERE epoch='epoch'").get()
).toEqual({ count: 20 })
activity.setSessionActive('streaming', false)
await started
expect(converted).toBe(true)
})
@@ -1,7 +1,7 @@
import type Database from '../../sqlite/sync-database'
export type JournalDomainMaintenance =
| { kind: 'domain-retired-prune' }
| { kind: 'domain-retired-prune'; sessionId: string; epoch: string }
| { kind: 'page-reclaim' }
| { kind: 'compaction-state' }
| { kind: 'compaction-complete'; now: number }
@@ -14,6 +14,7 @@ export type JournalDomainMaintenanceResult =
// Only a ready head naming another epoch proves retirement; an interrupted import has no head.
const RETIRED_TABLES = [
{ name: 'journal_items', key: 'item_id' },
{ name: 'journal_item_subagent_names', key: 'subagent_id,item_id' },
{ name: 'journal_item_tombstones', key: 'item_id' },
{ name: 'journal_item_aliases', key: 'provider_item_id' },
{ name: 'journal_submission_attempts', key: 'message_id,attempt_sequence,attempt_index' },
@@ -25,24 +26,26 @@ const RETIRED_TABLES = [
] as const
export function pruneJournalDomainRetiredPage(
db: Database.Database
db: Database.Database,
target: { sessionId: string; epoch: string }
): JournalDomainMaintenanceResult {
const head = db
.prepare('SELECT epoch FROM journal_chat_state WHERE session_id=?')
.get(target.sessionId)
if (head?.epoch !== target.epoch) {
return { kind: 'domain-retired', deleted: 0, more: false }
}
let remaining = 50
for (const { name, key } of RETIRED_TABLES) {
const columns = key
.split(',')
.map((column) => `retired.${column}`)
.join(',')
// Separate ranges seek past the current epoch rather than scanning its live rows.
for (const comparison of ['<', '>']) {
const result = db
.prepare(`DELETE FROM ${name} WHERE (session_id,epoch,${key}) IN (
SELECT retired.session_id,retired.epoch,${columns}
FROM journal_chat_state AS ready CROSS JOIN ${name} AS retired
WHERE retired.session_id=ready.session_id AND retired.epoch${comparison}ready.epoch
SELECT session_id,epoch,${key} FROM ${name}
WHERE session_id=? AND epoch${comparison}?
LIMIT ?
)`)
.run(remaining)
.run(target.sessionId, target.epoch, remaining)
remaining -= Number(result.changes)
if (remaining === 0) {
return { kind: 'domain-retired', deleted: 50, more: true }
@@ -8,7 +8,7 @@ import type { JournalSqlMaintenance } from './journal-sql-maintenance'
const MINUTE = 60_000
const COMPACTION_INTERVAL = 6 * 60 * MINUTE
/** Conversion and retired epochs wait for idle; periodic pruning shares this owner without starvation. */
/** Idle conversion and periodic pruning share one owner; activity never blocks retired cleanup. */
export class JournalSqlBackgroundMaintenance {
private readonly stopped = new AbortController()
private running: Promise<void> | undefined
@@ -16,6 +16,9 @@ export class JournalSqlBackgroundMaintenance {
private sweep = false
private periodic = false
private reclaim = 0
private retiredAfter: string | null = null
private retiredThrough: string | null = null
private work = 0
private wake = new AbortController()
private timer: NodeJS.Timeout | undefined
@@ -101,20 +104,13 @@ export class JournalSqlBackgroundMaintenance {
this.report(error)
}
}
if (this.sweep) {
this.sweep = false
try {
await this.pages({ kind: 'domain-retired-prune' }, Infinity, true)
} catch (error) {
this.report(error)
}
}
}
}
private async runOrdinaryWork(): Promise<void> {
if (this.periodic && !this.stopped.signal.aborted) {
this.periodic = false
this.sweep = true
try {
const state = await this.writer.maintain({ kind: 'compaction-state' })
if (
@@ -130,7 +126,6 @@ export class JournalSqlBackgroundMaintenance {
await this.pages({ kind: 'event-tail-prune', now })
if (!this.stopped.signal.aborted) {
await this.writer.maintain({ kind: 'compaction-complete', now: Date.now() })
this.sweep = true
}
}
await this.pages({ kind: 'page-reclaim' }, this.reclaim)
@@ -138,6 +133,46 @@ export class JournalSqlBackgroundMaintenance {
this.report(error)
}
}
if (this.sweep && !this.stopped.signal.aborted) {
this.sweep = false
try {
await this.retiredPages()
} catch (error) {
this.report(error)
}
}
}
private async retiredPages(): Promise<void> {
while (!this.stopped.signal.aborted) {
await setImmediate()
if (this.stopped.signal.aborted) {
return
}
const started = performance.now()
const page = await this.readers.database({
kind: 'retired-heads',
after: this.retiredAfter,
through: this.retiredThrough
})
await this.rest(started)
if (page.kind !== 'retired-heads') {
throw new Error('journal reader returned no retired heads')
}
this.retiredThrough = page.through
if (page.heads.length === 0) {
this.retiredAfter = null
this.retiredThrough = null
return
}
for (const target of page.heads) {
await this.pages({ kind: 'domain-retired-prune', ...target })
if (this.stopped.signal.aborted) {
return
}
this.retiredAfter = target.sessionId
}
}
}
private async pages(
@@ -145,46 +180,45 @@ export class JournalSqlBackgroundMaintenance {
JournalSqlMaintenance,
{ kind: 'event-tail-prune' | 'domain-retired-prune' | 'page-reclaim' }
>,
limit = Infinity,
idle = false
limit = Infinity
): Promise<void> {
let work = 0
for (let unit = 0; unit < limit && !this.stopped.signal.aborted; unit++) {
if (operation.kind !== 'page-reclaim' && this.reclaim > 0) {
await this.pages({ kind: 'page-reclaim' }, 1)
}
if (idle && !(await this.boundary())) {
return
}
await setImmediate()
if (this.stopped.signal.aborted) {
return
}
const started = performance.now()
const result = await this.writer.maintain(operation)
work += performance.now() - started
if (!result || typeof result !== 'object' || !('more' in result)) {
throw new Error('journal writer returned no maintenance page')
}
if (operation.kind === 'page-reclaim') {
this.reclaim = result.more ? Math.max(0, this.reclaim - 1) : 0
}
if (work >= 25) {
try {
await delay(work * 9, undefined, { signal: this.stopped.signal, ref: false })
} catch (error) {
if (!this.stopped.signal.aborted) {
throw error
}
}
work = 0
}
await this.rest(started)
if (!result.more) {
return
}
}
}
private async rest(started: number): Promise<void> {
this.work += performance.now() - started
if (this.work >= 25) {
try {
await delay(this.work * 9, undefined, { signal: this.stopped.signal, ref: false })
} catch (error) {
if (!this.stopped.signal.aborted) {
throw error
}
}
this.work = 0
}
}
private async convertSavedChats(): Promise<void> {
let after: string | null = null
while (await this.boundary()) {
@@ -200,7 +234,7 @@ export class JournalSqlBackgroundMaintenance {
return
}
try {
await this.conversions.ensure(sessionId, false)
await this.convertSavedChat(sessionId)
} catch (error) {
this.report(error, sessionId)
}
@@ -209,6 +243,36 @@ export class JournalSqlBackgroundMaintenance {
}
}
private async convertSavedChat(sessionId: string): Promise<void> {
let complete = false
const conversion = this.conversions.ensure(sessionId, false).then(
() => {
complete = true
},
(error) => {
complete = true
this.report(error, sessionId)
}
)
while (!complete && !this.stopped.signal.aborted) {
await this.runOrdinaryWork()
const signal = AbortSignal.any([this.stopped.signal, this.wake.signal])
if (complete || signal.aborted) {
continue
}
let ready = () => {}
const wake = new Promise<void>((resolve) => {
ready = resolve
signal.addEventListener('abort', ready, { once: true })
})
try {
await Promise.race([conversion, wake])
} finally {
signal.removeEventListener('abort', ready)
}
}
}
private report(error: unknown, sessionId?: string): void {
if (!this.stopped.signal.aborted) {
this.onError(error, sessionId)
@@ -14,6 +14,7 @@ export type JournalSqlDatabaseRead =
| { kind: 'receipt'; scope: CommandReceiptScope; operationId: string }
| { kind: 'epoch'; sessionId: string }
| { kind: 'sessions'; after: string | null }
| { kind: 'retired-heads'; after: string | null; through: string | null }
| { kind: 'attachment-claims'; after: number }
| { kind: 'attachment-sweeps'; after: string | null }
@@ -22,6 +23,11 @@ export type JournalSqlDatabaseReadResult =
| { kind: 'receipt'; receipt: ReturnType<typeof readCommandReceipt> }
| { kind: 'epoch'; epoch: string | null }
| { kind: 'sessions'; sessionIds: string[] }
| {
kind: 'retired-heads'
heads: { sessionId: string; epoch: string }[]
through: string | null
}
| {
kind: 'attachment-claims'
claims: { rowid: number; uploadId: string; sessionId: string }[]
@@ -73,6 +79,31 @@ function read(
.all(operation.after)
return { kind: 'sessions', sessionIds: rows.map((row) => String(row.session_id)) }
}
case 'retired-heads': {
const through =
operation.through ??
db
.prepare('SELECT session_id FROM journal_chat_state ORDER BY session_id DESC LIMIT 1')
.get()?.session_id
if (through === undefined) {
return { kind: 'retired-heads', heads: [], through: null }
}
const rows =
operation.after === null
? db
.prepare(`SELECT session_id,epoch FROM journal_chat_state
WHERE session_id<=? ORDER BY session_id LIMIT 16`)
.all(through)
: db
.prepare(`SELECT session_id,epoch FROM journal_chat_state
WHERE session_id>? AND session_id<=? ORDER BY session_id LIMIT 16`)
.all(operation.after, through)
return {
kind: 'retired-heads',
heads: rows.map((row) => ({ sessionId: String(row.session_id), epoch: String(row.epoch) })),
through: String(through)
}
}
case 'attachment-claims':
return {
kind: 'attachment-claims',