fix: coordinate journal WAL pressure with writer admission

This commit is contained in:
Brennan Benson
2026-11-08 06:39:57 -08:00
parent fc149b3ad3
commit c6c0fb3581
5 changed files with 399 additions and 86 deletions
@@ -6,10 +6,19 @@ import { journalCheckpointWorkerSource } from './journal-checkpoint-worker-sourc
import { JOURNAL_SIZE_LIMIT_BYTES } from './journal-database'
const HEALTH_TIMEOUT_MS = 5000
export const JOURNAL_WAL_BACKPRESSURE_FRAMES = 32_768
type CheckpointResponse = { type: 'checked'; id?: number }
type CheckpointResponse = {
type: 'checked' | 'truncate-request'
id?: number
generation: number
truncateId?: number
log: number
checkpointed: number
blocked: boolean
}
/** Maintenance never participates in write admission or the durability acknowledgement. */
/** One writer observes outside transactions; the checkpoint thread never races an admitted truncate. */
export class JournalCheckpointOwner {
private worker: WorkerRequestTransport | undefined
private exited: Promise<void> = Promise.resolve()
@@ -18,7 +27,13 @@ export class JournalCheckpointOwner {
private closing = false
private pending: number | undefined
private nextRequest = 0
private dirty = false
private generation = 0
private checkAgain = false
private log = 0
private unknown = true
private blocked = false
private truncate: { id: number; acknowledged: boolean } | undefined
private readonly waiters = new Set<() => void>()
private restoreNeeded = false
private warned = false
private closePromise: Promise<void> | undefined
@@ -34,7 +49,6 @@ export class JournalCheckpointOwner {
try {
const version = String(writer.prepare('SELECT sqlite_version() AS version').get()?.version)
const [major = 0, minor = 0] = version.split('.').map(Number)
// Older SQLite interprets NOOP as PASSIVE, which would move maintenance back onto the host.
if (major < 3 || (major === 3 && minor < 51)) {
this.fallback(new Error(`SQLite ${version} lacks NOOP checkpoints`))
return
@@ -43,7 +57,11 @@ export class JournalCheckpointOwner {
? options.factory()
: spawnWorkerThread(journalCheckpointWorkerSource, {
eval: true,
workerData: { filename, sizeLimit: JOURNAL_SIZE_LIMIT_BYTES }
workerData: {
filename,
sizeLimit: JOURNAL_SIZE_LIMIT_BYTES,
backpressureFrames: JOURNAL_WAL_BACKPRESSURE_FRAMES
}
})
this.worker = worker
this.exited = new Promise((resolve) => {
@@ -59,59 +77,154 @@ export class JournalCheckpointOwner {
worker.unref()
this.restoreNeeded = true
this.writer.pragma('wal_autocheckpoint = 0')
// Cover schema/setup debt even when no future write arrives.
this.afterCommit()
} catch (error) {
this.fallback(error)
}
}
/** Only the serialized writer calls this, before its outer transaction. */
async beforeWrite(): Promise<void> {
if (this.writer.isTransaction) {
throw new Error('journal WAL admission requires a transaction-free writer')
}
while (this.gated()) {
await new Promise<void>((resolve) => {
this.waiters.add(resolve)
this.request()
this.wake()
})
}
}
/** Also called in finally after rollback or partially successful autocommit maintenance. */
afterCommit(): void {
if (this.stopped || this.closing || this.writer.isTransaction) {
return
}
try {
if (this.closing) {
return
this.generation++
try {
const state = this.writer.prepare('PRAGMA wal_checkpoint(NOOP)').get()
if (!Number.isSafeInteger(state?.log) || !Number.isSafeInteger(state?.checkpointed)) {
throw new Error('invalid journal WAL frame counts')
}
this.log = Number(state!.log)
this.unknown = false
} catch {
this.unknown = true
}
if (this.stopped) {
this.restore()
return
}
this.dirty = true
if (this.pending !== undefined) {
return
}
this.dirty = false
this.pending = ++this.nextRequest
this.worker?.postMessage({ type: 'check', id: this.pending })
this.armDeadline()
this.request()
this.wake()
} catch (error) {
this.fallback(error)
}
}
close(): Promise<void> {
this.closing = true
this.clearDeadline()
this.closePromise ??= this.finishClose().catch((error: unknown) => {
this.closing = false
this.fallback(error)
this.closePromise = undefined
throw error
})
return this.closePromise
}
private gated(): boolean {
return (
!this.stopped &&
(this.unknown ||
this.blocked ||
this.truncate !== undefined ||
this.log >= JOURNAL_WAL_BACKPRESSURE_FRAMES)
)
}
private wake(): void {
if (this.gated()) {
return
}
const waiters = [...this.waiters]
this.waiters.clear()
for (const resume of waiters) {
resume()
}
}
private request(): void {
if (this.stopped || this.writer.isTransaction) {
return
}
if (this.truncate) {
this.acknowledge()
return
}
if (this.pending !== undefined) {
this.checkAgain = true
return
}
this.pending = ++this.nextRequest
this.send({ type: 'check', id: this.pending, generation: this.generation, force: this.gated() })
this.armDeadline()
}
private acknowledge(): void {
if (!this.truncate || this.truncate.acknowledged || this.writer.isTransaction || this.stopped) {
return
}
this.truncate.acknowledged = true
this.send({ type: 'truncate', truncateId: this.truncate.id })
this.armDeadline()
}
private send(message: unknown): void {
try {
this.worker?.postMessage(message)
} catch (error) {
this.fallback(error)
}
}
private receive(message: CheckpointResponse): void {
if (this.stopped || this.closing) {
return
}
try {
if (message.type === 'checked' && message.id === this.pending && this.pending !== undefined) {
if (message.type === 'truncate-request' && Number.isSafeInteger(message.truncateId)) {
this.truncate = { id: message.truncateId!, acknowledged: false }
this.blocked = true
this.acknowledge()
return
}
if (message.type !== 'checked') {
return
}
const completedTruncate =
this.truncate?.acknowledged === true && message.truncateId === this.truncate.id
if (completedTruncate) {
this.truncate = undefined
}
if (
(completedTruncate || message.generation === this.generation) &&
Number.isSafeInteger(message.log) &&
Number.isSafeInteger(message.checkpointed) &&
typeof message.blocked === 'boolean'
) {
this.log = message.log
this.blocked = message.blocked
this.unknown = false
}
// A stale response retires its request without clearing newer pressure.
if (message.id === this.pending && this.pending !== undefined) {
this.pending = undefined
}
if (this.pending === undefined && this.truncate === undefined) {
this.clearDeadline()
if (this.dirty) {
this.afterCommit()
if (this.checkAgain) {
this.checkAgain = false
this.request()
}
}
this.wake()
} catch (error) {
this.fallback(error)
}
@@ -123,6 +236,7 @@ export class JournalCheckpointOwner {
}
const first = !this.stopped
this.stopped = true
this.truncate = undefined
this.clearDeadline()
this.restoreNeeded = true
try {
@@ -130,6 +244,7 @@ export class JournalCheckpointOwner {
} catch (restoreError) {
this.warn(restoreError)
}
this.wake()
this.warn(error)
if (first) {
try {
@@ -141,11 +256,10 @@ export class JournalCheckpointOwner {
}
private restore(): void {
if (!this.restoreNeeded) {
return
if (this.restoreNeeded) {
this.writer.pragma('wal_autocheckpoint = 1000')
this.restoreNeeded = false
}
this.writer.pragma('wal_autocheckpoint = 1000')
this.restoreNeeded = false
}
private warn(error: unknown): void {
@@ -164,20 +278,15 @@ export class JournalCheckpointOwner {
}
private armDeadline(): void {
if (this.stopped) {
return
}
this.clearDeadline()
this.deadline = setTimeout(
() => this.fallback(new Error('checkpoint worker response timed out')),
this.options.timeoutMs ?? HEALTH_TIMEOUT_MS
)
const timer: unknown = this.deadline
if (
typeof timer === 'object' &&
timer !== null &&
'unref' in timer &&
typeof timer.unref === 'function'
) {
timer.unref()
}
this.deadline.unref()
}
private clearDeadline(): void {
@@ -186,13 +295,22 @@ export class JournalCheckpointOwner {
}
private async finishClose(): Promise<void> {
this.closing = true
this.stopped = true
this.truncate = undefined
this.clearDeadline()
this.restoreNeeded = true
try {
// Hand admission back before termination: close can arrive while the drain is waiting.
this.restore()
} finally {
this.wake()
}
const worker = this.worker
if (worker) {
await worker.terminate()
await this.exited
this.worker = undefined
}
this.restoreNeeded = true
this.restore()
}
}
@@ -0,0 +1,171 @@
import { EventEmitter } from 'node:events'
import { mkdtempSync, rmSync, statSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Worker } from 'node:worker_threads'
import { afterEach, expect, it, vi } from 'vitest'
import Database from '../../../sqlite/sync-database'
import { JournalSqlBatch } from '../journal-sql-batch'
import { JournalCheckpointOwner, JOURNAL_WAL_BACKPRESSURE_FRAMES } from './journal-checkpoint-owner'
import { JOURNAL_SIZE_LIMIT_BYTES } from './journal-database'
import { journalCheckpointWorkerSource } from './journal-checkpoint-worker-source'
const cleanup: (() => void | Promise<void>)[] = []
afterEach(async () => {
vi.restoreAllMocks()
for (const step of cleanup.splice(0).toReversed()) {
await step()
}
})
function fixture() {
const root = mkdtempSync(join(tmpdir(), 'journal-wal-pressure-'))
cleanup.push(() => rmSync(root, { recursive: true }))
const filename = join(root, 'chat.db')
const writer = new Database(filename)
cleanup.push(() => writer.close())
writer.exec(`PRAGMA page_size=512; PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;
PRAGMA wal_autocheckpoint=0; CREATE TABLE writes(id INTEGER PRIMARY KEY, body BLOB);
INSERT INTO writes VALUES(0, 'before');`)
return { filename, writer }
}
it.each(['before-write', 'after-write'] as const)(
'retains a queued send until a %s pinned reader releases, then truncates and admits it',
async (pinAt) => {
const { filename, writer } = fixture()
const reader = new Database(filename, { readonly: true })
cleanup.push(() => reader.close())
let observedBlocked!: () => void
const blocked = new Promise<void>((resolve) => {
observedBlocked = resolve
})
const worker = new Worker(journalCheckpointWorkerSource, {
eval: true,
workerData: {
filename,
sizeLimit: JOURNAL_SIZE_LIMIT_BYTES,
backpressureFrames: JOURNAL_WAL_BACKPRESSURE_FRAMES
}
})
worker.on('message', (message) => {
if (
message.type === 'checked' &&
message.blocked &&
(pinAt === 'before-write' || message.truncateId !== undefined)
) {
observedBlocked()
}
})
const owner = new JournalCheckpointOwner(writer, filename, { factory: () => worker })
cleanup.push(() => owner.close())
const batch = new JournalSqlBatch(writer, owner)
const pin = () => reader.exec('BEGIN; SELECT COUNT(*) FROM writes;')
if (pinAt === 'before-write') {
pin()
}
await batch.run([1], (id) =>
writer.prepare('INSERT INTO writes VALUES(?, zeroblob(18874368))').run(id)
)
if (pinAt === 'after-write') {
pin()
}
let admitted = false
const send = batch.run([2], (id) => {
admitted = true
writer.prepare("INSERT INTO writes VALUES(?, 'queued send')").run(id)
})
await blocked
expect(admitted).toBe(false)
expect(writer.prepare('SELECT id FROM writes ORDER BY id').all()).toEqual([
{ id: 0 },
{ id: 1 }
])
reader.exec('ROLLBACK')
await send
expect(writer.prepare('SELECT id, body FROM writes WHERE id=2').get()).toEqual({
id: 2,
body: 'queued send'
})
expect(Number(writer.prepare('PRAGMA wal_checkpoint(NOOP)').get()!.log)).toBeLessThan(32768)
expect(statSync(`${filename}-wal`).size).toBeLessThan(JOURNAL_SIZE_LIMIT_BYTES)
},
15000
)
class ControlledCheckpoint extends EventEmitter {
readonly sent: { type: string; id?: number; generation?: number }[] = []
private finish: (() => void) | undefined
unref() {
return this
}
postMessage(message: { type: string; id?: number; generation?: number }) {
this.sent.push(message)
}
terminate(): Promise<number> {
return new Promise((resolve) => {
this.finish = () => {
this.emit('exit', 0)
resolve(0)
}
})
}
exited() {
this.finish!()
}
}
it('does not admit a send from a stale checkpoint, and close hands back admission before physical retirement', async () => {
const { filename, writer } = fixture()
const worker = new ControlledCheckpoint()
const owner = new JournalCheckpointOwner(writer, filename, { factory: () => worker })
const batch = new JournalSqlBatch(writer, owner)
const old = worker.sent[0]!
writer.prepare('INSERT INTO writes VALUES(1, zeroblob(18874368))').run()
owner.afterCommit()
const send = batch.run([2], (id) =>
writer.prepare("INSERT INTO writes VALUES(?, 'send')").run(id)
)
worker.emit('message', {
type: 'checked',
id: old.id,
generation: old.generation,
log: 0,
checkpointed: 0,
blocked: false
})
await Promise.resolve()
expect(writer.prepare('SELECT id FROM writes WHERE id=2').get()).toBeUndefined()
let retired = false
const close = owner.close().then(() => {
retired = true
})
cleanup.push(() => close)
await send
expect(writer.pragma('wal_autocheckpoint', { simple: true })).toBe(1000)
expect(writer.prepare('SELECT body FROM writes WHERE id=2').get()).toEqual({ body: 'send' })
expect(retired).toBe(false)
worker.emit('message', { type: 'truncate-request', truncateId: 99 })
worker.exited()
await close
})
it('hands pending sends back to autocheckpoint when the checkpoint worker fails', async () => {
const { filename, writer } = fixture()
const worker = new ControlledCheckpoint()
const owner = new JournalCheckpointOwner(writer, filename, { factory: () => worker })
const warning = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
writer.prepare('INSERT INTO writes VALUES(1, zeroblob(18874368))').run()
owner.afterCommit()
const send = new JournalSqlBatch(writer, owner).run([2], (id) =>
writer.prepare("INSERT INTO writes VALUES(?, 'send')").run(id)
)
worker.emit('error', new Error('checkpoint unavailable'))
await send
expect(writer.pragma('wal_autocheckpoint', { simple: true })).toBe(1000)
expect(writer.prepare('SELECT body FROM writes WHERE id=2').get()).toEqual({ body: 'send' })
expect(warning).toHaveBeenCalled()
const close = owner.close()
worker.exited()
await close
})
@@ -5,30 +5,71 @@ const { DatabaseSync } = require('node:sqlite');
const { statSync } = require('node:fs');
let db;
let idle;
let generation = 0;
let nextTruncate = 0;
let pendingTruncate;
try {
db = new DatabaseSync(workerData.filename);
if (db.prepare('PRAGMA journal_mode').get().journal_mode !== 'wal') {
throw new Error('chat journal is not in WAL mode');
}
db.exec('PRAGMA busy_timeout = 0; PRAGMA synchronous = NORMAL; PRAGMA checkpoint_fullfsync = ON;');
db.exec('PRAGMA busy_timeout = 5000; PRAGMA synchronous = NORMAL; PRAGMA checkpoint_fullfsync = ON;');
db.exec('PRAGMA journal_size_limit = ' + workerData.sizeLimit);
const frames = db.prepare('PRAGMA wal_checkpoint(NOOP)');
const checkpoint = db.prepare('PRAGMA wal_checkpoint(PASSIVE)');
const check = (force, id) => {
clearTimeout(idle);
let state = frames.get();
const oversized = statSync(workerData.filename + '-wal').size > workerData.sizeLimit;
if (force || oversized || state.log - state.checkpointed >= 1000) {
checkpoint.get();
state = frames.get();
}
parentPort.postMessage({ type: 'checked', id, ...state });
// Let active low-volume writes share the WAL instead of flushing and restarting it every burst.
if (state.log > state.checkpointed) {
const truncate = db.prepare('PRAGMA wal_checkpoint(TRUNCATE)');
const oversized = () => statSync(workerData.filename + '-wal').size > workerData.sizeLimit;
const ceiling = (state) => state.log >= workerData.backpressureFrames;
const schedule = (state, blocked) => {
if (blocked || state.log > state.checkpointed) {
idle = setTimeout(() => check(true), 1000);
}
};
parentPort.on('message', (request) => check(false, request.id));
const check = (force, id, nextGeneration = generation) => {
clearTimeout(idle);
generation = Math.max(generation, nextGeneration);
let state = frames.get();
if (force || oversized() || state.log - state.checkpointed >= 1000) {
checkpoint.get();
state = frames.get();
}
if (state.log === state.checkpointed && (ceiling(state) || oversized())) {
pendingTruncate = { truncateId: ++nextTruncate, id, generation, state };
parentPort.postMessage({ type: 'truncate-request', ...pendingTruncate, ...state });
return;
}
const blocked = (ceiling(state) || oversized()) && state.checkpointed < state.log;
parentPort.postMessage({ type: 'checked', id, generation, ...state, blocked });
schedule(state, blocked);
};
parentPort.on('message', (request) => {
clearTimeout(idle);
if (request.type === 'truncate') {
if (!pendingTruncate || request.truncateId !== pendingTruncate.truncateId) return;
const pending = pendingTruncate;
let busy;
const timeout = db.prepare('PRAGMA busy_timeout').get().timeout;
db.exec('PRAGMA busy_timeout = 0;');
try {
busy = truncate.get().busy !== 0;
} finally {
db.exec('PRAGMA busy_timeout = ' + timeout);
}
const state = frames.get();
const blocked = (ceiling(state) && (busy || state.checkpointed < state.log)) || oversized();
pendingTruncate = undefined;
parentPort.postMessage({ type: 'checked', id: pending.id, generation: pending.generation,
truncateId: pending.truncateId, ...state, blocked });
schedule(state, blocked);
return;
}
if (pendingTruncate) {
parentPort.postMessage({ type: 'checked', id: request.id, generation: pendingTruncate.generation,
...pendingTruncate.state, blocked: true });
return;
}
check(request.force, request.id, request.generation);
});
} catch (error) {
db?.close();
throw error;
@@ -24,7 +24,10 @@ export class JournalSqlBatchError extends Error {
export class JournalSqlBatch {
private failure: unknown
constructor(private readonly db: Database.Database) {}
constructor(
private readonly db: Database.Database,
private readonly checkpoint?: { beforeWrite(): Promise<void>; afterCommit(): void }
) {}
get fault(): unknown {
return this.failure
@@ -47,6 +50,7 @@ export class JournalSqlBatch {
let committing = false
let failed: JournalSqlBatchError | undefined
try {
await this.checkpoint?.beforeWrite()
this.db.exec('BEGIN IMMEDIATE')
began = true
for (const command of drain) {
@@ -80,6 +84,8 @@ export class JournalSqlBatch {
if (committing || rollback) {
this.failure = failed
}
} finally {
this.checkpoint?.afterCommit()
}
if (failed) {
throw failed
@@ -8,7 +8,7 @@ import { JournalSqlBatch, JournalSqlBatchError } from './journal-sql-batch'
import { JournalSqlCommand } from './journal-sql-command'
import { takeJournalSqlReadyBatch } from './journal-sql-ready-batch'
import { applyJournalWriterEffect } from './journal-writer-effects'
import { journalSqlFailure, type JournalSqlFailure } from './journal-sql-failure'
import { journalSqlFailure } from './journal-sql-failure'
import type { JournalSqlRequest, JournalSqlResponse } from './journal-sql-protocol'
import { assertJournalWritable } from './writing/journal-write-guards'
import { applyJournalSqlMaintenance } from './journal-sql-maintenance'
@@ -30,6 +30,7 @@ let db: Database.Database | undefined
let batch: JournalSqlBatch | undefined
let commands: JournalSqlCommand | undefined
let checkpoint: JournalCheckpointOwner | undefined
let checkpointClosing: Promise<void> | undefined
let readOnly = true
let fault: unknown
let closing = false
@@ -41,13 +42,13 @@ try {
const opened = openJournalDatabase(workerData.filename)
db = opened.db
readOnly = opened.readOnly
batch = new JournalSqlBatch(db)
commands = new JournalSqlCommand(db, stateDirectory)
if (!readOnly) {
initializeJournalSqlProjection(db)
migrateJournalDomainSchema(db)
checkpoint = new JournalCheckpointOwner(db, workerData.filename)
}
batch = new JournalSqlBatch(db, checkpoint)
reply({ type: 'ready', readOnly })
} catch (error) {
fault = error
@@ -68,6 +69,8 @@ port.on('message', (request: JournalSqlRequest) => {
}
if (request.command.type === 'close') {
closing = true
checkpointClosing = checkpoint?.close()
void checkpointClosing?.catch(() => undefined)
}
queue.push(request)
if (!scheduled) {
@@ -111,7 +114,6 @@ async function drain(): Promise<void> {
if (writes.length > 0 && database && batch && commands) {
const executor = commands
let responses: JournalSqlResponse[] = []
let postcommit: JournalSqlFailure | undefined
try {
if (fault) {
throw fault
@@ -136,11 +138,6 @@ async function drain(): Promise<void> {
)
}
})
try {
checkpoint?.afterCommit()
} catch (error) {
postcommit ??= journalSqlFailure(error)
}
// All acknowledged commits precede a fault notice; it never retracts a successful COMMIT.
responses = result.outcomes.map((outcome, index) =>
outcome.committed
@@ -152,9 +149,6 @@ async function drain(): Promise<void> {
error: journalSqlFailure(outcome.error)
}
)
if (postcommit) {
fault = postcommit
}
} catch (error) {
const outcome = error instanceof JournalSqlBatchError ? error.outcome : 'not-dispatched'
fault ??= batch.fault
@@ -170,19 +164,17 @@ async function drain(): Promise<void> {
for (const response of responses) {
reply(response)
}
if (postcommit) {
reply({ type: 'failed', outcome: 'not-dispatched', fatal: true, error: postcommit })
}
}
if (close) {
try {
await checkpoint?.close()
await (checkpointClosing ?? checkpoint?.close())
db?.close()
db = undefined
reply({ type: 'done', id: close.id })
port.close()
} catch (error) {
closing = false
checkpointClosing = undefined
reply({
type: 'failed',
id: close.id,
@@ -228,21 +220,6 @@ async function drainMessages(requests: readonly MessageRequest[]): Promise<void>
acknowledged.add(id)
}
}
let postcommit: unknown
try {
checkpoint?.afterCommit()
} catch (error) {
postcommit ??= error
}
if (postcommit) {
fault = postcommit
reply({
type: 'failed',
outcome: 'not-dispatched',
fatal: true,
error: journalSqlFailure(postcommit)
})
}
} catch (error) {
fault ??= batch.fault
for (const request of requests) {