mirror of
https://github.com/stablyai/orca.git
synced 2026-11-18 03:45:08 +00:00
fix: coordinate journal WAL pressure with writer admission
This commit is contained in:
@@ -6,10 +6,19 @@ import { journalCheckpointWorkerSource } from './journal-checkpoint-worker-sourc
|
||||
import { JOURNAL_SIZE_LIMIT_BYTES } from './journal-database'
|
||||
|
||||
const HEALTH_TIMEOUT_MS = 5000
|
||||
export const JOURNAL_WAL_BACKPRESSURE_FRAMES = 32_768
|
||||
|
||||
type CheckpointResponse = { type: 'checked'; id?: number }
|
||||
type CheckpointResponse = {
|
||||
type: 'checked' | 'truncate-request'
|
||||
id?: number
|
||||
generation: number
|
||||
truncateId?: number
|
||||
log: number
|
||||
checkpointed: number
|
||||
blocked: boolean
|
||||
}
|
||||
|
||||
/** Maintenance never participates in write admission or the durability acknowledgement. */
|
||||
/** One writer observes outside transactions; the checkpoint thread never races an admitted truncate. */
|
||||
export class JournalCheckpointOwner {
|
||||
private worker: WorkerRequestTransport | undefined
|
||||
private exited: Promise<void> = Promise.resolve()
|
||||
@@ -18,7 +27,13 @@ export class JournalCheckpointOwner {
|
||||
private closing = false
|
||||
private pending: number | undefined
|
||||
private nextRequest = 0
|
||||
private dirty = false
|
||||
private generation = 0
|
||||
private checkAgain = false
|
||||
private log = 0
|
||||
private unknown = true
|
||||
private blocked = false
|
||||
private truncate: { id: number; acknowledged: boolean } | undefined
|
||||
private readonly waiters = new Set<() => void>()
|
||||
private restoreNeeded = false
|
||||
private warned = false
|
||||
private closePromise: Promise<void> | undefined
|
||||
@@ -34,7 +49,6 @@ export class JournalCheckpointOwner {
|
||||
try {
|
||||
const version = String(writer.prepare('SELECT sqlite_version() AS version').get()?.version)
|
||||
const [major = 0, minor = 0] = version.split('.').map(Number)
|
||||
// Older SQLite interprets NOOP as PASSIVE, which would move maintenance back onto the host.
|
||||
if (major < 3 || (major === 3 && minor < 51)) {
|
||||
this.fallback(new Error(`SQLite ${version} lacks NOOP checkpoints`))
|
||||
return
|
||||
@@ -43,7 +57,11 @@ export class JournalCheckpointOwner {
|
||||
? options.factory()
|
||||
: spawnWorkerThread(journalCheckpointWorkerSource, {
|
||||
eval: true,
|
||||
workerData: { filename, sizeLimit: JOURNAL_SIZE_LIMIT_BYTES }
|
||||
workerData: {
|
||||
filename,
|
||||
sizeLimit: JOURNAL_SIZE_LIMIT_BYTES,
|
||||
backpressureFrames: JOURNAL_WAL_BACKPRESSURE_FRAMES
|
||||
}
|
||||
})
|
||||
this.worker = worker
|
||||
this.exited = new Promise((resolve) => {
|
||||
@@ -59,59 +77,154 @@ export class JournalCheckpointOwner {
|
||||
worker.unref()
|
||||
this.restoreNeeded = true
|
||||
this.writer.pragma('wal_autocheckpoint = 0')
|
||||
// Cover schema/setup debt even when no future write arrives.
|
||||
this.afterCommit()
|
||||
} catch (error) {
|
||||
this.fallback(error)
|
||||
}
|
||||
}
|
||||
|
||||
/** Only the serialized writer calls this, before its outer transaction. */
|
||||
async beforeWrite(): Promise<void> {
|
||||
if (this.writer.isTransaction) {
|
||||
throw new Error('journal WAL admission requires a transaction-free writer')
|
||||
}
|
||||
while (this.gated()) {
|
||||
await new Promise<void>((resolve) => {
|
||||
this.waiters.add(resolve)
|
||||
this.request()
|
||||
this.wake()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/** Also called in finally after rollback or partially successful autocommit maintenance. */
|
||||
afterCommit(): void {
|
||||
if (this.stopped || this.closing || this.writer.isTransaction) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
if (this.closing) {
|
||||
return
|
||||
this.generation++
|
||||
try {
|
||||
const state = this.writer.prepare('PRAGMA wal_checkpoint(NOOP)').get()
|
||||
if (!Number.isSafeInteger(state?.log) || !Number.isSafeInteger(state?.checkpointed)) {
|
||||
throw new Error('invalid journal WAL frame counts')
|
||||
}
|
||||
this.log = Number(state!.log)
|
||||
this.unknown = false
|
||||
} catch {
|
||||
this.unknown = true
|
||||
}
|
||||
if (this.stopped) {
|
||||
this.restore()
|
||||
return
|
||||
}
|
||||
this.dirty = true
|
||||
if (this.pending !== undefined) {
|
||||
return
|
||||
}
|
||||
this.dirty = false
|
||||
this.pending = ++this.nextRequest
|
||||
this.worker?.postMessage({ type: 'check', id: this.pending })
|
||||
this.armDeadline()
|
||||
this.request()
|
||||
this.wake()
|
||||
} catch (error) {
|
||||
this.fallback(error)
|
||||
}
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
this.closing = true
|
||||
this.clearDeadline()
|
||||
this.closePromise ??= this.finishClose().catch((error: unknown) => {
|
||||
this.closing = false
|
||||
this.fallback(error)
|
||||
this.closePromise = undefined
|
||||
throw error
|
||||
})
|
||||
return this.closePromise
|
||||
}
|
||||
|
||||
private gated(): boolean {
|
||||
return (
|
||||
!this.stopped &&
|
||||
(this.unknown ||
|
||||
this.blocked ||
|
||||
this.truncate !== undefined ||
|
||||
this.log >= JOURNAL_WAL_BACKPRESSURE_FRAMES)
|
||||
)
|
||||
}
|
||||
|
||||
private wake(): void {
|
||||
if (this.gated()) {
|
||||
return
|
||||
}
|
||||
const waiters = [...this.waiters]
|
||||
this.waiters.clear()
|
||||
for (const resume of waiters) {
|
||||
resume()
|
||||
}
|
||||
}
|
||||
|
||||
private request(): void {
|
||||
if (this.stopped || this.writer.isTransaction) {
|
||||
return
|
||||
}
|
||||
if (this.truncate) {
|
||||
this.acknowledge()
|
||||
return
|
||||
}
|
||||
if (this.pending !== undefined) {
|
||||
this.checkAgain = true
|
||||
return
|
||||
}
|
||||
this.pending = ++this.nextRequest
|
||||
this.send({ type: 'check', id: this.pending, generation: this.generation, force: this.gated() })
|
||||
this.armDeadline()
|
||||
}
|
||||
|
||||
private acknowledge(): void {
|
||||
if (!this.truncate || this.truncate.acknowledged || this.writer.isTransaction || this.stopped) {
|
||||
return
|
||||
}
|
||||
this.truncate.acknowledged = true
|
||||
this.send({ type: 'truncate', truncateId: this.truncate.id })
|
||||
this.armDeadline()
|
||||
}
|
||||
|
||||
private send(message: unknown): void {
|
||||
try {
|
||||
this.worker?.postMessage(message)
|
||||
} catch (error) {
|
||||
this.fallback(error)
|
||||
}
|
||||
}
|
||||
|
||||
private receive(message: CheckpointResponse): void {
|
||||
if (this.stopped || this.closing) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
if (message.type === 'checked' && message.id === this.pending && this.pending !== undefined) {
|
||||
if (message.type === 'truncate-request' && Number.isSafeInteger(message.truncateId)) {
|
||||
this.truncate = { id: message.truncateId!, acknowledged: false }
|
||||
this.blocked = true
|
||||
this.acknowledge()
|
||||
return
|
||||
}
|
||||
if (message.type !== 'checked') {
|
||||
return
|
||||
}
|
||||
const completedTruncate =
|
||||
this.truncate?.acknowledged === true && message.truncateId === this.truncate.id
|
||||
if (completedTruncate) {
|
||||
this.truncate = undefined
|
||||
}
|
||||
if (
|
||||
(completedTruncate || message.generation === this.generation) &&
|
||||
Number.isSafeInteger(message.log) &&
|
||||
Number.isSafeInteger(message.checkpointed) &&
|
||||
typeof message.blocked === 'boolean'
|
||||
) {
|
||||
this.log = message.log
|
||||
this.blocked = message.blocked
|
||||
this.unknown = false
|
||||
}
|
||||
// A stale response retires its request without clearing newer pressure.
|
||||
if (message.id === this.pending && this.pending !== undefined) {
|
||||
this.pending = undefined
|
||||
}
|
||||
if (this.pending === undefined && this.truncate === undefined) {
|
||||
this.clearDeadline()
|
||||
if (this.dirty) {
|
||||
this.afterCommit()
|
||||
if (this.checkAgain) {
|
||||
this.checkAgain = false
|
||||
this.request()
|
||||
}
|
||||
}
|
||||
this.wake()
|
||||
} catch (error) {
|
||||
this.fallback(error)
|
||||
}
|
||||
@@ -123,6 +236,7 @@ export class JournalCheckpointOwner {
|
||||
}
|
||||
const first = !this.stopped
|
||||
this.stopped = true
|
||||
this.truncate = undefined
|
||||
this.clearDeadline()
|
||||
this.restoreNeeded = true
|
||||
try {
|
||||
@@ -130,6 +244,7 @@ export class JournalCheckpointOwner {
|
||||
} catch (restoreError) {
|
||||
this.warn(restoreError)
|
||||
}
|
||||
this.wake()
|
||||
this.warn(error)
|
||||
if (first) {
|
||||
try {
|
||||
@@ -141,11 +256,10 @@ export class JournalCheckpointOwner {
|
||||
}
|
||||
|
||||
private restore(): void {
|
||||
if (!this.restoreNeeded) {
|
||||
return
|
||||
if (this.restoreNeeded) {
|
||||
this.writer.pragma('wal_autocheckpoint = 1000')
|
||||
this.restoreNeeded = false
|
||||
}
|
||||
this.writer.pragma('wal_autocheckpoint = 1000')
|
||||
this.restoreNeeded = false
|
||||
}
|
||||
|
||||
private warn(error: unknown): void {
|
||||
@@ -164,20 +278,15 @@ export class JournalCheckpointOwner {
|
||||
}
|
||||
|
||||
private armDeadline(): void {
|
||||
if (this.stopped) {
|
||||
return
|
||||
}
|
||||
this.clearDeadline()
|
||||
this.deadline = setTimeout(
|
||||
() => this.fallback(new Error('checkpoint worker response timed out')),
|
||||
this.options.timeoutMs ?? HEALTH_TIMEOUT_MS
|
||||
)
|
||||
const timer: unknown = this.deadline
|
||||
if (
|
||||
typeof timer === 'object' &&
|
||||
timer !== null &&
|
||||
'unref' in timer &&
|
||||
typeof timer.unref === 'function'
|
||||
) {
|
||||
timer.unref()
|
||||
}
|
||||
this.deadline.unref()
|
||||
}
|
||||
|
||||
private clearDeadline(): void {
|
||||
@@ -186,13 +295,22 @@ export class JournalCheckpointOwner {
|
||||
}
|
||||
|
||||
private async finishClose(): Promise<void> {
|
||||
this.closing = true
|
||||
this.stopped = true
|
||||
this.truncate = undefined
|
||||
this.clearDeadline()
|
||||
this.restoreNeeded = true
|
||||
try {
|
||||
// Hand admission back before termination: close can arrive while the drain is waiting.
|
||||
this.restore()
|
||||
} finally {
|
||||
this.wake()
|
||||
}
|
||||
const worker = this.worker
|
||||
if (worker) {
|
||||
await worker.terminate()
|
||||
await this.exited
|
||||
this.worker = undefined
|
||||
}
|
||||
this.restoreNeeded = true
|
||||
this.restore()
|
||||
}
|
||||
}
|
||||
|
||||
+171
@@ -0,0 +1,171 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { mkdtempSync, rmSync, statSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { Worker } from 'node:worker_threads'
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import Database from '../../../sqlite/sync-database'
|
||||
import { JournalSqlBatch } from '../journal-sql-batch'
|
||||
import { JournalCheckpointOwner, JOURNAL_WAL_BACKPRESSURE_FRAMES } from './journal-checkpoint-owner'
|
||||
import { JOURNAL_SIZE_LIMIT_BYTES } from './journal-database'
|
||||
import { journalCheckpointWorkerSource } from './journal-checkpoint-worker-source'
|
||||
|
||||
const cleanup: (() => void | Promise<void>)[] = []
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks()
|
||||
for (const step of cleanup.splice(0).toReversed()) {
|
||||
await step()
|
||||
}
|
||||
})
|
||||
|
||||
function fixture() {
|
||||
const root = mkdtempSync(join(tmpdir(), 'journal-wal-pressure-'))
|
||||
cleanup.push(() => rmSync(root, { recursive: true }))
|
||||
const filename = join(root, 'chat.db')
|
||||
const writer = new Database(filename)
|
||||
cleanup.push(() => writer.close())
|
||||
writer.exec(`PRAGMA page_size=512; PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;
|
||||
PRAGMA wal_autocheckpoint=0; CREATE TABLE writes(id INTEGER PRIMARY KEY, body BLOB);
|
||||
INSERT INTO writes VALUES(0, 'before');`)
|
||||
return { filename, writer }
|
||||
}
|
||||
|
||||
it.each(['before-write', 'after-write'] as const)(
|
||||
'retains a queued send until a %s pinned reader releases, then truncates and admits it',
|
||||
async (pinAt) => {
|
||||
const { filename, writer } = fixture()
|
||||
const reader = new Database(filename, { readonly: true })
|
||||
cleanup.push(() => reader.close())
|
||||
let observedBlocked!: () => void
|
||||
const blocked = new Promise<void>((resolve) => {
|
||||
observedBlocked = resolve
|
||||
})
|
||||
const worker = new Worker(journalCheckpointWorkerSource, {
|
||||
eval: true,
|
||||
workerData: {
|
||||
filename,
|
||||
sizeLimit: JOURNAL_SIZE_LIMIT_BYTES,
|
||||
backpressureFrames: JOURNAL_WAL_BACKPRESSURE_FRAMES
|
||||
}
|
||||
})
|
||||
worker.on('message', (message) => {
|
||||
if (
|
||||
message.type === 'checked' &&
|
||||
message.blocked &&
|
||||
(pinAt === 'before-write' || message.truncateId !== undefined)
|
||||
) {
|
||||
observedBlocked()
|
||||
}
|
||||
})
|
||||
const owner = new JournalCheckpointOwner(writer, filename, { factory: () => worker })
|
||||
cleanup.push(() => owner.close())
|
||||
const batch = new JournalSqlBatch(writer, owner)
|
||||
const pin = () => reader.exec('BEGIN; SELECT COUNT(*) FROM writes;')
|
||||
if (pinAt === 'before-write') {
|
||||
pin()
|
||||
}
|
||||
await batch.run([1], (id) =>
|
||||
writer.prepare('INSERT INTO writes VALUES(?, zeroblob(18874368))').run(id)
|
||||
)
|
||||
if (pinAt === 'after-write') {
|
||||
pin()
|
||||
}
|
||||
let admitted = false
|
||||
const send = batch.run([2], (id) => {
|
||||
admitted = true
|
||||
writer.prepare("INSERT INTO writes VALUES(?, 'queued send')").run(id)
|
||||
})
|
||||
await blocked
|
||||
expect(admitted).toBe(false)
|
||||
expect(writer.prepare('SELECT id FROM writes ORDER BY id').all()).toEqual([
|
||||
{ id: 0 },
|
||||
{ id: 1 }
|
||||
])
|
||||
reader.exec('ROLLBACK')
|
||||
await send
|
||||
expect(writer.prepare('SELECT id, body FROM writes WHERE id=2').get()).toEqual({
|
||||
id: 2,
|
||||
body: 'queued send'
|
||||
})
|
||||
expect(Number(writer.prepare('PRAGMA wal_checkpoint(NOOP)').get()!.log)).toBeLessThan(32768)
|
||||
expect(statSync(`${filename}-wal`).size).toBeLessThan(JOURNAL_SIZE_LIMIT_BYTES)
|
||||
},
|
||||
15000
|
||||
)
|
||||
|
||||
class ControlledCheckpoint extends EventEmitter {
|
||||
readonly sent: { type: string; id?: number; generation?: number }[] = []
|
||||
private finish: (() => void) | undefined
|
||||
unref() {
|
||||
return this
|
||||
}
|
||||
postMessage(message: { type: string; id?: number; generation?: number }) {
|
||||
this.sent.push(message)
|
||||
}
|
||||
terminate(): Promise<number> {
|
||||
return new Promise((resolve) => {
|
||||
this.finish = () => {
|
||||
this.emit('exit', 0)
|
||||
resolve(0)
|
||||
}
|
||||
})
|
||||
}
|
||||
exited() {
|
||||
this.finish!()
|
||||
}
|
||||
}
|
||||
|
||||
it('does not admit a send from a stale checkpoint, and close hands back admission before physical retirement', async () => {
|
||||
const { filename, writer } = fixture()
|
||||
const worker = new ControlledCheckpoint()
|
||||
const owner = new JournalCheckpointOwner(writer, filename, { factory: () => worker })
|
||||
const batch = new JournalSqlBatch(writer, owner)
|
||||
const old = worker.sent[0]!
|
||||
writer.prepare('INSERT INTO writes VALUES(1, zeroblob(18874368))').run()
|
||||
owner.afterCommit()
|
||||
const send = batch.run([2], (id) =>
|
||||
writer.prepare("INSERT INTO writes VALUES(?, 'send')").run(id)
|
||||
)
|
||||
worker.emit('message', {
|
||||
type: 'checked',
|
||||
id: old.id,
|
||||
generation: old.generation,
|
||||
log: 0,
|
||||
checkpointed: 0,
|
||||
blocked: false
|
||||
})
|
||||
await Promise.resolve()
|
||||
expect(writer.prepare('SELECT id FROM writes WHERE id=2').get()).toBeUndefined()
|
||||
let retired = false
|
||||
const close = owner.close().then(() => {
|
||||
retired = true
|
||||
})
|
||||
cleanup.push(() => close)
|
||||
await send
|
||||
expect(writer.pragma('wal_autocheckpoint', { simple: true })).toBe(1000)
|
||||
expect(writer.prepare('SELECT body FROM writes WHERE id=2').get()).toEqual({ body: 'send' })
|
||||
expect(retired).toBe(false)
|
||||
worker.emit('message', { type: 'truncate-request', truncateId: 99 })
|
||||
worker.exited()
|
||||
await close
|
||||
})
|
||||
|
||||
it('hands pending sends back to autocheckpoint when the checkpoint worker fails', async () => {
|
||||
const { filename, writer } = fixture()
|
||||
const worker = new ControlledCheckpoint()
|
||||
const owner = new JournalCheckpointOwner(writer, filename, { factory: () => worker })
|
||||
const warning = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
writer.prepare('INSERT INTO writes VALUES(1, zeroblob(18874368))').run()
|
||||
owner.afterCommit()
|
||||
const send = new JournalSqlBatch(writer, owner).run([2], (id) =>
|
||||
writer.prepare("INSERT INTO writes VALUES(?, 'send')").run(id)
|
||||
)
|
||||
worker.emit('error', new Error('checkpoint unavailable'))
|
||||
await send
|
||||
expect(writer.pragma('wal_autocheckpoint', { simple: true })).toBe(1000)
|
||||
expect(writer.prepare('SELECT body FROM writes WHERE id=2').get()).toEqual({ body: 'send' })
|
||||
expect(warning).toHaveBeenCalled()
|
||||
const close = owner.close()
|
||||
worker.exited()
|
||||
await close
|
||||
})
|
||||
+54
-13
@@ -5,30 +5,71 @@ const { DatabaseSync } = require('node:sqlite');
|
||||
const { statSync } = require('node:fs');
|
||||
let db;
|
||||
let idle;
|
||||
let generation = 0;
|
||||
let nextTruncate = 0;
|
||||
let pendingTruncate;
|
||||
try {
|
||||
db = new DatabaseSync(workerData.filename);
|
||||
if (db.prepare('PRAGMA journal_mode').get().journal_mode !== 'wal') {
|
||||
throw new Error('chat journal is not in WAL mode');
|
||||
}
|
||||
db.exec('PRAGMA busy_timeout = 0; PRAGMA synchronous = NORMAL; PRAGMA checkpoint_fullfsync = ON;');
|
||||
db.exec('PRAGMA busy_timeout = 5000; PRAGMA synchronous = NORMAL; PRAGMA checkpoint_fullfsync = ON;');
|
||||
db.exec('PRAGMA journal_size_limit = ' + workerData.sizeLimit);
|
||||
const frames = db.prepare('PRAGMA wal_checkpoint(NOOP)');
|
||||
const checkpoint = db.prepare('PRAGMA wal_checkpoint(PASSIVE)');
|
||||
const check = (force, id) => {
|
||||
clearTimeout(idle);
|
||||
let state = frames.get();
|
||||
const oversized = statSync(workerData.filename + '-wal').size > workerData.sizeLimit;
|
||||
if (force || oversized || state.log - state.checkpointed >= 1000) {
|
||||
checkpoint.get();
|
||||
state = frames.get();
|
||||
}
|
||||
parentPort.postMessage({ type: 'checked', id, ...state });
|
||||
// Let active low-volume writes share the WAL instead of flushing and restarting it every burst.
|
||||
if (state.log > state.checkpointed) {
|
||||
const truncate = db.prepare('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
const oversized = () => statSync(workerData.filename + '-wal').size > workerData.sizeLimit;
|
||||
const ceiling = (state) => state.log >= workerData.backpressureFrames;
|
||||
const schedule = (state, blocked) => {
|
||||
if (blocked || state.log > state.checkpointed) {
|
||||
idle = setTimeout(() => check(true), 1000);
|
||||
}
|
||||
};
|
||||
parentPort.on('message', (request) => check(false, request.id));
|
||||
const check = (force, id, nextGeneration = generation) => {
|
||||
clearTimeout(idle);
|
||||
generation = Math.max(generation, nextGeneration);
|
||||
let state = frames.get();
|
||||
if (force || oversized() || state.log - state.checkpointed >= 1000) {
|
||||
checkpoint.get();
|
||||
state = frames.get();
|
||||
}
|
||||
if (state.log === state.checkpointed && (ceiling(state) || oversized())) {
|
||||
pendingTruncate = { truncateId: ++nextTruncate, id, generation, state };
|
||||
parentPort.postMessage({ type: 'truncate-request', ...pendingTruncate, ...state });
|
||||
return;
|
||||
}
|
||||
const blocked = (ceiling(state) || oversized()) && state.checkpointed < state.log;
|
||||
parentPort.postMessage({ type: 'checked', id, generation, ...state, blocked });
|
||||
schedule(state, blocked);
|
||||
};
|
||||
parentPort.on('message', (request) => {
|
||||
clearTimeout(idle);
|
||||
if (request.type === 'truncate') {
|
||||
if (!pendingTruncate || request.truncateId !== pendingTruncate.truncateId) return;
|
||||
const pending = pendingTruncate;
|
||||
let busy;
|
||||
const timeout = db.prepare('PRAGMA busy_timeout').get().timeout;
|
||||
db.exec('PRAGMA busy_timeout = 0;');
|
||||
try {
|
||||
busy = truncate.get().busy !== 0;
|
||||
} finally {
|
||||
db.exec('PRAGMA busy_timeout = ' + timeout);
|
||||
}
|
||||
const state = frames.get();
|
||||
const blocked = (ceiling(state) && (busy || state.checkpointed < state.log)) || oversized();
|
||||
pendingTruncate = undefined;
|
||||
parentPort.postMessage({ type: 'checked', id: pending.id, generation: pending.generation,
|
||||
truncateId: pending.truncateId, ...state, blocked });
|
||||
schedule(state, blocked);
|
||||
return;
|
||||
}
|
||||
if (pendingTruncate) {
|
||||
parentPort.postMessage({ type: 'checked', id: request.id, generation: pendingTruncate.generation,
|
||||
...pendingTruncate.state, blocked: true });
|
||||
return;
|
||||
}
|
||||
check(request.force, request.id, request.generation);
|
||||
});
|
||||
} catch (error) {
|
||||
db?.close();
|
||||
throw error;
|
||||
|
||||
@@ -24,7 +24,10 @@ export class JournalSqlBatchError extends Error {
|
||||
export class JournalSqlBatch {
|
||||
private failure: unknown
|
||||
|
||||
constructor(private readonly db: Database.Database) {}
|
||||
constructor(
|
||||
private readonly db: Database.Database,
|
||||
private readonly checkpoint?: { beforeWrite(): Promise<void>; afterCommit(): void }
|
||||
) {}
|
||||
|
||||
get fault(): unknown {
|
||||
return this.failure
|
||||
@@ -47,6 +50,7 @@ export class JournalSqlBatch {
|
||||
let committing = false
|
||||
let failed: JournalSqlBatchError | undefined
|
||||
try {
|
||||
await this.checkpoint?.beforeWrite()
|
||||
this.db.exec('BEGIN IMMEDIATE')
|
||||
began = true
|
||||
for (const command of drain) {
|
||||
@@ -80,6 +84,8 @@ export class JournalSqlBatch {
|
||||
if (committing || rollback) {
|
||||
this.failure = failed
|
||||
}
|
||||
} finally {
|
||||
this.checkpoint?.afterCommit()
|
||||
}
|
||||
if (failed) {
|
||||
throw failed
|
||||
|
||||
@@ -8,7 +8,7 @@ import { JournalSqlBatch, JournalSqlBatchError } from './journal-sql-batch'
|
||||
import { JournalSqlCommand } from './journal-sql-command'
|
||||
import { takeJournalSqlReadyBatch } from './journal-sql-ready-batch'
|
||||
import { applyJournalWriterEffect } from './journal-writer-effects'
|
||||
import { journalSqlFailure, type JournalSqlFailure } from './journal-sql-failure'
|
||||
import { journalSqlFailure } from './journal-sql-failure'
|
||||
import type { JournalSqlRequest, JournalSqlResponse } from './journal-sql-protocol'
|
||||
import { assertJournalWritable } from './writing/journal-write-guards'
|
||||
import { applyJournalSqlMaintenance } from './journal-sql-maintenance'
|
||||
@@ -30,6 +30,7 @@ let db: Database.Database | undefined
|
||||
let batch: JournalSqlBatch | undefined
|
||||
let commands: JournalSqlCommand | undefined
|
||||
let checkpoint: JournalCheckpointOwner | undefined
|
||||
let checkpointClosing: Promise<void> | undefined
|
||||
let readOnly = true
|
||||
let fault: unknown
|
||||
let closing = false
|
||||
@@ -41,13 +42,13 @@ try {
|
||||
const opened = openJournalDatabase(workerData.filename)
|
||||
db = opened.db
|
||||
readOnly = opened.readOnly
|
||||
batch = new JournalSqlBatch(db)
|
||||
commands = new JournalSqlCommand(db, stateDirectory)
|
||||
if (!readOnly) {
|
||||
initializeJournalSqlProjection(db)
|
||||
migrateJournalDomainSchema(db)
|
||||
checkpoint = new JournalCheckpointOwner(db, workerData.filename)
|
||||
}
|
||||
batch = new JournalSqlBatch(db, checkpoint)
|
||||
reply({ type: 'ready', readOnly })
|
||||
} catch (error) {
|
||||
fault = error
|
||||
@@ -68,6 +69,8 @@ port.on('message', (request: JournalSqlRequest) => {
|
||||
}
|
||||
if (request.command.type === 'close') {
|
||||
closing = true
|
||||
checkpointClosing = checkpoint?.close()
|
||||
void checkpointClosing?.catch(() => undefined)
|
||||
}
|
||||
queue.push(request)
|
||||
if (!scheduled) {
|
||||
@@ -111,7 +114,6 @@ async function drain(): Promise<void> {
|
||||
if (writes.length > 0 && database && batch && commands) {
|
||||
const executor = commands
|
||||
let responses: JournalSqlResponse[] = []
|
||||
let postcommit: JournalSqlFailure | undefined
|
||||
try {
|
||||
if (fault) {
|
||||
throw fault
|
||||
@@ -136,11 +138,6 @@ async function drain(): Promise<void> {
|
||||
)
|
||||
}
|
||||
})
|
||||
try {
|
||||
checkpoint?.afterCommit()
|
||||
} catch (error) {
|
||||
postcommit ??= journalSqlFailure(error)
|
||||
}
|
||||
// All acknowledged commits precede a fault notice; it never retracts a successful COMMIT.
|
||||
responses = result.outcomes.map((outcome, index) =>
|
||||
outcome.committed
|
||||
@@ -152,9 +149,6 @@ async function drain(): Promise<void> {
|
||||
error: journalSqlFailure(outcome.error)
|
||||
}
|
||||
)
|
||||
if (postcommit) {
|
||||
fault = postcommit
|
||||
}
|
||||
} catch (error) {
|
||||
const outcome = error instanceof JournalSqlBatchError ? error.outcome : 'not-dispatched'
|
||||
fault ??= batch.fault
|
||||
@@ -170,19 +164,17 @@ async function drain(): Promise<void> {
|
||||
for (const response of responses) {
|
||||
reply(response)
|
||||
}
|
||||
if (postcommit) {
|
||||
reply({ type: 'failed', outcome: 'not-dispatched', fatal: true, error: postcommit })
|
||||
}
|
||||
}
|
||||
if (close) {
|
||||
try {
|
||||
await checkpoint?.close()
|
||||
await (checkpointClosing ?? checkpoint?.close())
|
||||
db?.close()
|
||||
db = undefined
|
||||
reply({ type: 'done', id: close.id })
|
||||
port.close()
|
||||
} catch (error) {
|
||||
closing = false
|
||||
checkpointClosing = undefined
|
||||
reply({
|
||||
type: 'failed',
|
||||
id: close.id,
|
||||
@@ -228,21 +220,6 @@ async function drainMessages(requests: readonly MessageRequest[]): Promise<void>
|
||||
acknowledged.add(id)
|
||||
}
|
||||
}
|
||||
let postcommit: unknown
|
||||
try {
|
||||
checkpoint?.afterCommit()
|
||||
} catch (error) {
|
||||
postcommit ??= error
|
||||
}
|
||||
if (postcommit) {
|
||||
fault = postcommit
|
||||
reply({
|
||||
type: 'failed',
|
||||
outcome: 'not-dispatched',
|
||||
fatal: true,
|
||||
error: journalSqlFailure(postcommit)
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
fault ??= batch.fault
|
||||
for (const request of requests) {
|
||||
|
||||
Reference in New Issue
Block a user