perf(orchestration): stop orchestration.db growth by pruning settled chat output copies (#42839)

* perf(orchestration): prune settled chat output copies and reclaim orchestration.db space

Background maintenance rewrites chat workers' saved output copies of Runs settled over a day ago
to the reference later releases store, every six hours at a tenth of the thread, and returns
free pages in bounded steps once the file uses incremental auto_vacuum. New databases start
that way; Settings > Agent orchestration can restart Orca to rewrite an existing one.

* fix(orchestration): prune only copies whose chat is readable here, in bounded pages

Pages now examine a fixed window of archive rows and check each Run once, so a large Run
cannot make one slice quadratic. A copy whose chat this host cannot read stays. The restart
rewrite keeps its temporary copy on disk and records a failure for Settings to show.

* fix(orchestration): check chat readability without re-registering released workers

The readability probe builds the identity from the durable row instead of rehydrating it into
the live registry. Free-space checks use SQLite's own temp folder; a page's Run verdicts are
shared across a pass; a failed failure record no longer hides the VACUUM error.

* chore(orchestration): annotate prune casts, follow Button spacing, sync the runtime catalog

* chore(i18n): resync the runtime English catalog after rebase

* fix(orchestration): share Run verdicts across a prune pass; check the temp drive SQLite uses on Windows

* perf(orchestration): compact legacy storage during update shutdown

* test(orchestration): mock filesystem space checks through ESM factory

* Align update maintenance checks with shutdown-only work

* Drop the one-time update VACUUM; keep prune and incremental reclaim

Existing non-incremental orchestration databases stop growing because freed
pages are reused; nothing shrinks them automatically.
This commit is contained in:
Brennan Benson
2026-11-16 01:01:58 -08:00
committed by GitHub
parent 980352d897
commit 6db9aabbe1
7 changed files with 334 additions and 3 deletions
@@ -0,0 +1,123 @@
import { setImmediate, setTimeout as delay } from 'node:timers/promises'
import type { OrchestrationDb } from '../orchestration-db'
import { pruneSettledChatOutputCopies } from './settled-chat-output-copy-prune'
const MINUTE = 60_000
const PRUNE_INTERVAL_MS = 6 * 60 * MINUTE
// Bound each minute's reclaim, as the chat journal does.
const RECLAIM_PAGES_PER_PASS = 1024
const RECLAIM_PAGES_PER_STEP = 16
/** Prunes settled internal history every six hours and returns free pages, at a tenth of the thread. */
export class OrchestrationDbMaintenance {
private readonly stopped = new AbortController()
private timer: NodeJS.Timeout | undefined
private work = 0
constructor(
private readonly orchestrationDb: OrchestrationDb,
private readonly chatIsReadable: (dispatchId: string) => boolean
) {}
start(): void {
this.arm()
}
stop(): void {
this.stopped.abort()
clearTimeout(this.timer)
this.timer = undefined
}
private arm(): void {
if (this.stopped.signal.aborted) {
return
}
this.timer = setTimeout(() => {
this.timer = undefined
void this.pass()
.catch((error) => {
if (!this.stopped.signal.aborted) {
console.warn('[orchestration] database maintenance failed', error)
}
})
.finally(() => this.arm())
}, MINUTE)
this.timer.unref()
}
private async pass(): Promise<void> {
if (this.pruneIsDue()) {
const settledRuns = new Map<string, boolean>()
for (let after: string | null = ''; after !== null;) {
const page = await this.slice(() =>
pruneSettledChatOutputCopies(
this.orchestrationDb,
after!,
this.chatIsReadable,
settledRuns
)
)
if (!page) {
return
}
after = page.next
}
// Persisted so a restart does not rescan before the interval.
this.recordPrune()
}
// Only incremental files can return pages; others reuse them as they are freed.
if (Number(this.orchestrationDb.db.pragma('auto_vacuum', { simple: true })) === 2) {
for (let page = 0; page < RECLAIM_PAGES_PER_PASS; page += RECLAIM_PAGES_PER_STEP) {
if (!(await this.slice(() => this.reclaimFreePages()))) {
return
}
}
}
}
private pruneIsDue(): boolean {
const row = this.orchestrationDb.db
.prepare('SELECT pruned_at FROM orchestration_maintenance WHERE id = 1')
.get()
return !row || Date.now() - Number(row.pruned_at) >= PRUNE_INTERVAL_MS
}
private recordPrune(): void {
this.orchestrationDb.db
.prepare(
`INSERT INTO orchestration_maintenance (id, pruned_at) VALUES (1, ?)
ON CONFLICT(id) DO UPDATE SET pruned_at = excluded.pruned_at`
)
.run(Date.now())
}
/** True while free pages remain. */
private reclaimFreePages(): boolean {
const free = Number(this.orchestrationDb.db.pragma('freelist_count', { simple: true }))
if (free > 0) {
this.orchestrationDb.db.exec(`PRAGMA incremental_vacuum(${RECLAIM_PAGES_PER_STEP})`)
}
return free > RECLAIM_PAGES_PER_STEP
}
/** Runs one bounded step, then rests nine times its work once 25 ms accumulates. */
private async slice<T>(step: () => T): Promise<T | undefined> {
await setImmediate()
if (this.stopped.signal.aborted) {
return undefined
}
const started = performance.now()
const result = step()
this.work += performance.now() - started
if (this.work >= 25) {
try {
await delay(this.work * 9, undefined, { signal: this.stopped.signal, ref: false })
} catch {
return undefined
}
this.work = 0
}
return result
}
}
@@ -0,0 +1,74 @@
import { afterEach, describe, expect, it } from 'vitest'
import { OrchestrationDb } from '../orchestration-db'
import { pruneSettledChatOutputCopies } from './settled-chat-output-copy-prune'
const CHAT_COPY = JSON.stringify({
version: 1,
agent: 'claude',
processIncarnation: 'pi_1',
messages: [{ id: 'm1', role: 'assistant', blocks: [{ type: 'text', text: 'x'.repeat(8_000) }] }],
limited: false,
warnings: []
})
describe('pruning settled chat output copies', () => {
let db: OrchestrationDb
afterEach(() => db.close())
function worker(
id: string,
run: string,
options: { kind?: string; status?: string; completedDaysAgo?: number } = {}
): void {
const { kind = 'structured_journal', status = 'completed', completedDaysAgo = 3 } = options
const at = `-${completedDaysAgo * 24} hours`
db.db.prepare('INSERT OR IGNORE INTO runs (id, objective) VALUES (?, ?)').run(run, 'objective')
db.db
.prepare(
`INSERT INTO dispatch_contexts (id, run_id, task_id, status, completed_at)
VALUES (?, ?, 'task', ?, CASE WHEN ? IN ('pending', 'dispatched') THEN NULL ELSE datetime('now', ?) END)`
)
.run(id, run, status, status, at)
db.db
.prepare(
`INSERT INTO worker_terminal_resources (id, origin_dispatch_id, owner_dispatch_id, terminal_handle,
ownership_state, release_state, release_completed_at)
VALUES (?, ?, ?, ?, 'released', 'released', datetime('now', ?))`
)
.run(`res_${id}`, id, id, `term_${id}`, at)
db.db
.prepare(
'INSERT INTO worker_terminal_archives (dispatch_id, resource_id, kind, content) VALUES (?, ?, ?, ?)'
)
.run(id, `res_${id}`, kind, CHAT_COPY)
}
function content(id: string): string {
return String(
db.db.prepare('SELECT content FROM worker_terminal_archives WHERE dispatch_id = ?').get(id)
?.content
)
}
it('points copies of Runs settled over a day ago at their readable chat and keeps the rest', () => {
db = new OrchestrationDb(':memory:')
worker('settled', 'run_settled')
worker('terminal', 'run_settled', { kind: 'transcript_pin' })
worker('recent', 'run_recent', { completedDaysAgo: 0.1 })
worker('live_done', 'run_live')
worker('live_working', 'run_live', { status: 'dispatched' })
worker('unreadable', 'run_settled')
expect(
pruneSettledChatOutputCopies(db, '', (dispatchId) => dispatchId !== 'unreadable')
).toEqual({ pruned: 1, next: null })
expect(content('settled')).toMatch(
/^\{"version":3,"agent":"claude","processIncarnation":"pi_1","messages":\[\],"limited":true,/
)
for (const kept of ['terminal', 'recent', 'live_done', 'live_working', 'unreadable']) {
expect(content(kept)).toBe(CHAT_COPY)
}
})
})
@@ -0,0 +1,100 @@
import type { OrchestrationDb } from '../orchestration-db'
import type { AgentType } from '../../../../../shared/agent-status/contracts/agent-status-types'
import { releaseCommitFieldsForCapturedArchive } from '../../workers/archives/worker-output-archive'
// Rows examined per page, eligible or not, so one page never scans the whole table.
const PAGE_ROWS = 32
// Each rewrite parses a copy of up to 256 KB.
const REWRITES_PER_PAGE = 4
/**
* One page of chat workers' saved output copies whose Run settled over a day ago, rewritten to
* the reference later releases store, so released reads open the chat itself. A copy whose chat
* this host cannot read stays: it is the only output left. Terminal workers' copies stay too.
*/
export function pruneSettledChatOutputCopies(
{ db }: OrchestrationDb,
after: string,
chatIsReadable: (dispatchId: string) => boolean,
// Shared across a pass's pages, so a large settled Run is checked once per pass.
settledRuns = new Map<string, boolean>()
): { pruned: number; next: string | null } {
db.exec('BEGIN IMMEDIATE')
try {
// octet_length() reads only the record header, never the saved copy.
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the projection names exactly these three columns; the adapter exposes unknown.
const rows = db
.prepare(
`SELECT a.dispatch_id, d.run_id,
a.kind = 'structured_journal' AND octet_length(a.content) > 2048
AND r.release_state = 'released'
AND julianday(r.release_completed_at) < julianday('now', '-1 day') AS candidate
FROM worker_terminal_archives a
LEFT JOIN worker_terminal_resources r ON r.id = a.resource_id
LEFT JOIN dispatch_contexts d ON d.id = a.dispatch_id
WHERE a.dispatch_id > ?
ORDER BY a.dispatch_id
LIMIT ?`
)
.all(after, PAGE_ROWS) as { dispatch_id: string; run_id: string | null; candidate: number }[]
const runIsBusy = db.prepare(
`SELECT 1 FROM dispatch_contexts
WHERE run_id = ? AND (status IN ('pending', 'dispatched')
OR julianday(completed_at) >= julianday('now', '-1 day'))
LIMIT 1`
)
const runSettled = (runId: string): boolean => {
let settled = settledRuns.get(runId)
if (settled === undefined) {
settled = !runIsBusy.get(runId)
settledRuns.set(runId, settled)
}
return settled
}
const locator = db.prepare(
`SELECT CASE WHEN json_valid(content) THEN json_extract(content, '$.agent') END AS agent,
CASE WHEN json_valid(content) THEN json_extract(content, '$.processIncarnation') END
AS process_incarnation
FROM worker_terminal_archives WHERE dispatch_id = ?`
)
const rewrite = db.prepare(
"UPDATE worker_terminal_archives SET content = ? WHERE dispatch_id = ? AND kind = 'structured_journal'"
)
let pruned = 0
let next: string | null = rows.length < PAGE_ROWS ? null : rows.at(-1)!.dispatch_id
for (const row of rows) {
if (
!row.candidate ||
row.run_id === null ||
!runSettled(row.run_id) ||
!chatIsReadable(row.dispatch_id)
) {
continue
}
const copy = locator.get(row.dispatch_id)
const { content } = releaseCommitFieldsForCapturedArchive({
kind: 'structured_journal',
status: 'captured',
content: {
version: 3,
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the copy's release wrote this field from an AgentType.
...(typeof copy?.agent === 'string' ? { agent: copy.agent as AgentType } : {}),
...(typeof copy?.process_incarnation === 'string'
? { processIncarnation: copy.process_incarnation }
: {})
}
})
rewrite.run(content, row.dispatch_id)
pruned += 1
if (pruned === REWRITES_PER_PAGE) {
next = row.dispatch_id
break
}
}
db.exec('COMMIT')
return { pruned, next }
} catch (error) {
db.exec('ROLLBACK')
throw error
}
}
@@ -39,6 +39,8 @@ class OrchestrationDbCore {
constructor(dbPath: (string & {}) | ':memory:') {
this.db = new Database(dbPath)
// Before WAL and any table, so a new file returns free pages in bounded steps; an existing one needs a VACUUM.
this.db.pragma('auto_vacuum = INCREMENTAL')
this.db.pragma('journal_mode = WAL')
this.db.pragma('synchronous = NORMAL')
this.db.pragma('busy_timeout = 5000')
@@ -1,11 +1,17 @@
import { getWorkerTerminalAutoReleaseScheduler } from '../terminal-authority/release/worker-terminal-auto-release-scheduler'
import { OrchestrationDb } from './orchestration-db'
import { OrchestrationDbMaintenance } from './maintenance/orchestration-db-maintenance'
import { structuredWorkerSession } from '../terminal-authority/structured-sessions/structured-worker-authority'
import { structuredWorkerIdentityFromRow } from '../terminal-authority/structured-sessions/structured-worker-identity'
import { isStructuredWorkerHandle } from '../../../../shared/orchestration/workers/structured-worker-handle'
import { committedHandBackTypes, type MailHandBack } from './messages/helper-mailbox-hand-back'
import { existsSync } from 'node:fs'
import { closeOrphanedTerminalApprovals } from '../workers/approvals/terminal-approval-orphans'
import { OrcaRuntimeWithAutomationApiOperations } from '../../workspaces/persisted-surfaces/automation-api-operations'
export abstract class RuntimeOrchestrationDatabaseAccess extends OrcaRuntimeWithAutomationApiOperations {
private orchestrationDbMaintenance: OrchestrationDbMaintenance | undefined
abstract ensureOrchestrationFederationRelay(
runId?: string,
options?: { keepAwake?: boolean }
@@ -19,7 +25,12 @@ export abstract class RuntimeOrchestrationDatabaseAccess extends OrcaRuntimeWith
// Defer until userData is final; tests can inject an in-memory DB.
getOrchestrationDb(): OrchestrationDb {
if (!this._orchestrationDb) {
this._orchestrationDb = new OrchestrationDb(this.orchestrationDbPath())
const db = new OrchestrationDb(this.orchestrationDbPath())
this._orchestrationDb = db
this.orchestrationDbMaintenance = new OrchestrationDbMaintenance(db, (dispatchId) =>
isReleasedChatReadable(db, dispatchId)
)
this.orchestrationDbMaintenance.start()
this.bindMailHandBack(this._orchestrationDb)
this.bindCrewChanges(this._orchestrationDb)
this.ensureOrchestrationFederationRelay()
@@ -42,6 +53,8 @@ export abstract class RuntimeOrchestrationDatabaseAccess extends OrcaRuntimeWith
}
setOrchestrationDb(db: OrchestrationDb): void {
this.orchestrationDbMaintenance?.stop()
this.orchestrationDbMaintenance = undefined
this.orchestrationFederation.resetForDatabaseChange()
this.mailPointerRepointScheduler.clear()
this._orchestrationDb = db
@@ -77,3 +90,15 @@ export abstract class RuntimeOrchestrationDatabaseAccess extends OrcaRuntimeWith
}
}
}
/** Whether a released chat worker's reference read would find its chat on this host. */
function isReleasedChatReadable(db: OrchestrationDb, dispatchId: string): boolean {
const handle =
db.getWorkerDispatch(dispatchId)?.agent_terminal_handle ??
db.getDispatchContextById(dispatchId)?.assignee_handle
const row =
handle && isStructuredWorkerHandle(handle) ? db.getWorkerTerminalResourceByHandle(handle) : null
// From the row, not the identity registry: released workers must stay unregistered.
const identity = row ? structuredWorkerIdentityFromRow(row) : null
return identity !== null && structuredWorkerSession(identity).kind === 'here'
}
@@ -240,5 +240,11 @@ CREATE TABLE IF NOT EXISTS worker_terminal_archives (
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
-- When maintenance last pruned settled history (epoch ms), so a restart waits out the interval.
CREATE TABLE IF NOT EXISTS orchestration_maintenance (
id INTEGER PRIMARY KEY CHECK(id = 1),
pruned_at INTEGER
);
`
}
@@ -236,11 +236,12 @@ export const SQLITE_TABLE_DISPOSITIONS: Record<string, WslMoveFieldDisposition>
'legacy_mail_receipts',
'legacy_operation_receipts',
'removed_orchestration_work',
'removed_orchestration_work_coverage'
'removed_orchestration_work_coverage',
'orchestration_maintenance'
],
notApplicableDisposition(
53,
'Idempotency receipts: a replay returns the bytes first answered.'
'Idempotency receipts (a replay returns the bytes first answered) and when maintenance last ran.'
)
),
...tables(