From 6db9aabbe149ebb45c8aca0a26e3db7b9f301b4b Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 16 Nov 2026 01:01:58 -0800 Subject: [PATCH] perf(orchestration): stop orchestration.db growth by pruning settled chat output copies (#42839) * perf(orchestration): prune settled chat output copies and reclaim orchestration.db space Background maintenance rewrites chat workers' saved output copies of Runs settled over a day ago to the reference later releases store, every six hours at a tenth of the thread, and returns free pages in bounded steps once the file uses incremental auto_vacuum. New databases start that way; Settings > Agent orchestration can restart Orca to rewrite an existing one. * fix(orchestration): prune only copies whose chat is readable here, in bounded pages Pages now examine a fixed window of archive rows and check each Run once, so a large Run cannot make one slice quadratic. A copy whose chat this host cannot read stays. The restart rewrite keeps its temporary copy on disk and records a failure for Settings to show. * fix(orchestration): check chat readability without re-registering released workers The readability probe builds the identity from the durable row instead of rehydrating it into the live registry. Free-space checks use SQLite's own temp folder; a page's Run verdicts are shared across a pass; a failed failure record no longer hides the VACUUM error. * chore(orchestration): annotate prune casts, follow Button spacing, sync the runtime catalog * chore(i18n): resync the runtime English catalog after rebase * fix(orchestration): share Run verdicts across a prune pass; check the temp drive SQLite uses on Windows * perf(orchestration): compact legacy storage during update shutdown * test(orchestration): mock filesystem space checks through ESM factory * Align update maintenance checks with shutdown-only work * Drop the one-time update VACUUM; keep prune and incremental reclaim Existing non-incremental orchestration databases stop growing because freed pages are reused; nothing shrinks them automatically. --- .../orchestration-db-maintenance.ts | 123 ++++++++++++++++++ .../settled-chat-output-copy-prune.test.ts | 74 +++++++++++ .../settled-chat-output-copy-prune.ts | 100 ++++++++++++++ .../orchestration/db/orchestration-db.ts | 2 + .../runtime-orchestration-database-access.ts | 27 +++- .../db/schema/create-core-tables-sql.ts | 6 + .../wsl-move/wsl-move-storage-dispositions.ts | 5 +- 7 files changed, 334 insertions(+), 3 deletions(-) create mode 100644 src/main/runtime/orchestration/db/maintenance/orchestration-db-maintenance.ts create mode 100644 src/main/runtime/orchestration/db/maintenance/settled-chat-output-copy-prune.test.ts create mode 100644 src/main/runtime/orchestration/db/maintenance/settled-chat-output-copy-prune.ts diff --git a/src/main/runtime/orchestration/db/maintenance/orchestration-db-maintenance.ts b/src/main/runtime/orchestration/db/maintenance/orchestration-db-maintenance.ts new file mode 100644 index 00000000000..5fd3fff24c0 --- /dev/null +++ b/src/main/runtime/orchestration/db/maintenance/orchestration-db-maintenance.ts @@ -0,0 +1,123 @@ +import { setImmediate, setTimeout as delay } from 'node:timers/promises' +import type { OrchestrationDb } from '../orchestration-db' +import { pruneSettledChatOutputCopies } from './settled-chat-output-copy-prune' + +const MINUTE = 60_000 +const PRUNE_INTERVAL_MS = 6 * 60 * MINUTE +// Bound each minute's reclaim, as the chat journal does. +const RECLAIM_PAGES_PER_PASS = 1024 +const RECLAIM_PAGES_PER_STEP = 16 + +/** Prunes settled internal history every six hours and returns free pages, at a tenth of the thread. */ +export class OrchestrationDbMaintenance { + private readonly stopped = new AbortController() + private timer: NodeJS.Timeout | undefined + private work = 0 + + constructor( + private readonly orchestrationDb: OrchestrationDb, + private readonly chatIsReadable: (dispatchId: string) => boolean + ) {} + + start(): void { + this.arm() + } + + stop(): void { + this.stopped.abort() + clearTimeout(this.timer) + this.timer = undefined + } + + private arm(): void { + if (this.stopped.signal.aborted) { + return + } + this.timer = setTimeout(() => { + this.timer = undefined + void this.pass() + .catch((error) => { + if (!this.stopped.signal.aborted) { + console.warn('[orchestration] database maintenance failed', error) + } + }) + .finally(() => this.arm()) + }, MINUTE) + this.timer.unref() + } + + private async pass(): Promise { + if (this.pruneIsDue()) { + const settledRuns = new Map() + for (let after: string | null = ''; after !== null;) { + const page = await this.slice(() => + pruneSettledChatOutputCopies( + this.orchestrationDb, + after!, + this.chatIsReadable, + settledRuns + ) + ) + if (!page) { + return + } + after = page.next + } + // Persisted so a restart does not rescan before the interval. + this.recordPrune() + } + // Only incremental files can return pages; others reuse them as they are freed. + if (Number(this.orchestrationDb.db.pragma('auto_vacuum', { simple: true })) === 2) { + for (let page = 0; page < RECLAIM_PAGES_PER_PASS; page += RECLAIM_PAGES_PER_STEP) { + if (!(await this.slice(() => this.reclaimFreePages()))) { + return + } + } + } + } + + private pruneIsDue(): boolean { + const row = this.orchestrationDb.db + .prepare('SELECT pruned_at FROM orchestration_maintenance WHERE id = 1') + .get() + return !row || Date.now() - Number(row.pruned_at) >= PRUNE_INTERVAL_MS + } + + private recordPrune(): void { + this.orchestrationDb.db + .prepare( + `INSERT INTO orchestration_maintenance (id, pruned_at) VALUES (1, ?) + ON CONFLICT(id) DO UPDATE SET pruned_at = excluded.pruned_at` + ) + .run(Date.now()) + } + + /** True while free pages remain. */ + private reclaimFreePages(): boolean { + const free = Number(this.orchestrationDb.db.pragma('freelist_count', { simple: true })) + if (free > 0) { + this.orchestrationDb.db.exec(`PRAGMA incremental_vacuum(${RECLAIM_PAGES_PER_STEP})`) + } + return free > RECLAIM_PAGES_PER_STEP + } + + /** Runs one bounded step, then rests nine times its work once 25 ms accumulates. */ + private async slice(step: () => T): Promise { + await setImmediate() + if (this.stopped.signal.aborted) { + return undefined + } + const started = performance.now() + const result = step() + this.work += performance.now() - started + if (this.work >= 25) { + try { + await delay(this.work * 9, undefined, { signal: this.stopped.signal, ref: false }) + } catch { + return undefined + } + this.work = 0 + } + return result + } +} diff --git a/src/main/runtime/orchestration/db/maintenance/settled-chat-output-copy-prune.test.ts b/src/main/runtime/orchestration/db/maintenance/settled-chat-output-copy-prune.test.ts new file mode 100644 index 00000000000..f9cb1d53b48 --- /dev/null +++ b/src/main/runtime/orchestration/db/maintenance/settled-chat-output-copy-prune.test.ts @@ -0,0 +1,74 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { OrchestrationDb } from '../orchestration-db' +import { pruneSettledChatOutputCopies } from './settled-chat-output-copy-prune' + +const CHAT_COPY = JSON.stringify({ + version: 1, + agent: 'claude', + processIncarnation: 'pi_1', + messages: [{ id: 'm1', role: 'assistant', blocks: [{ type: 'text', text: 'x'.repeat(8_000) }] }], + limited: false, + warnings: [] +}) + +describe('pruning settled chat output copies', () => { + let db: OrchestrationDb + + afterEach(() => db.close()) + + function worker( + id: string, + run: string, + options: { kind?: string; status?: string; completedDaysAgo?: number } = {} + ): void { + const { kind = 'structured_journal', status = 'completed', completedDaysAgo = 3 } = options + const at = `-${completedDaysAgo * 24} hours` + db.db.prepare('INSERT OR IGNORE INTO runs (id, objective) VALUES (?, ?)').run(run, 'objective') + db.db + .prepare( + `INSERT INTO dispatch_contexts (id, run_id, task_id, status, completed_at) + VALUES (?, ?, 'task', ?, CASE WHEN ? IN ('pending', 'dispatched') THEN NULL ELSE datetime('now', ?) END)` + ) + .run(id, run, status, status, at) + db.db + .prepare( + `INSERT INTO worker_terminal_resources (id, origin_dispatch_id, owner_dispatch_id, terminal_handle, + ownership_state, release_state, release_completed_at) + VALUES (?, ?, ?, ?, 'released', 'released', datetime('now', ?))` + ) + .run(`res_${id}`, id, id, `term_${id}`, at) + db.db + .prepare( + 'INSERT INTO worker_terminal_archives (dispatch_id, resource_id, kind, content) VALUES (?, ?, ?, ?)' + ) + .run(id, `res_${id}`, kind, CHAT_COPY) + } + + function content(id: string): string { + return String( + db.db.prepare('SELECT content FROM worker_terminal_archives WHERE dispatch_id = ?').get(id) + ?.content + ) + } + + it('points copies of Runs settled over a day ago at their readable chat and keeps the rest', () => { + db = new OrchestrationDb(':memory:') + worker('settled', 'run_settled') + worker('terminal', 'run_settled', { kind: 'transcript_pin' }) + worker('recent', 'run_recent', { completedDaysAgo: 0.1 }) + worker('live_done', 'run_live') + worker('live_working', 'run_live', { status: 'dispatched' }) + worker('unreadable', 'run_settled') + + expect( + pruneSettledChatOutputCopies(db, '', (dispatchId) => dispatchId !== 'unreadable') + ).toEqual({ pruned: 1, next: null }) + + expect(content('settled')).toMatch( + /^\{"version":3,"agent":"claude","processIncarnation":"pi_1","messages":\[\],"limited":true,/ + ) + for (const kept of ['terminal', 'recent', 'live_done', 'live_working', 'unreadable']) { + expect(content(kept)).toBe(CHAT_COPY) + } + }) +}) diff --git a/src/main/runtime/orchestration/db/maintenance/settled-chat-output-copy-prune.ts b/src/main/runtime/orchestration/db/maintenance/settled-chat-output-copy-prune.ts new file mode 100644 index 00000000000..4b7acd93c1c --- /dev/null +++ b/src/main/runtime/orchestration/db/maintenance/settled-chat-output-copy-prune.ts @@ -0,0 +1,100 @@ +import type { OrchestrationDb } from '../orchestration-db' +import type { AgentType } from '../../../../../shared/agent-status/contracts/agent-status-types' +import { releaseCommitFieldsForCapturedArchive } from '../../workers/archives/worker-output-archive' + +// Rows examined per page, eligible or not, so one page never scans the whole table. +const PAGE_ROWS = 32 +// Each rewrite parses a copy of up to 256 KB. +const REWRITES_PER_PAGE = 4 + +/** + * One page of chat workers' saved output copies whose Run settled over a day ago, rewritten to + * the reference later releases store, so released reads open the chat itself. A copy whose chat + * this host cannot read stays: it is the only output left. Terminal workers' copies stay too. + */ +export function pruneSettledChatOutputCopies( + { db }: OrchestrationDb, + after: string, + chatIsReadable: (dispatchId: string) => boolean, + // Shared across a pass's pages, so a large settled Run is checked once per pass. + settledRuns = new Map() +): { pruned: number; next: string | null } { + db.exec('BEGIN IMMEDIATE') + try { + // octet_length() reads only the record header, never the saved copy. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the projection names exactly these three columns; the adapter exposes unknown. + const rows = db + .prepare( + `SELECT a.dispatch_id, d.run_id, + a.kind = 'structured_journal' AND octet_length(a.content) > 2048 + AND r.release_state = 'released' + AND julianday(r.release_completed_at) < julianday('now', '-1 day') AS candidate + FROM worker_terminal_archives a + LEFT JOIN worker_terminal_resources r ON r.id = a.resource_id + LEFT JOIN dispatch_contexts d ON d.id = a.dispatch_id + WHERE a.dispatch_id > ? + ORDER BY a.dispatch_id + LIMIT ?` + ) + .all(after, PAGE_ROWS) as { dispatch_id: string; run_id: string | null; candidate: number }[] + const runIsBusy = db.prepare( + `SELECT 1 FROM dispatch_contexts + WHERE run_id = ? AND (status IN ('pending', 'dispatched') + OR julianday(completed_at) >= julianday('now', '-1 day')) + LIMIT 1` + ) + const runSettled = (runId: string): boolean => { + let settled = settledRuns.get(runId) + if (settled === undefined) { + settled = !runIsBusy.get(runId) + settledRuns.set(runId, settled) + } + return settled + } + const locator = db.prepare( + `SELECT CASE WHEN json_valid(content) THEN json_extract(content, '$.agent') END AS agent, + CASE WHEN json_valid(content) THEN json_extract(content, '$.processIncarnation') END + AS process_incarnation + FROM worker_terminal_archives WHERE dispatch_id = ?` + ) + const rewrite = db.prepare( + "UPDATE worker_terminal_archives SET content = ? WHERE dispatch_id = ? AND kind = 'structured_journal'" + ) + let pruned = 0 + let next: string | null = rows.length < PAGE_ROWS ? null : rows.at(-1)!.dispatch_id + for (const row of rows) { + if ( + !row.candidate || + row.run_id === null || + !runSettled(row.run_id) || + !chatIsReadable(row.dispatch_id) + ) { + continue + } + const copy = locator.get(row.dispatch_id) + const { content } = releaseCommitFieldsForCapturedArchive({ + kind: 'structured_journal', + status: 'captured', + content: { + version: 3, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the copy's release wrote this field from an AgentType. + ...(typeof copy?.agent === 'string' ? { agent: copy.agent as AgentType } : {}), + ...(typeof copy?.process_incarnation === 'string' + ? { processIncarnation: copy.process_incarnation } + : {}) + } + }) + rewrite.run(content, row.dispatch_id) + pruned += 1 + if (pruned === REWRITES_PER_PAGE) { + next = row.dispatch_id + break + } + } + db.exec('COMMIT') + return { pruned, next } + } catch (error) { + db.exec('ROLLBACK') + throw error + } +} diff --git a/src/main/runtime/orchestration/db/orchestration-db.ts b/src/main/runtime/orchestration/db/orchestration-db.ts index 405cc6327f0..cbe0f9537be 100644 --- a/src/main/runtime/orchestration/db/orchestration-db.ts +++ b/src/main/runtime/orchestration/db/orchestration-db.ts @@ -39,6 +39,8 @@ class OrchestrationDbCore { constructor(dbPath: (string & {}) | ':memory:') { this.db = new Database(dbPath) + // Before WAL and any table, so a new file returns free pages in bounded steps; an existing one needs a VACUUM. + this.db.pragma('auto_vacuum = INCREMENTAL') this.db.pragma('journal_mode = WAL') this.db.pragma('synchronous = NORMAL') this.db.pragma('busy_timeout = 5000') diff --git a/src/main/runtime/orchestration/db/runtime-orchestration-database-access.ts b/src/main/runtime/orchestration/db/runtime-orchestration-database-access.ts index 2c5180064da..dc9d0cf4ca2 100644 --- a/src/main/runtime/orchestration/db/runtime-orchestration-database-access.ts +++ b/src/main/runtime/orchestration/db/runtime-orchestration-database-access.ts @@ -1,11 +1,17 @@ import { getWorkerTerminalAutoReleaseScheduler } from '../terminal-authority/release/worker-terminal-auto-release-scheduler' import { OrchestrationDb } from './orchestration-db' +import { OrchestrationDbMaintenance } from './maintenance/orchestration-db-maintenance' +import { structuredWorkerSession } from '../terminal-authority/structured-sessions/structured-worker-authority' +import { structuredWorkerIdentityFromRow } from '../terminal-authority/structured-sessions/structured-worker-identity' +import { isStructuredWorkerHandle } from '../../../../shared/orchestration/workers/structured-worker-handle' import { committedHandBackTypes, type MailHandBack } from './messages/helper-mailbox-hand-back' import { existsSync } from 'node:fs' import { closeOrphanedTerminalApprovals } from '../workers/approvals/terminal-approval-orphans' import { OrcaRuntimeWithAutomationApiOperations } from '../../workspaces/persisted-surfaces/automation-api-operations' export abstract class RuntimeOrchestrationDatabaseAccess extends OrcaRuntimeWithAutomationApiOperations { + private orchestrationDbMaintenance: OrchestrationDbMaintenance | undefined + abstract ensureOrchestrationFederationRelay( runId?: string, options?: { keepAwake?: boolean } @@ -19,7 +25,12 @@ export abstract class RuntimeOrchestrationDatabaseAccess extends OrcaRuntimeWith // Defer until userData is final; tests can inject an in-memory DB. getOrchestrationDb(): OrchestrationDb { if (!this._orchestrationDb) { - this._orchestrationDb = new OrchestrationDb(this.orchestrationDbPath()) + const db = new OrchestrationDb(this.orchestrationDbPath()) + this._orchestrationDb = db + this.orchestrationDbMaintenance = new OrchestrationDbMaintenance(db, (dispatchId) => + isReleasedChatReadable(db, dispatchId) + ) + this.orchestrationDbMaintenance.start() this.bindMailHandBack(this._orchestrationDb) this.bindCrewChanges(this._orchestrationDb) this.ensureOrchestrationFederationRelay() @@ -42,6 +53,8 @@ export abstract class RuntimeOrchestrationDatabaseAccess extends OrcaRuntimeWith } setOrchestrationDb(db: OrchestrationDb): void { + this.orchestrationDbMaintenance?.stop() + this.orchestrationDbMaintenance = undefined this.orchestrationFederation.resetForDatabaseChange() this.mailPointerRepointScheduler.clear() this._orchestrationDb = db @@ -77,3 +90,15 @@ export abstract class RuntimeOrchestrationDatabaseAccess extends OrcaRuntimeWith } } } + +/** Whether a released chat worker's reference read would find its chat on this host. */ +function isReleasedChatReadable(db: OrchestrationDb, dispatchId: string): boolean { + const handle = + db.getWorkerDispatch(dispatchId)?.agent_terminal_handle ?? + db.getDispatchContextById(dispatchId)?.assignee_handle + const row = + handle && isStructuredWorkerHandle(handle) ? db.getWorkerTerminalResourceByHandle(handle) : null + // From the row, not the identity registry: released workers must stay unregistered. + const identity = row ? structuredWorkerIdentityFromRow(row) : null + return identity !== null && structuredWorkerSession(identity).kind === 'here' +} diff --git a/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts b/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts index c57647eb27b..4bae266eed6 100644 --- a/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts +++ b/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts @@ -240,5 +240,11 @@ CREATE TABLE IF NOT EXISTS worker_terminal_archives ( created_at TEXT NOT NULL DEFAULT (datetime('now')) ); +-- When maintenance last pruned settled history (epoch ms), so a restart waits out the interval. +CREATE TABLE IF NOT EXISTS orchestration_maintenance ( + id INTEGER PRIMARY KEY CHECK(id = 1), + pruned_at INTEGER +); + ` } diff --git a/src/shared/wsl-move/wsl-move-storage-dispositions.ts b/src/shared/wsl-move/wsl-move-storage-dispositions.ts index 37034c04abc..1e171a80a31 100644 --- a/src/shared/wsl-move/wsl-move-storage-dispositions.ts +++ b/src/shared/wsl-move/wsl-move-storage-dispositions.ts @@ -236,11 +236,12 @@ export const SQLITE_TABLE_DISPOSITIONS: Record 'legacy_mail_receipts', 'legacy_operation_receipts', 'removed_orchestration_work', - 'removed_orchestration_work_coverage' + 'removed_orchestration_work_coverage', + 'orchestration_maintenance' ], notApplicableDisposition( 53, - 'Idempotency receipts: a replay returns the bytes first answered.' + 'Idempotency receipts (a replay returns the bytes first answered) and when maintenance last ran.' ) ), ...tables(