mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 19:07:41 +00:00
perf(orchestration): stop orchestration.db growth by pruning settled chat output copies (#42839)
* perf(orchestration): prune settled chat output copies and reclaim orchestration.db space Background maintenance rewrites chat workers' saved output copies of Runs settled over a day ago to the reference later releases store, every six hours at a tenth of the thread, and returns free pages in bounded steps once the file uses incremental auto_vacuum. New databases start that way; Settings > Agent orchestration can restart Orca to rewrite an existing one. * fix(orchestration): prune only copies whose chat is readable here, in bounded pages Pages now examine a fixed window of archive rows and check each Run once, so a large Run cannot make one slice quadratic. A copy whose chat this host cannot read stays. The restart rewrite keeps its temporary copy on disk and records a failure for Settings to show. * fix(orchestration): check chat readability without re-registering released workers The readability probe builds the identity from the durable row instead of rehydrating it into the live registry. Free-space checks use SQLite's own temp folder; a page's Run verdicts are shared across a pass; a failed failure record no longer hides the VACUUM error. * chore(orchestration): annotate prune casts, follow Button spacing, sync the runtime catalog * chore(i18n): resync the runtime English catalog after rebase * fix(orchestration): share Run verdicts across a prune pass; check the temp drive SQLite uses on Windows * perf(orchestration): compact legacy storage during update shutdown * test(orchestration): mock filesystem space checks through ESM factory * Align update maintenance checks with shutdown-only work * Drop the one-time update VACUUM; keep prune and incremental reclaim Existing non-incremental orchestration databases stop growing because freed pages are reused; nothing shrinks them automatically.
This commit is contained in:
@@ -0,0 +1,123 @@
|
||||
import { setImmediate, setTimeout as delay } from 'node:timers/promises'
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
import { pruneSettledChatOutputCopies } from './settled-chat-output-copy-prune'
|
||||
|
||||
const MINUTE = 60_000
|
||||
const PRUNE_INTERVAL_MS = 6 * 60 * MINUTE
|
||||
// Bound each minute's reclaim, as the chat journal does.
|
||||
const RECLAIM_PAGES_PER_PASS = 1024
|
||||
const RECLAIM_PAGES_PER_STEP = 16
|
||||
|
||||
/** Prunes settled internal history every six hours and returns free pages, at a tenth of the thread. */
|
||||
export class OrchestrationDbMaintenance {
|
||||
private readonly stopped = new AbortController()
|
||||
private timer: NodeJS.Timeout | undefined
|
||||
private work = 0
|
||||
|
||||
constructor(
|
||||
private readonly orchestrationDb: OrchestrationDb,
|
||||
private readonly chatIsReadable: (dispatchId: string) => boolean
|
||||
) {}
|
||||
|
||||
start(): void {
|
||||
this.arm()
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.stopped.abort()
|
||||
clearTimeout(this.timer)
|
||||
this.timer = undefined
|
||||
}
|
||||
|
||||
private arm(): void {
|
||||
if (this.stopped.signal.aborted) {
|
||||
return
|
||||
}
|
||||
this.timer = setTimeout(() => {
|
||||
this.timer = undefined
|
||||
void this.pass()
|
||||
.catch((error) => {
|
||||
if (!this.stopped.signal.aborted) {
|
||||
console.warn('[orchestration] database maintenance failed', error)
|
||||
}
|
||||
})
|
||||
.finally(() => this.arm())
|
||||
}, MINUTE)
|
||||
this.timer.unref()
|
||||
}
|
||||
|
||||
private async pass(): Promise<void> {
|
||||
if (this.pruneIsDue()) {
|
||||
const settledRuns = new Map<string, boolean>()
|
||||
for (let after: string | null = ''; after !== null;) {
|
||||
const page = await this.slice(() =>
|
||||
pruneSettledChatOutputCopies(
|
||||
this.orchestrationDb,
|
||||
after!,
|
||||
this.chatIsReadable,
|
||||
settledRuns
|
||||
)
|
||||
)
|
||||
if (!page) {
|
||||
return
|
||||
}
|
||||
after = page.next
|
||||
}
|
||||
// Persisted so a restart does not rescan before the interval.
|
||||
this.recordPrune()
|
||||
}
|
||||
// Only incremental files can return pages; others reuse them as they are freed.
|
||||
if (Number(this.orchestrationDb.db.pragma('auto_vacuum', { simple: true })) === 2) {
|
||||
for (let page = 0; page < RECLAIM_PAGES_PER_PASS; page += RECLAIM_PAGES_PER_STEP) {
|
||||
if (!(await this.slice(() => this.reclaimFreePages()))) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private pruneIsDue(): boolean {
|
||||
const row = this.orchestrationDb.db
|
||||
.prepare('SELECT pruned_at FROM orchestration_maintenance WHERE id = 1')
|
||||
.get()
|
||||
return !row || Date.now() - Number(row.pruned_at) >= PRUNE_INTERVAL_MS
|
||||
}
|
||||
|
||||
private recordPrune(): void {
|
||||
this.orchestrationDb.db
|
||||
.prepare(
|
||||
`INSERT INTO orchestration_maintenance (id, pruned_at) VALUES (1, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET pruned_at = excluded.pruned_at`
|
||||
)
|
||||
.run(Date.now())
|
||||
}
|
||||
|
||||
/** True while free pages remain. */
|
||||
private reclaimFreePages(): boolean {
|
||||
const free = Number(this.orchestrationDb.db.pragma('freelist_count', { simple: true }))
|
||||
if (free > 0) {
|
||||
this.orchestrationDb.db.exec(`PRAGMA incremental_vacuum(${RECLAIM_PAGES_PER_STEP})`)
|
||||
}
|
||||
return free > RECLAIM_PAGES_PER_STEP
|
||||
}
|
||||
|
||||
/** Runs one bounded step, then rests nine times its work once 25 ms accumulates. */
|
||||
private async slice<T>(step: () => T): Promise<T | undefined> {
|
||||
await setImmediate()
|
||||
if (this.stopped.signal.aborted) {
|
||||
return undefined
|
||||
}
|
||||
const started = performance.now()
|
||||
const result = step()
|
||||
this.work += performance.now() - started
|
||||
if (this.work >= 25) {
|
||||
try {
|
||||
await delay(this.work * 9, undefined, { signal: this.stopped.signal, ref: false })
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
this.work = 0
|
||||
}
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { OrchestrationDb } from '../orchestration-db'
|
||||
import { pruneSettledChatOutputCopies } from './settled-chat-output-copy-prune'
|
||||
|
||||
const CHAT_COPY = JSON.stringify({
|
||||
version: 1,
|
||||
agent: 'claude',
|
||||
processIncarnation: 'pi_1',
|
||||
messages: [{ id: 'm1', role: 'assistant', blocks: [{ type: 'text', text: 'x'.repeat(8_000) }] }],
|
||||
limited: false,
|
||||
warnings: []
|
||||
})
|
||||
|
||||
describe('pruning settled chat output copies', () => {
|
||||
let db: OrchestrationDb
|
||||
|
||||
afterEach(() => db.close())
|
||||
|
||||
function worker(
|
||||
id: string,
|
||||
run: string,
|
||||
options: { kind?: string; status?: string; completedDaysAgo?: number } = {}
|
||||
): void {
|
||||
const { kind = 'structured_journal', status = 'completed', completedDaysAgo = 3 } = options
|
||||
const at = `-${completedDaysAgo * 24} hours`
|
||||
db.db.prepare('INSERT OR IGNORE INTO runs (id, objective) VALUES (?, ?)').run(run, 'objective')
|
||||
db.db
|
||||
.prepare(
|
||||
`INSERT INTO dispatch_contexts (id, run_id, task_id, status, completed_at)
|
||||
VALUES (?, ?, 'task', ?, CASE WHEN ? IN ('pending', 'dispatched') THEN NULL ELSE datetime('now', ?) END)`
|
||||
)
|
||||
.run(id, run, status, status, at)
|
||||
db.db
|
||||
.prepare(
|
||||
`INSERT INTO worker_terminal_resources (id, origin_dispatch_id, owner_dispatch_id, terminal_handle,
|
||||
ownership_state, release_state, release_completed_at)
|
||||
VALUES (?, ?, ?, ?, 'released', 'released', datetime('now', ?))`
|
||||
)
|
||||
.run(`res_${id}`, id, id, `term_${id}`, at)
|
||||
db.db
|
||||
.prepare(
|
||||
'INSERT INTO worker_terminal_archives (dispatch_id, resource_id, kind, content) VALUES (?, ?, ?, ?)'
|
||||
)
|
||||
.run(id, `res_${id}`, kind, CHAT_COPY)
|
||||
}
|
||||
|
||||
function content(id: string): string {
|
||||
return String(
|
||||
db.db.prepare('SELECT content FROM worker_terminal_archives WHERE dispatch_id = ?').get(id)
|
||||
?.content
|
||||
)
|
||||
}
|
||||
|
||||
it('points copies of Runs settled over a day ago at their readable chat and keeps the rest', () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
worker('settled', 'run_settled')
|
||||
worker('terminal', 'run_settled', { kind: 'transcript_pin' })
|
||||
worker('recent', 'run_recent', { completedDaysAgo: 0.1 })
|
||||
worker('live_done', 'run_live')
|
||||
worker('live_working', 'run_live', { status: 'dispatched' })
|
||||
worker('unreadable', 'run_settled')
|
||||
|
||||
expect(
|
||||
pruneSettledChatOutputCopies(db, '', (dispatchId) => dispatchId !== 'unreadable')
|
||||
).toEqual({ pruned: 1, next: null })
|
||||
|
||||
expect(content('settled')).toMatch(
|
||||
/^\{"version":3,"agent":"claude","processIncarnation":"pi_1","messages":\[\],"limited":true,/
|
||||
)
|
||||
for (const kept of ['terminal', 'recent', 'live_done', 'live_working', 'unreadable']) {
|
||||
expect(content(kept)).toBe(CHAT_COPY)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,100 @@
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
import type { AgentType } from '../../../../../shared/agent-status/contracts/agent-status-types'
|
||||
import { releaseCommitFieldsForCapturedArchive } from '../../workers/archives/worker-output-archive'
|
||||
|
||||
// Rows examined per page, eligible or not, so one page never scans the whole table.
|
||||
const PAGE_ROWS = 32
|
||||
// Each rewrite parses a copy of up to 256 KB.
|
||||
const REWRITES_PER_PAGE = 4
|
||||
|
||||
/**
|
||||
* One page of chat workers' saved output copies whose Run settled over a day ago, rewritten to
|
||||
* the reference later releases store, so released reads open the chat itself. A copy whose chat
|
||||
* this host cannot read stays: it is the only output left. Terminal workers' copies stay too.
|
||||
*/
|
||||
export function pruneSettledChatOutputCopies(
|
||||
{ db }: OrchestrationDb,
|
||||
after: string,
|
||||
chatIsReadable: (dispatchId: string) => boolean,
|
||||
// Shared across a pass's pages, so a large settled Run is checked once per pass.
|
||||
settledRuns = new Map<string, boolean>()
|
||||
): { pruned: number; next: string | null } {
|
||||
db.exec('BEGIN IMMEDIATE')
|
||||
try {
|
||||
// octet_length() reads only the record header, never the saved copy.
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the projection names exactly these three columns; the adapter exposes unknown.
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT a.dispatch_id, d.run_id,
|
||||
a.kind = 'structured_journal' AND octet_length(a.content) > 2048
|
||||
AND r.release_state = 'released'
|
||||
AND julianday(r.release_completed_at) < julianday('now', '-1 day') AS candidate
|
||||
FROM worker_terminal_archives a
|
||||
LEFT JOIN worker_terminal_resources r ON r.id = a.resource_id
|
||||
LEFT JOIN dispatch_contexts d ON d.id = a.dispatch_id
|
||||
WHERE a.dispatch_id > ?
|
||||
ORDER BY a.dispatch_id
|
||||
LIMIT ?`
|
||||
)
|
||||
.all(after, PAGE_ROWS) as { dispatch_id: string; run_id: string | null; candidate: number }[]
|
||||
const runIsBusy = db.prepare(
|
||||
`SELECT 1 FROM dispatch_contexts
|
||||
WHERE run_id = ? AND (status IN ('pending', 'dispatched')
|
||||
OR julianday(completed_at) >= julianday('now', '-1 day'))
|
||||
LIMIT 1`
|
||||
)
|
||||
const runSettled = (runId: string): boolean => {
|
||||
let settled = settledRuns.get(runId)
|
||||
if (settled === undefined) {
|
||||
settled = !runIsBusy.get(runId)
|
||||
settledRuns.set(runId, settled)
|
||||
}
|
||||
return settled
|
||||
}
|
||||
const locator = db.prepare(
|
||||
`SELECT CASE WHEN json_valid(content) THEN json_extract(content, '$.agent') END AS agent,
|
||||
CASE WHEN json_valid(content) THEN json_extract(content, '$.processIncarnation') END
|
||||
AS process_incarnation
|
||||
FROM worker_terminal_archives WHERE dispatch_id = ?`
|
||||
)
|
||||
const rewrite = db.prepare(
|
||||
"UPDATE worker_terminal_archives SET content = ? WHERE dispatch_id = ? AND kind = 'structured_journal'"
|
||||
)
|
||||
let pruned = 0
|
||||
let next: string | null = rows.length < PAGE_ROWS ? null : rows.at(-1)!.dispatch_id
|
||||
for (const row of rows) {
|
||||
if (
|
||||
!row.candidate ||
|
||||
row.run_id === null ||
|
||||
!runSettled(row.run_id) ||
|
||||
!chatIsReadable(row.dispatch_id)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
const copy = locator.get(row.dispatch_id)
|
||||
const { content } = releaseCommitFieldsForCapturedArchive({
|
||||
kind: 'structured_journal',
|
||||
status: 'captured',
|
||||
content: {
|
||||
version: 3,
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the copy's release wrote this field from an AgentType.
|
||||
...(typeof copy?.agent === 'string' ? { agent: copy.agent as AgentType } : {}),
|
||||
...(typeof copy?.process_incarnation === 'string'
|
||||
? { processIncarnation: copy.process_incarnation }
|
||||
: {})
|
||||
}
|
||||
})
|
||||
rewrite.run(content, row.dispatch_id)
|
||||
pruned += 1
|
||||
if (pruned === REWRITES_PER_PAGE) {
|
||||
next = row.dispatch_id
|
||||
break
|
||||
}
|
||||
}
|
||||
db.exec('COMMIT')
|
||||
return { pruned, next }
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,8 @@ class OrchestrationDbCore {
|
||||
|
||||
constructor(dbPath: (string & {}) | ':memory:') {
|
||||
this.db = new Database(dbPath)
|
||||
// Before WAL and any table, so a new file returns free pages in bounded steps; an existing one needs a VACUUM.
|
||||
this.db.pragma('auto_vacuum = INCREMENTAL')
|
||||
this.db.pragma('journal_mode = WAL')
|
||||
this.db.pragma('synchronous = NORMAL')
|
||||
this.db.pragma('busy_timeout = 5000')
|
||||
|
||||
@@ -1,11 +1,17 @@
|
||||
import { getWorkerTerminalAutoReleaseScheduler } from '../terminal-authority/release/worker-terminal-auto-release-scheduler'
|
||||
import { OrchestrationDb } from './orchestration-db'
|
||||
import { OrchestrationDbMaintenance } from './maintenance/orchestration-db-maintenance'
|
||||
import { structuredWorkerSession } from '../terminal-authority/structured-sessions/structured-worker-authority'
|
||||
import { structuredWorkerIdentityFromRow } from '../terminal-authority/structured-sessions/structured-worker-identity'
|
||||
import { isStructuredWorkerHandle } from '../../../../shared/orchestration/workers/structured-worker-handle'
|
||||
import { committedHandBackTypes, type MailHandBack } from './messages/helper-mailbox-hand-back'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { closeOrphanedTerminalApprovals } from '../workers/approvals/terminal-approval-orphans'
|
||||
import { OrcaRuntimeWithAutomationApiOperations } from '../../workspaces/persisted-surfaces/automation-api-operations'
|
||||
|
||||
export abstract class RuntimeOrchestrationDatabaseAccess extends OrcaRuntimeWithAutomationApiOperations {
|
||||
private orchestrationDbMaintenance: OrchestrationDbMaintenance | undefined
|
||||
|
||||
abstract ensureOrchestrationFederationRelay(
|
||||
runId?: string,
|
||||
options?: { keepAwake?: boolean }
|
||||
@@ -19,7 +25,12 @@ export abstract class RuntimeOrchestrationDatabaseAccess extends OrcaRuntimeWith
|
||||
// Defer until userData is final; tests can inject an in-memory DB.
|
||||
getOrchestrationDb(): OrchestrationDb {
|
||||
if (!this._orchestrationDb) {
|
||||
this._orchestrationDb = new OrchestrationDb(this.orchestrationDbPath())
|
||||
const db = new OrchestrationDb(this.orchestrationDbPath())
|
||||
this._orchestrationDb = db
|
||||
this.orchestrationDbMaintenance = new OrchestrationDbMaintenance(db, (dispatchId) =>
|
||||
isReleasedChatReadable(db, dispatchId)
|
||||
)
|
||||
this.orchestrationDbMaintenance.start()
|
||||
this.bindMailHandBack(this._orchestrationDb)
|
||||
this.bindCrewChanges(this._orchestrationDb)
|
||||
this.ensureOrchestrationFederationRelay()
|
||||
@@ -42,6 +53,8 @@ export abstract class RuntimeOrchestrationDatabaseAccess extends OrcaRuntimeWith
|
||||
}
|
||||
|
||||
setOrchestrationDb(db: OrchestrationDb): void {
|
||||
this.orchestrationDbMaintenance?.stop()
|
||||
this.orchestrationDbMaintenance = undefined
|
||||
this.orchestrationFederation.resetForDatabaseChange()
|
||||
this.mailPointerRepointScheduler.clear()
|
||||
this._orchestrationDb = db
|
||||
@@ -77,3 +90,15 @@ export abstract class RuntimeOrchestrationDatabaseAccess extends OrcaRuntimeWith
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether a released chat worker's reference read would find its chat on this host. */
|
||||
function isReleasedChatReadable(db: OrchestrationDb, dispatchId: string): boolean {
|
||||
const handle =
|
||||
db.getWorkerDispatch(dispatchId)?.agent_terminal_handle ??
|
||||
db.getDispatchContextById(dispatchId)?.assignee_handle
|
||||
const row =
|
||||
handle && isStructuredWorkerHandle(handle) ? db.getWorkerTerminalResourceByHandle(handle) : null
|
||||
// From the row, not the identity registry: released workers must stay unregistered.
|
||||
const identity = row ? structuredWorkerIdentityFromRow(row) : null
|
||||
return identity !== null && structuredWorkerSession(identity).kind === 'here'
|
||||
}
|
||||
|
||||
@@ -240,5 +240,11 @@ CREATE TABLE IF NOT EXISTS worker_terminal_archives (
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- When maintenance last pruned settled history (epoch ms), so a restart waits out the interval.
|
||||
CREATE TABLE IF NOT EXISTS orchestration_maintenance (
|
||||
id INTEGER PRIMARY KEY CHECK(id = 1),
|
||||
pruned_at INTEGER
|
||||
);
|
||||
|
||||
`
|
||||
}
|
||||
|
||||
@@ -236,11 +236,12 @@ export const SQLITE_TABLE_DISPOSITIONS: Record<string, WslMoveFieldDisposition>
|
||||
'legacy_mail_receipts',
|
||||
'legacy_operation_receipts',
|
||||
'removed_orchestration_work',
|
||||
'removed_orchestration_work_coverage'
|
||||
'removed_orchestration_work_coverage',
|
||||
'orchestration_maintenance'
|
||||
],
|
||||
notApplicableDisposition(
|
||||
53,
|
||||
'Idempotency receipts: a replay returns the bytes first answered.'
|
||||
'Idempotency receipts (a replay returns the bytes first answered) and when maintenance last ran.'
|
||||
)
|
||||
),
|
||||
...tables(
|
||||
|
||||
Reference in New Issue
Block a user