perf(agent-hooks): save agent status as per-pane SQLite rows, every few minutes (#39995)

last-status.json was rewritten whole (~190 KB) up to every 5 s while agents ran. Status now
lives in agent-hooks/last-status.db with one row per pane and per authority commitment, and a
save writes only rows whose serialized content changed. Routine saves wait 3 minutes; authority
changes still save at once, and stop, quit and hook-disable flush. Rows past the 7-day TTL are
no longer saved. An existing last-status.json (written by this or an older build) is imported
on start and then removed.
This commit is contained in:
Neil
2026-11-08 14:24:39 -08:00
committed by GitHub
parent fe40fd2576
commit 06a528f774
29 changed files with 852 additions and 387 deletions
-1
View File
@@ -2,7 +2,6 @@
# This is a RATCHET: counts may only go DOWN. Run `pnpm lint:ast-grep:flaky-tests` to list them.
# Prune after a fix: pnpm check:flaky-test-pattern-ratchet --prune
1 test-no-in-test-import-actual:src/cli/runtime/launch-serve-replaced-app-restart.test.ts
1 test-no-in-test-import-actual:src/main/agent-hooks/server/testing/persistence/server-status-persist-nonblocking.test.ts
1 test-no-in-test-import-actual:src/main/bitbucket/credential-connection.test.ts
4 test-no-in-test-import-actual:src/main/bitbucket/credential-store.test.ts
1 test-no-in-test-import-actual:src/main/bitbucket/status-no-decrypt.test.ts
+3 -2
View File
@@ -8,7 +8,8 @@
11 src/main/agent-hooks/installation/configuration/hooks-json-write.ts
6 src/main/agent-hooks/installation/scripts/locked-executable-rename.ts
9 src/main/agent-hooks/installation/scripts/managed-hook-script-write.ts
1 src/main/agent-hooks/server/server-hydration.ts
2 src/main/agent-hooks/server/last-status-legacy-file.ts
1 src/main/agent-hooks/server/last-status-saved-rows.ts
1 src/main/agent-hooks/wsl-relay/wsl-hook-relay-deps.ts
3 src/main/agent-hooks/wsl-relay/wsl-hook-relay-launch.ts
1 src/main/agent-launch/trust/agent-trust-ownership-ledger.ts
@@ -501,7 +502,7 @@
2 src/main/speech/stt-worker-paths.ts
2 src/main/sqlite/harden-database-files.ts
1 src/main/sqlite/sync-database.ts
1 src/main/sqlite/versioned-cache-database.ts
2 src/main/sqlite/versioned-cache-database.ts
1 src/main/ssh/authentication/ssh-auth-resolution.ts
2 src/main/ssh/config/ssh-config-alias-claim.ts
5 src/main/ssh/config/ssh-config-include-expander.ts
@@ -0,0 +1,106 @@
import { createHash } from 'node:crypto'
import { readFileSync, rmSync } from 'node:fs'
/**
* The JSON file older builds read and write. This build only imports it: the file is removed
* once rows holding its admitted contents have committed, and only if it still has the bytes
* that were imported, so a file another build rewrote meanwhile survives for the next start.
*/
export class LegacyLastStatusFile {
private readHash: string | undefined
private removableHash: string | undefined
/** Whether the removable bytes were imported (recorded in history) or merely rejected. */
private removableImported = false
constructor(readonly path: string) {}
/** Parsed contents and their hash, or undefined when absent or unreadable. */
read(): { document: unknown; hash: string } | undefined {
const raw = this.readRaw()
if (raw === undefined) {
return undefined
}
const hash = hashContent(raw)
try {
const document: unknown = JSON.parse(raw)
this.readHash = hash
return { document, hash }
} catch {
console.warn('[agent-hooks] last-status file is not valid JSON; ignoring')
// Unreadable either way; dropping it keeps it from shadowing the rows on every start.
this.removableHash = hash
this.removableImported = false
return undefined
}
}
/** Hydration admitted the contents read last; they may go once rows hold them. */
confirmRead(): void {
if (this.readHash !== undefined) {
this.markImported(this.readHash)
}
}
/** Hydration rejected the contents read last: remove them, without recording an import. */
discardRead(): void {
if (this.readHash !== undefined) {
this.removableHash = this.readHash
this.removableImported = false
}
}
/** Contents imported now, or earlier when the file could not be removed. */
markImported(hash: string): void {
this.removableHash = hash
this.removableImported = true
}
/** Hash to record as imported with the rows that hold it. */
get pendingImportHash(): string | undefined {
return this.removableImported ? this.removableHash : undefined
}
/** Call only after the rows holding the imported contents have committed. */
removeIfUnchanged(): void {
const expected = this.removableHash
if (expected === undefined) {
return
}
const raw = this.readRaw()
if (raw !== undefined && hashContent(raw) !== expected) {
// Rewritten by another build since import: the next start imports it instead.
this.removableHash = undefined
return
}
try {
if (raw !== undefined) {
rmSync(this.path, { force: true })
}
this.removableHash = undefined
} catch (err) {
console.warn('[agent-hooks] failed to remove migrated last-status file:', err)
}
}
reset(): void {
this.readHash = undefined
this.removableHash = undefined
this.removableImported = false
}
private readRaw(): string | undefined {
try {
return readFileSync(this.path, 'utf8')
} catch (err) {
// Why: missing file is normal (migrated or first launch); other errors degrade to the rows.
if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
console.warn('[agent-hooks] failed to read last-status file:', err)
}
return undefined
}
}
}
export function hashContent(content: string): string {
return createHash('sha256').update(content).digest('base64')
}
@@ -0,0 +1,139 @@
import { existsSync } from 'node:fs'
import SyncDatabase from '../../sqlite/sync-database'
import { isSqliteCorruption } from '../../sqlite/sqlite-read-failure'
import { removeVersionedCacheDatabase } from '../../sqlite/versioned-cache-database'
import { LAST_STATUS_FILE_VERSION } from './server-constants'
import type { LastStatusFile } from './server-types'
import { hashContent } from './last-status-legacy-file'
export type RowKind = 'entry' | 'commitment' | 'exits'
export type StoredRow = { kind: RowKind; key: string; payload: string }
export const IMPORTED_LEGACY_HASH = 'imported_legacy_hash'
// Follows LAST_STATUS_FILE_VERSION: a mismatched cache rebuilds empty, as the file did.
export const LAST_STATUS_SCHEMA_VERSION = LAST_STATUS_FILE_VERSION
export const rowId = (kind: RowKind, key: string): string => `${kind}\u0000${key}`
export type SavedRows =
| { kind: 'rows'; rows: StoredRow[]; importedLegacyHash: string | undefined }
/** Nothing usable is saved: no file, another schema, or a damaged file (now removed). */
| { kind: 'empty' }
/** Saved rows exist but cannot be read now (usually another connection's lock). */
| { kind: 'unavailable'; error: unknown }
/**
* Read the saved rows without writing: a read-only connection needs no write lock, so a
* writer elsewhere cannot block startup the way schema initialization would.
*/
export function readSavedRows(databasePath: string, busyTimeoutMs: number): SavedRows {
if (!existsSync(databasePath)) {
return { kind: 'empty' }
}
let db: SyncDatabase | null = null
try {
db = new SyncDatabase(databasePath, { readonly: true, fileMustExist: true })
db.pragma(`busy_timeout = ${busyTimeoutMs}`)
db.exec('BEGIN')
try {
const hasMeta = db
.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'rows')")
.all()
if (hasMeta.length !== 2) {
return { kind: 'empty' }
}
const version = db.prepare("SELECT value FROM meta WHERE key = 'schema_version'").get()
if (version?.value !== String(LAST_STATUS_SCHEMA_VERSION)) {
return { kind: 'empty' }
}
const marker = db.prepare('SELECT value FROM meta WHERE key = ?').get(IMPORTED_LEGACY_HASH)
return {
kind: 'rows',
rows: readRows(db),
importedLegacyHash: typeof marker?.value === 'string' ? marker.value : undefined
}
} finally {
if (db.isTransaction) {
db.exec('COMMIT')
}
}
} catch (error) {
if (isSqliteCorruption(error)) {
db?.close()
db = null
// Damaged rows hold nothing recoverable; the next write rebuilds the cache.
try {
removeVersionedCacheDatabase(databasePath, 'fail-fast')
} catch (removeError) {
// A held file stays until a later start; this session reads nothing from it.
return { kind: 'unavailable', error: removeError }
}
return { kind: 'empty' }
}
return { kind: 'unavailable', error }
} finally {
db?.close()
}
}
export function readRows(db: SyncDatabase): StoredRow[] {
return db
.prepare('SELECT kind, key, payload FROM rows')
.all()
.flatMap((row) =>
isRowKind(row.kind) && typeof row.key === 'string' && typeof row.payload === 'string'
? [{ kind: row.kind, key: row.key, payload: row.payload }]
: []
)
}
export function readDataVersion(db: SyncDatabase): unknown {
return db.prepare('PRAGMA data_version').get()?.data_version
}
export function hashRows(rows: readonly StoredRow[]): Map<string, string> {
return new Map(rows.map((row) => [rowId(row.kind, row.key), hashContent(row.payload)]))
}
export function* rowsOf(file: LastStatusFile): Generator<StoredRow> {
for (const [key, value] of Object.entries(file.entries)) {
yield { kind: 'entry', key, payload: JSON.stringify(value) }
}
for (const [key, value] of Object.entries(file.authorityCommitments ?? {})) {
yield { kind: 'commitment', key, payload: JSON.stringify(value) }
}
if (file.agentExits && file.agentExits.length > 0) {
yield { kind: 'exits', key: '', payload: JSON.stringify(file.agentExits) }
}
}
export function assemble(rows: readonly StoredRow[]): Required<LastStatusFile> {
const file: Required<LastStatusFile> = {
version: LAST_STATUS_FILE_VERSION,
entries: {},
authorityCommitments: {},
agentExits: []
}
for (const row of rows) {
let value: unknown
try {
value = JSON.parse(row.payload)
} catch {
// An unreadable row is dropped here; it is not in the saved document, so a save deletes it.
continue
}
if (row.kind === 'entry') {
Object.assign(file.entries, { [row.key]: value })
} else if (row.kind === 'commitment') {
Object.assign(file.authorityCommitments, { [row.key]: value })
} else if (Array.isArray(value)) {
file.agentExits = value
}
}
return file
}
function isRowKind(kind: unknown): kind is RowKind {
return kind === 'entry' || kind === 'commitment' || kind === 'exits'
}
@@ -0,0 +1,218 @@
import { dirname, join } from 'node:path'
import type SyncDatabase from '../../sqlite/sync-database'
import { hardenSqliteDatabaseFiles } from '../../sqlite/harden-database-files'
import { isSqliteCorruption } from '../../sqlite/sqlite-read-failure'
import {
openVersionedCacheDatabase,
removeVersionedCacheDatabase
} from '../../sqlite/versioned-cache-database'
import type { LastStatusFile } from './server-types'
import { hashContent, LegacyLastStatusFile } from './last-status-legacy-file'
import {
assemble,
hashRows,
IMPORTED_LEGACY_HASH,
LAST_STATUS_SCHEMA_VERSION,
readDataVersion,
readRows,
readSavedRows,
rowId,
rowsOf,
type StoredRow
} from './last-status-saved-rows'
/**
* Restart cache for agent status: one row per pane, per authority commitment, plus the exit
* list. A save rewrites only rows whose content changed, where the whole-file JSON it
* replaces was rewritten on every save. It runs on the main thread, so it never waits on
* another connection's lock beyond the caller's small bound; callers retry later instead.
*/
const SCHEMA = {
version: LAST_STATUS_SCHEMA_VERSION,
sql: `
CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS rows(
kind TEXT NOT NULL,
key TEXT NOT NULL,
payload TEXT NOT NULL,
PRIMARY KEY(kind, key)
) WITHOUT ROWID;
`,
// Rows hold prompts and assistant messages; keep the directory user-only.
directoryMode: 0o700,
// The main thread must not sleep through Windows unlock retries; saves retry instead.
removal: 'fail-fast' as const
}
export type SavedStatus = { document: unknown; source: 'legacy' | 'rows' }
export function lastStatusDatabasePath(legacyJsonPath: string): string {
return join(dirname(legacyJsonPath), 'last-status.db')
}
export class LastStatusStore {
private db: SyncDatabase | null = null
/** Hashes of the rows on disk as this connection last saw them, labelled by data_version. */
private baseline: Map<string, string> | undefined
private dataVersion: unknown
private savedRows: StoredRow[] | undefined
/**
* Set when the saved rows (or the import receipt) could not be read at startup. The session
* then neither writes nor deletes, so an incomplete view cannot overwrite rows it never
* read, and the legacy file stays for a later start. This also means authority commitments
* are not persisted for this session, as when the JSON file was unwritable before.
*/
private disabled = false
private readonly legacy: LegacyLastStatusFile
private readonly databasePath: string
constructor(legacyJsonPath: string) {
this.legacy = new LegacyLastStatusFile(legacyJsonPath)
this.databasePath = lastStatusDatabasePath(legacyJsonPath)
}
/** True for a session whose saved rows could not be read; it saves nothing. */
get isDisabled(): boolean {
return this.disabled
}
/** The saved document (unvalidated) and where it came from, or undefined when none. */
read(busyTimeoutMs: number): SavedStatus | undefined {
const legacy = this.legacy.read()
const saved = readSavedRows(this.databasePath, busyTimeoutMs)
if (saved.kind === 'unavailable') {
this.disabled = true
throw saved.error
}
this.savedRows = saved.kind === 'rows' ? saved.rows : []
if (legacy) {
// Only an older build writes the file, so it is newer than the rows, unless this build
// imported exactly these bytes before and could not remove them.
if (saved.kind !== 'rows' || saved.importedLegacyHash !== legacy.hash) {
return { document: legacy.document, source: 'legacy' }
}
this.legacy.markImported(legacy.hash)
}
return this.readRows()
}
/** The saved rows from the last read, for when hydration rejects a legacy file. */
readRows(): SavedStatus | undefined {
const rows = this.savedRows ?? []
return rows.length > 0 ? { document: assemble(rows), source: 'rows' } : undefined
}
/** Hydration admitted the legacy file read last; it may go once rows hold it. */
confirmLegacyImport(): void {
this.legacy.confirmRead()
}
/** Hydration rejected the legacy file; drop it after the next commit and keep the rows. */
rejectLegacy(): void {
this.legacy.discardRead()
}
/** Writes changed rows in one transaction; throws on lock or I/O failure for a later retry. */
write(file: LastStatusFile, busyTimeoutMs: number): boolean {
if (this.disabled) {
return false
}
const next = new Map<string, string>()
const rows: (StoredRow & { hash: string })[] = []
for (const row of rowsOf(file)) {
const hash = hashContent(row.payload)
next.set(rowId(row.kind, row.key), hash)
rows.push({ ...row, hash })
}
const db = this.open(busyTimeoutMs)
let wrote = false
try {
db.exec('BEGIN IMMEDIATE')
// Sampled under the write lock, so no other commit lands before COMMIT, and this
// connection's own commit leaves it unchanged. A changed version means stale hashes.
const version = readDataVersion(db)
const baseline =
this.baseline !== undefined && version === this.dataVersion
? this.baseline
: hashRows(readRows(db))
const upsert = db.prepare(
'INSERT INTO rows(kind, key, payload) VALUES (?, ?, ?) ON CONFLICT(kind, key) DO UPDATE SET payload = excluded.payload'
)
for (const row of rows) {
const id = rowId(row.kind, row.key)
if (baseline.get(id) !== row.hash) {
upsert.run(row.kind, row.key, row.payload)
wrote = true
}
}
const remove = db.prepare('DELETE FROM rows WHERE kind = ? AND key = ?')
for (const id of baseline.keys()) {
if (next.has(id)) {
continue
}
const [kind, key] = id.split('\u0000')
remove.run(kind, key)
wrote = true
}
const importHash = this.legacy.pendingImportHash
if (importHash !== undefined) {
db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)').run(
IMPORTED_LEGACY_HASH,
importHash
)
}
db.exec(wrote || importHash !== undefined ? 'COMMIT' : 'ROLLBACK')
this.dataVersion = version
this.baseline = next
} catch (error) {
if (db.isTransaction) {
db.exec('ROLLBACK')
}
this.handleFailure(error)
throw error
}
this.legacy.removeIfUnchanged()
return wrote
}
close(): void {
this.db?.close()
this.db = null
this.baseline = undefined
this.dataVersion = undefined
this.savedRows = undefined
this.disabled = false
this.legacy.reset()
}
private open(busyTimeoutMs: number): SyncDatabase {
if (this.db) {
this.db.pragma(`busy_timeout = ${busyTimeoutMs}`)
return this.db
}
const db = openVersionedCacheDatabase(this.databasePath, { ...SCHEMA, busyTimeoutMs })
hardenSqliteDatabaseFiles(this.databasePath)
this.db = db
// data_version is per connection; the first write reloads the hashes under its lock.
this.baseline = undefined
this.dataVersion = undefined
return db
}
private handleFailure(error: unknown): void {
if (!isSqliteCorruption(error)) {
return
}
// A damaged cache is rebuilt empty on the next attempt, as the helper does at open.
this.db?.close()
this.db = null
this.baseline = undefined
this.dataVersion = undefined
try {
removeVersionedCacheDatabase(this.databasePath, 'fail-fast')
} catch {
// Held by another process (Windows); the next save retries the rebuild.
}
}
}
@@ -7,8 +7,14 @@ export const CODEX_SUBAGENT_POLL_MS = 1_000
// Version 1 lacked ordering clocks; mismatched caches hydrate empty.
export const LAST_STATUS_FILE_VERSION = 2
// Throttle ordinary writes; authority changes bypass the window and quit flushes synchronously.
export const STATUS_PERSIST_MIN_INTERVAL_MS = 5_000
// Saved status only seeds the "restored, unconfirmed" view after a restart, so routine saves can
// wait minutes; authority changes bypass the window and quit, hook-disable and stop flush now.
export const STATUS_PERSIST_MIN_INTERVAL_MS = 3 * 60_000
// A failed save (often another connection's lock) retries on its own short backoff.
export const STATUS_PERSIST_RETRY_MIN_MS = 1_000
export const STATUS_PERSIST_RETRY_MAX_MS = 30_000
// Routine saves never wait on a lock; quit-time flushes wait at most this long.
export const STATUS_FLUSH_BUSY_TIMEOUT_MS = 200
export const AGENT_PROMPT_SENT_AGENT_KINDS = new Set<AgentKind>(AGENT_KIND_VALUES)
// Expire orphaned PTY rows so old work cannot resurrect on hydration.
+36 -25
View File
@@ -1,5 +1,3 @@
import { readFileSync } from 'node:fs'
import {
admitLegacyAgentStatus,
clearLegacyAgentStatuses
@@ -11,7 +9,8 @@ import {
import { seedCodexStateFromSnapshot } from '../../../shared/agent-hook-listener/providers/codex-state'
import { AGENT_STATUS_PERSISTED_HYDRATION_MODE } from '../../../shared/agent-status/transport/agent-status-legacy-adapter'
import { AGENT_STATUS_HYDRATE_MAX_AGE_MS as HYDRATE_MAX_AGE_MS } from '../../../shared/agent-status/contracts/agent-status-freshness'
import { LAST_STATUS_FILE_VERSION } from './server-constants'
import { LAST_STATUS_FILE_VERSION, STATUS_FLUSH_BUSY_TIMEOUT_MS } from './server-constants'
import type { SavedStatus } from './last-status-store'
import type { EnrichedAgentHookEventPayload, LastStatusFile } from './server-types'
import {
authorityCommitmentsMatch,
@@ -33,26 +32,43 @@ export abstract class AgentHookServerHydration extends AgentHookServerTabCleanup
clearLegacyAgentStatuses(this.state)
this.hydratedLaunchTokenHashByPaneKey.clear()
this.persistedAuthorityCommitmentsByPaneKey.clear()
let raw: string
const store = this.getStatusStore()
let saved: SavedStatus | undefined
try {
raw = readFileSync(this.lastStatusFilePath, 'utf8')
saved = store?.read(STATUS_FLUSH_BUSY_TIMEOUT_MS)
} catch (err) {
// Why: missing file is normal (first launch); other errors degrade to empty hydration + one warn.
if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
console.warn('[agent-hooks] failed to read last-status file:', err)
// Status is a restart cache; an unreadable store degrades to empty hydration + one warn.
// The store then stays read-only for this session, so rows it never read survive.
console.warn('[agent-hooks] failed to read saved agent status:', err)
return
}
if (!store || !saved) {
return
}
if (saved.source === 'legacy') {
if (this.admitSavedStatus(saved.document)) {
// Only an admitted file is recorded as imported, once rows holding it commit.
store.confirmLegacyImport()
} else {
// A rejected file must not stand in for the rows: drop it and restore those instead.
store.rejectLegacy()
const rows = store.readRows()
if (rows) {
this.admitSavedStatus(rows.document)
}
}
return
}
let parsed: unknown
try {
parsed = JSON.parse(raw)
} catch {
console.warn('[agent-hooks] last-status file is not valid JSON; ignoring')
return
} else {
this.admitSavedStatus(saved.document)
}
// Repairs and a migrated legacy file are saved now; an unchanged store writes nothing.
this.runStatusPersist()
}
/** False when the document is rejected whole; nothing from it is admitted then. */
private admitSavedStatus(parsed: unknown): boolean {
if (typeof parsed !== 'object' || parsed === null) {
console.warn('[agent-hooks] last-status file is not an object; ignoring')
return
return false
}
const file = parsed as Partial<LastStatusFile>
if (file.version !== LAST_STATUS_FILE_VERSION) {
@@ -61,12 +77,12 @@ export abstract class AgentHookServerHydration extends AgentHookServerTabCleanup
file.version
)} != ${LAST_STATUS_FILE_VERSION}); ignoring`
)
return
return false
}
const entries = file.entries
if (typeof entries !== 'object' || entries === null) {
console.warn('[agent-hooks] last-status file entries missing or wrong shape; ignoring')
return
return false
}
let hydrated = 0
let dropped = 0
@@ -163,12 +179,7 @@ export abstract class AgentHookServerHydration extends AgentHookServerTabCleanup
`[agent-hooks] last-status hydrate dropped ${dropped} entries (kept ${hydrated})`
)
}
if (dropped > 0 || prunedLegacyClaudeSubagents > 0 || scrubbedLegacyLaunchTokens > 0) {
this.runStatusPersist()
} else if (hydrated > 0) {
// Reuse raw bytes for dedup only after lossless hydration.
this.primeStatusPersistBaseline(raw)
}
return true
}
listUnconfirmedRestoredStatuses(): EnrichedAgentHookEventPayload[] {
@@ -2,7 +2,13 @@ import { createHash } from 'node:crypto'
import { isValidPaneKey } from './server-status-identity'
import { AGENT_STATUS_HYDRATE_MAX_AGE_MS as HYDRATE_MAX_AGE_MS } from '../../../shared/agent-status/contracts/agent-status-freshness'
import { LAST_STATUS_FILE_VERSION, STATUS_PERSIST_MIN_INTERVAL_MS } from './server-constants'
import {
LAST_STATUS_FILE_VERSION,
STATUS_FLUSH_BUSY_TIMEOUT_MS,
STATUS_PERSIST_MIN_INTERVAL_MS,
STATUS_PERSIST_RETRY_MAX_MS,
STATUS_PERSIST_RETRY_MIN_MS
} from './server-constants'
import type {
EnrichedAgentHookEventPayload,
LastStatusFile,
@@ -12,19 +18,25 @@ import type {
import { authorityCommitmentsMatch } from './server-persistence-validation'
import { AgentHookServerHydration } from './server-hydration'
import { pruneAgentExits, serializeAgentExits } from './agent-exit-evidence'
import { AtomicSnapshotWriter } from '../../persistence/atomic-snapshot-writer'
import { LastStatusStore } from './last-status-store'
export abstract class AgentHookServerPersistence extends AgentHookServerHydration {
private statusSnapshotWriter: AtomicSnapshotWriter | null = null
private statusSnapshotWriterPath: string | null = null
private statusStore: LastStatusStore | null = null
private statusStorePath: string | null = null
private statusPersistFailing = false
private statusPersistRetryDelayMs = 0
private statusPersistRetryAt = 0
protected serializeStatusFile(): string {
protected buildStatusFile(): LastStatusFile {
const entries: Record<string, PersistedAgentHookEventPayload> = {}
const authorityCommitments: Record<string, PersistedAgentHookAuthorityCommitment> = {}
const conflictedCommitments = new Set<string>()
// Restart drops anything older than the TTL, so saving it would only carry dead rows forward.
const ttlCutoff = Date.now() - HYDRATE_MAX_AGE_MS
for (const [paneKey, commitment] of this.persistedAuthorityCommitmentsByPaneKey) {
authorityCommitments[paneKey] = { ...commitment }
if (commitment.observedAt >= ttlCutoff) {
authorityCommitments[paneKey] = { ...commitment }
}
}
for (const [paneKey, payload] of this.state.lastStatusByPaneKey) {
// Why: never persist invalid keys (matches the hydrate-path invariant).
@@ -33,7 +45,7 @@ export abstract class AgentHookServerPersistence extends AgentHookServerHydratio
}
const enrichedPayload = payload as EnrichedAgentHookEventPayload
// Journal projections must not compete with unconfirmed copies hydrated from this cache.
if (enrichedPayload.structuredHost) {
if (enrichedPayload.structuredHost || enrichedPayload.receivedAt < ttlCutoff) {
continue
}
const {
@@ -77,17 +89,20 @@ export abstract class AgentHookServerPersistence extends AgentHookServerHydratio
}
}
// Panes without successor agents would otherwise retain exit evidence forever.
pruneAgentExits(this.agentExitedAtByPaneKey, Date.now() - HYDRATE_MAX_AGE_MS, (paneKey) =>
pruneAgentExits(this.agentExitedAtByPaneKey, ttlCutoff, (paneKey) =>
this.state.lastStatusByPaneKey.has(paneKey)
)
const agentExits = serializeAgentExits(this.agentExitedAtByPaneKey)
const file: LastStatusFile = {
// Runtime keeps an old exit for a live row; the saved copy keeps it only beside a saved row.
const agentExits = serializeAgentExits(this.agentExitedAtByPaneKey).filter(
(exit) =>
exit.at >= ttlCutoff || exit.paneKeys.some((paneKey) => Object.hasOwn(entries, paneKey))
)
return {
version: LAST_STATUS_FILE_VERSION,
entries,
authorityCommitments,
...(agentExits.length > 0 ? { agentExits } : {})
}
return JSON.stringify(file)
}
// Leading-edge throttling prevents event streams from postponing writes indefinitely (#26720).
@@ -99,12 +114,14 @@ export abstract class AgentHookServerPersistence extends AgentHookServerHydratio
// Persist authority changes immediately to preserve spool replay filtering after a crash.
const urgent = this.currentAuthorityFingerprint() !== this.lastPersistedAuthorityFingerprint
// A wall-clock rollback must not defer persistence indefinitely.
const dueAt = urgent
const throttledAt = urgent
? now
: Math.min(
this.lastStatusPersistStartedAt + STATUS_PERSIST_MIN_INTERVAL_MS,
now + STATUS_PERSIST_MIN_INTERVAL_MS
)
// After a failed save every attempt, urgent or not, waits for the short retry backoff.
const dueAt = this.statusPersistRetryAt > now ? this.statusPersistRetryAt : throttledAt
if (this.statusPersistTimer) {
if (dueAt >= this.statusPersistDueAt) {
return
@@ -114,7 +131,7 @@ export abstract class AgentHookServerPersistence extends AgentHookServerHydratio
this.statusPersistDueAt = Math.max(dueAt, now)
this.statusPersistTimer = setTimeout(() => {
this.statusPersistTimer = null
this.startStatusPersist()
this.runStatusPersist()
}, this.statusPersistDueAt - now)
// Quit flushes synchronously; this timer must not keep the runtime alive.
if (typeof this.statusPersistTimer.unref === 'function') {
@@ -122,60 +139,71 @@ export abstract class AgentHookServerPersistence extends AgentHookServerHydratio
}
}
flushStatusPersistSync(): void {
/** Quit, stop and hook-disable: one bounded lock wait, then the failure is reported. */
flushStatusPersistSync(): boolean {
this.cancelStatusPersistTimer()
if (!this.lastStatusFilePath) {
return
return true
}
this.runStatusPersist()
return this.runStatusPersist(STATUS_FLUSH_BUSY_TIMEOUT_MS)
}
/** Quit, hook-disable and hydration repair need synchronous writes; routine updates are async. */
protected runStatusPersist(): void {
const writer = this.getStatusSnapshotWriter()
if (!writer) {
return
/** Writes only changed rows; a failure (including a held lock) retries with backoff. */
protected runStatusPersist(busyTimeoutMs = 0): boolean {
const store = this.getStatusStore()
if (!store) {
return true
}
if (store.isDisabled) {
// Nothing reaches disk this session; flush callers must not read that as saved.
return false
}
this.lastStatusPersistStartedAt = Date.now()
this.lastPersistedAuthorityFingerprint = this.currentAuthorityFingerprint()
const fingerprint = this.currentAuthorityFingerprint()
try {
writer.writeSync(() => this.serializeStatusFile())
this.statusPersistFailing = false
store.write(this.buildStatusFile(), busyTimeoutMs)
} catch (err) {
this.reportStatusPersistFailure(err)
this.statusPersistRetryDelayMs = Math.min(
STATUS_PERSIST_RETRY_MAX_MS,
Math.max(STATUS_PERSIST_RETRY_MIN_MS, this.statusPersistRetryDelayMs * 2)
)
this.statusPersistRetryAt = Date.now() + this.statusPersistRetryDelayMs
// Retry even if no later mutation arrives.
this.scheduleStatusPersist()
return false
}
}
protected primeStatusPersistBaseline(onDiskJson: string): void {
this.getStatusSnapshotWriter()?.primeCommittedContent(onDiskJson)
this.lastPersistedAuthorityFingerprint = this.currentAuthorityFingerprint()
// Only a committed save can stand in for the authority it carries.
this.lastPersistedAuthorityFingerprint = fingerprint
this.statusPersistFailing = false
this.statusPersistRetryDelayMs = 0
this.statusPersistRetryAt = 0
return true
}
protected closeStatusSnapshotWriter(): void {
this.cancelStatusPersistTimer()
this.statusSnapshotWriter?.close()
this.statusSnapshotWriter = null
this.statusSnapshotWriterPath = null
this.statusStore?.close()
this.statusStore = null
this.statusStorePath = null
this.lastStatusPersistStartedAt = Number.NEGATIVE_INFINITY
this.lastPersistedAuthorityFingerprint = null
this.statusPersistFailing = false
this.statusPersistRetryDelayMs = 0
this.statusPersistRetryAt = 0
}
protected getStatusSnapshotWriter(): AtomicSnapshotWriter | null {
protected getStatusStore(): LastStatusStore | null {
const filePath = this.lastStatusFilePath
if (!filePath) {
return null
}
if (!this.statusSnapshotWriter || this.statusSnapshotWriterPath !== filePath) {
this.statusSnapshotWriter?.close()
this.statusSnapshotWriter = new AtomicSnapshotWriter(() => filePath, {
fileMode: 0o600,
directoryMode: 0o700,
skipUnchanged: true
})
this.statusSnapshotWriterPath = filePath
if (!this.statusStore || this.statusStorePath !== filePath) {
this.statusStore?.close()
this.statusStore = new LastStatusStore(filePath)
this.statusStorePath = filePath
}
return this.statusSnapshotWriter
return this.statusStore
}
/** Revision counters and live tokens avoid per-event hashing or serialization. */
@@ -191,30 +219,6 @@ export abstract class AgentHookServerPersistence extends AgentHookServerHydratio
return fingerprint
}
private startStatusPersist(): void {
const writer = this.getStatusSnapshotWriter()
if (!writer) {
return
}
this.lastStatusPersistStartedAt = Date.now()
this.lastPersistedAuthorityFingerprint = this.currentAuthorityFingerprint()
writer
.write(() => this.serializeStatusFile())
.then(
() => {
this.statusPersistFailing = false
},
(err: unknown) => {
if (this.statusSnapshotWriter !== writer) {
return
}
this.reportStatusPersistFailure(err)
// Retry even if no later mutation arrives.
this.scheduleStatusPersist()
}
)
}
private cancelStatusPersistTimer(): void {
if (this.statusPersistTimer) {
clearTimeout(this.statusPersistTimer)
@@ -224,7 +228,7 @@ export abstract class AgentHookServerPersistence extends AgentHookServerHydratio
private reportStatusPersistFailure(err: unknown): void {
if (!this.statusPersistFailing) {
console.warn('[agent-hooks] failed to write last-status file:', err)
console.warn('[agent-hooks] failed to save agent status:', err)
}
this.statusPersistFailing = true
}
+3 -3
View File
@@ -56,6 +56,7 @@ import type {
} from './server-types'
import { RevisionedMap } from './revisioned-map'
import { forgetAgentExit, type AgentExitEvidence } from './agent-exit-evidence'
import type { LastStatusStore } from './last-status-store'
export abstract class AgentHookServerState {
protected canWriteLegacyStatusRow(entry: AgentHookEventPayload): boolean {
@@ -308,10 +309,9 @@ export abstract class AgentHookServerState {
payload: AgentHookEventPayload | EnrichedAgentHookEventPayload,
launchTokenHashOverride?: string
): AgentHookAuthorityEvidence | null
protected abstract serializeStatusFile(): string
protected abstract scheduleStatusPersist(): void
protected abstract runStatusPersist(): void
protected abstract primeStatusPersistBaseline(onDiskJson: string): void
protected abstract runStatusPersist(): boolean
protected abstract getStatusStore(): LastStatusStore | null
abstract _getStateForTests(): HookListenerState
abstract _resetPromptSentDedupeForTests(): void
@@ -1,5 +1,6 @@
import { readSavedStatus } from '../persistence/last-status.test-fixtures'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { AgentHookServer, _internals } from '../../agent-hook-server'
@@ -58,7 +59,7 @@ describe('accepted lead reply', () => {
expect(replyForCurrentRow()).toBe(REPLY)
server.flushStatusPersistSync()
for (const published of [
readFileSync(join(userDataPath, 'agent-hooks', 'last-status.json'), 'utf8'),
readSavedStatus(join(userDataPath, 'agent-hooks', 'last-status.json')),
JSON.stringify(server.getStatusSnapshot())
]) {
expect(published).not.toContain('MIDDLE-MARKER')
@@ -1,4 +1,5 @@
import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
import { readSavedStatus, hasSavedStatus } from '../persistence/last-status.test-fixtures'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
@@ -80,10 +81,10 @@ describe('reminted $$ pane keys on the OMP hook pipeline', () => {
})
])
server.flushStatusPersistSync()
expect(existsSync(lastStatusPath())).toBe(true)
const file = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) as {
expect(hasSavedStatus(lastStatusPath())).toBe(true)
const file: {
entries: Record<string, { payload: { state: string } }>
}
} = JSON.parse(readSavedStatus(lastStatusPath()))
expect(file.entries[PANE]?.payload.state).toBe('done')
expect(file.entries[REMINTED]).toBeUndefined()
} finally {
@@ -1,5 +1,6 @@
import { readSavedStatus } from '../persistence/last-status.test-fixtures'
import { createHash } from 'node:crypto'
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
@@ -72,7 +73,7 @@ describe('removed-worktree foreign authority', () => {
expect(server.getCurrentAuthorityObservations()).toEqual([])
server.flushStatusPersistSync()
const file = JSON.parse(
readFileSync(join(userDataPath, 'agent-hooks', 'last-status.json'), 'utf8')
readSavedStatus(join(userDataPath, 'agent-hooks', 'last-status.json'))
)
expect(file.authorityCommitments[PANE]).toBeUndefined()
} finally {
@@ -141,7 +142,7 @@ describe('removed-worktree foreign authority', () => {
)
server.flushStatusPersistSync()
const file = JSON.parse(
readFileSync(join(userDataPath, 'agent-hooks', 'last-status.json'), 'utf8')
readSavedStatus(join(userDataPath, 'agent-hooks', 'last-status.json'))
)
expect(file.authorityCommitments[PANE]).toMatchObject({
worktreeId: KEPT,
@@ -219,7 +220,7 @@ describe('removed-worktree foreign authority', () => {
}
server.flushStatusPersistSync()
const file = JSON.parse(
readFileSync(join(userDataPath, 'agent-hooks', 'last-status.json'), 'utf8')
readSavedStatus(join(userDataPath, 'agent-hooks', 'last-status.json'))
)
expect(file.authorityCommitments[PANE]).toMatchObject({
worktreeId: KEPT,
@@ -1,5 +1,6 @@
import { readSavedStatus } from '../persistence/last-status.test-fixtures'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { AgentHookServer, _internals } from '../../agent-hook-server'
@@ -72,7 +73,7 @@ describe('AgentHookServer removed-worktree retirement', () => {
server.clearStatusEntriesForConnection('idle@box')
const persisted = () => {
server.flushStatusPersistSync()
const file = JSON.parse(readFileSync(lastStatusPath(), 'utf8'))
const file = JSON.parse(readSavedStatus(lastStatusPath()))
return {
entries: Object.keys(file.entries).sort(),
commitments: Object.keys(file.authorityCommitments ?? {})
@@ -156,7 +157,7 @@ describe('AgentHookServer removed-worktree retirement', () => {
reportLocally('done')
server.flushStatusPersistSync()
const file = JSON.parse(readFileSync(lastStatusPath(), 'utf8'))
const file = JSON.parse(readSavedStatus(lastStatusPath()))
if (localWorktree === REMOVED) {
expect(file.authorityCommitments?.[pane]).toMatchObject({ worktreeId: KEPT })
expect(file.entries[pane]).toBeUndefined()
@@ -1,4 +1,5 @@
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { readSavedStatus, hasSavedStatus } from '../persistence/last-status.test-fixtures'
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { describe, expect, it, vi } from 'vitest'
@@ -28,8 +29,8 @@ describe('startup prompt control with status hooks disabled', () => {
throw new Error('missing status persistence path')
}
server.stop()
expect(existsSync(statusPath)).toBe(true)
expect(JSON.parse(readFileSync(statusPath, 'utf8')).entries[PANE]).toMatchObject({
expect(hasSavedStatus(statusPath)).toBe(true)
expect(JSON.parse(readSavedStatus(statusPath)).entries[PANE]).toMatchObject({
paneKey: PANE,
worktreeId: 'folder-1',
payload: { state: 'done', prompt: 'terminal status survives quit', agentType: 'opencode' }
@@ -1,4 +1,5 @@
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { readSavedStatus } from '../persistence/last-status.test-fixtures'
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
@@ -104,7 +105,7 @@ describe('Claude session continued into a daemon job', () => {
expect(after?.prompt).toBe(before?.prompt)
server.flushStatusPersistSync()
const persisted = JSON.parse(
readFileSync(join(root, 'orca', 'agent-hooks', 'last-status.json'), 'utf8')
readSavedStatus(join(root, 'orca', 'agent-hooks', 'last-status.json'))
)
expect(persisted.entries[PANE].providerSession.id).toBe(FORK)
})
@@ -1,5 +1,6 @@
import { readSavedStatus } from '../persistence/last-status.test-fixtures'
import type { BrowserWindow } from 'electron'
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
@@ -257,8 +258,9 @@ it.each([257, 512])(
server.stop()
// Exact legacy representation: a helper ID with no positive observation of its own.
const path = join(dir, 'agent-hooks', 'last-status.json')
const saved = JSON.parse(readFileSync(path, 'utf8'))
const saved = JSON.parse(readSavedStatus(path))
saved.entries[PANE].agentPresence.session = id
// An older build's file wins over the saved rows on the next start.
writeFileSync(path, JSON.stringify(saved))
await restart()
await tool('different-session')
@@ -15,12 +15,6 @@ vi.mock('../../../../../shared/agent-status/process/agent-process-presence-probe
probeAgentProcessPresence: async () => 'unverifiable'
}))
class WriterObservationTestServer extends AgentHookServer {
persistedWriterState(): string {
return this.serializeStatusFile()
}
}
const servers: AgentHookServer[] = []
afterEach(() => {
for (const server of servers.splice(0)) {
@@ -28,8 +22,8 @@ afterEach(() => {
}
})
async function createServer(): Promise<WriterObservationTestServer> {
const server = new WriterObservationTestServer()
async function createServer(): Promise<AgentHookServer> {
const server = new AgentHookServer()
servers.push(server)
await server.start({ env: 'production' })
return server
@@ -120,7 +114,6 @@ describe('OpenCode writer observation', () => {
expect(JSON.stringify(value)).not.toContain('pane-owned')
expect(Object.getOwnPropertySymbols(structuredClone(value).providerSession ?? {})).toEqual([])
}
expect(server.persistedWriterState()).not.toContain('pane-owned')
for (const extra of [{}, { opencodeSharedServer: 1 }]) {
await post(extra)
expect(observation()?.topology).not.toBe('pane-owned')
@@ -1,5 +1,6 @@
import { readSavedStatus } from '../persistence/last-status.test-fixtures'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { AgentHookServer, _internals } from '../../agent-hook-server'
@@ -124,9 +125,7 @@ describe('the host-stamped turn start', () => {
expect(pushed.at(-1)).toBe(started)
server.flushStatusPersistSync()
server.stop()
const file = JSON.parse(
readFileSync(join(userDataPath, 'agent-hooks', 'last-status.json'), 'utf8')
)
const file = JSON.parse(readSavedStatus(join(userDataPath, 'agent-hooks', 'last-status.json')))
expect(file.entries[PANE].turnStartedAt).toBe(started)
const restarted = await startServer()
@@ -0,0 +1,57 @@
import { existsSync, mkdirSync } from 'node:fs'
import { dirname } from 'node:path'
import SyncDatabase from '../../../../sqlite/sync-database'
import { lastStatusDatabasePath } from '../../last-status-store'
import { LAST_STATUS_FILE_VERSION } from '../../server-constants'
/** What a restart restores, in the last-status file shape, read straight from the saved rows. */
export function readSavedStatus(lastStatusPath: string): string {
const path = lastStatusDatabasePath(lastStatusPath)
const db = new SyncDatabase(path, { readonly: true, fileMustExist: true })
try {
const entries: Record<string, unknown> = {}
const authorityCommitments: Record<string, unknown> = {}
const file: {
version: number
entries: typeof entries
authorityCommitments: typeof authorityCommitments
agentExits?: unknown
} = {
version: LAST_STATUS_FILE_VERSION,
entries,
authorityCommitments
}
for (const row of db.prepare('SELECT kind, key, payload FROM rows').all()) {
const value: unknown = JSON.parse(String(row.payload))
if (row.kind === 'entry') {
entries[String(row.key)] = value
} else if (row.kind === 'commitment') {
authorityCommitments[String(row.key)] = value
} else {
file.agentExits = value
}
}
return JSON.stringify(file)
} finally {
db.close()
}
}
export function hasSavedStatus(lastStatusPath: string): boolean {
if (!existsSync(lastStatusDatabasePath(lastStatusPath))) {
return false
}
return Object.keys(JSON.parse(readSavedStatus(lastStatusPath)).entries).length > 0
}
/** Another connection holding the write lock, as a second process would. */
export function holdSavedStatusLock(lastStatusPath: string): () => void {
mkdirSync(dirname(lastStatusPath), { recursive: true })
const db = new SyncDatabase(lastStatusDatabasePath(lastStatusPath))
db.exec('BEGIN EXCLUSIVE')
return () => {
db.exec('ROLLBACK')
db.close()
}
}
@@ -1,14 +1,6 @@
import { readSavedStatus, hasSavedStatus } from './last-status.test-fixtures'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
statSync,
utimesSync,
writeFileSync
} from 'node:fs'
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { AgentHookServer, _internals } from '../../agent-hook-server'
@@ -19,8 +11,7 @@ import {
recentTs,
PANE,
GOOD_PANE,
LEAF_2,
LEAF_5
LEAF_2
} from '../server.test-fixtures'
const { getCohortAtEmitMock, trackMock } = vi.hoisted(() => ({
@@ -124,42 +115,18 @@ describe('Last-status persistence', () => {
buildBody({ hook_event_name: 'UserPromptSubmit', prompt: 'about to drop' })
)
server.flushStatusPersistSync()
let parsed = JSON.parse(readFileSync(lastStatusPath(), 'utf8'))
let parsed = JSON.parse(readSavedStatus(lastStatusPath()))
expect(parsed.entries[PANE]).toBeTruthy()
server.clearPaneState(PANE)
server.flushStatusPersistSync()
parsed = JSON.parse(readFileSync(lastStatusPath(), 'utf8'))
parsed = JSON.parse(readSavedStatus(lastStatusPath()))
expect(parsed.entries[PANE]).toBeUndefined()
} finally {
server.stop()
}
})
it('skips a write when the serialized contents are byte-identical to the previous write', async () => {
const server = new AgentHookServer()
await server.start({
env: 'production',
userDataPath
})
try {
await postHookEvent(
server,
buildBody({ hook_event_name: 'UserPromptSubmit', prompt: 'first' })
)
server.flushStatusPersistSync()
utimesSync(lastStatusPath(), 1, 1)
const firstMtime = statSync(lastStatusPath()).mtimeMs
server.clearPaneState(makePaneKey('non-existent', LEAF_5))
server.flushStatusPersistSync()
const secondMtime = statSync(lastStatusPath()).mtimeMs
expect(secondMtime).toBe(firstMtime)
} finally {
server.stop()
}
})
it('stop() flushes pending debounced writes synchronously', async () => {
const server = new AgentHookServer()
await server.start({
@@ -175,8 +142,8 @@ describe('Last-status persistence', () => {
} finally {
server.stop()
}
expect(existsSync(lastStatusPath())).toBe(true)
const parsed = JSON.parse(readFileSync(lastStatusPath(), 'utf8'))
expect(hasSavedStatus(lastStatusPath())).toBe(true)
const parsed = JSON.parse(readSavedStatus(lastStatusPath()))
expect(parsed.entries[PANE]?.payload?.prompt).toBe('flush me')
})
})
@@ -0,0 +1,94 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { AgentHookServer, _internals } from '../../agent-hook-server'
import { recentTs, PANE } from '../server.test-fixtures'
import { holdSavedStatusLock } from './last-status.test-fixtures'
vi.mock('../../../../telemetry/client', () => ({ track: vi.fn() }))
vi.mock('../../../../telemetry/cohort-classifier', () => ({
getCohortAtEmit: () => ({ nth_repo_added: 2 })
}))
describe('saved agent status from an older build', () => {
let userDataPath: string
const lastStatusPath = (): string => join(userDataPath, 'agent-hooks', 'last-status.json')
const writeLegacyFile = (prompt: string): void => {
mkdirSync(join(userDataPath, 'agent-hooks'), { recursive: true })
writeFileSync(
lastStatusPath(),
JSON.stringify({
version: 2,
entries: {
[PANE]: {
paneKey: PANE,
tabId: 'tab-1',
receivedAt: recentTs(),
stateStartedAt: recentTs(),
payload: { state: 'done', prompt, agentType: 'claude' }
}
}
})
)
}
const restoredPrompt = async (): Promise<string | undefined> => {
const server = new AgentHookServer()
await server.start({ env: 'production', userDataPath })
try {
return server.getStatusSnapshot()[0]?.prompt
} finally {
server.stop()
}
}
beforeEach(() => {
_internals.resetCachesForTests()
userDataPath = mkdtempSync(join(tmpdir(), 'orca-last-status-migration-'))
})
afterEach(() => {
rmSync(userDataPath, { recursive: true, force: true })
})
it('moves the old file into the saved rows once and keeps restoring from them', async () => {
writeLegacyFile('from the old file')
expect(await restoredPrompt()).toBe('from the old file')
expect(existsSync(lastStatusPath())).toBe(false)
expect(await restoredPrompt()).toBe('from the old file')
})
it('leaves the old file alone for a session whose rows cannot be read', async () => {
writeLegacyFile('from the old file')
const release = holdSavedStatusLock(lastStatusPath())
vi.spyOn(console, 'warn').mockImplementation(() => {})
const server = new AgentHookServer()
await server.start({ env: 'production', userDataPath })
release()
server.stop()
expect(existsSync(lastStatusPath())).toBe(true)
expect(await restoredPrompt()).toBe('from the old file')
expect(existsSync(lastStatusPath())).toBe(false)
})
it('keeps the saved rows when an old file is rejected', async () => {
writeLegacyFile('saved in rows')
expect(await restoredPrompt()).toBe('saved in rows')
writeFileSync(lastStatusPath(), 'null')
vi.spyOn(console, 'warn').mockImplementation(() => {})
expect(await restoredPrompt()).toBe('saved in rows')
expect(existsSync(lastStatusPath())).toBe(false)
expect(await restoredPrompt()).toBe('saved in rows')
})
it('prefers a file an older build wrote after the rows were saved', async () => {
writeLegacyFile('first')
expect(await restoredPrompt()).toBe('first')
writeLegacyFile('written by an older build later')
expect(await restoredPrompt()).toBe('written by an older build later')
})
})
@@ -1,6 +1,7 @@
import { readSavedStatus, hasSavedStatus } from './last-status.test-fixtures'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { createHash } from 'node:crypto'
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { AgentHookServer, _internals } from '../../agent-hook-server'
@@ -67,8 +68,8 @@ describe('Last-status persistence', () => {
)
)
server.flushStatusPersistSync()
expect(existsSync(lastStatusPath())).toBe(true)
const file = JSON.parse(readFileSync(lastStatusPath(), 'utf8'))
expect(hasSavedStatus(lastStatusPath())).toBe(true)
const file = JSON.parse(readSavedStatus(lastStatusPath()))
expect(file.version).toBe(2)
expect(file.entries[PANE]).toMatchObject({
paneKey: PANE,
@@ -92,7 +93,7 @@ describe('Last-status persistence', () => {
state: 'done',
stateStartedAt: expect.any(Number)
})
expect(readFileSync(lastStatusPath(), 'utf8')).not.toContain('launch-bearer-must-not-persist')
expect(readSavedStatus(lastStatusPath())).not.toContain('launch-bearer-must-not-persist')
} finally {
server.stop()
}
@@ -135,7 +136,7 @@ describe('Last-status persistence', () => {
])
expect(server.getStatusSnapshotForPane(PANE)[0]?.launchToken).toBeUndefined()
const persisted = readFileSync(lastStatusPath(), 'utf8')
const persisted = readSavedStatus(lastStatusPath())
expect(persisted).not.toContain(launchToken)
expect(JSON.parse(persisted).entries[PANE]).toMatchObject({ launchTokenHash })
} finally {
@@ -192,7 +193,7 @@ describe('Last-status persistence', () => {
first.flushStatusPersistSync()
first.stop()
const afterClear = JSON.parse(readFileSync(lastStatusPath(), 'utf8'))
const afterClear = JSON.parse(readSavedStatus(lastStatusPath()))
expect(afterClear.entries).toEqual({})
expect(afterClear.authorityCommitments[PANE]).toMatchObject({
paneKey: PANE,
@@ -263,7 +264,7 @@ describe('Last-status persistence', () => {
expect(trackMock).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything())
firstServer.flushStatusPersistSync()
const file = JSON.parse(readFileSync(lastStatusPath(), 'utf8'))
const file = JSON.parse(readSavedStatus(lastStatusPath()))
expect(file.entries[PANE]).toMatchObject({
providerSessionOnly: true,
providerSession: {
@@ -1,5 +1,6 @@
import { hasSavedStatus, readSavedStatus } from './last-status.test-fixtures'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { AgentHookServer, _internals } from '../../agent-hook-server'
@@ -60,8 +61,8 @@ describe('agent status observation provenance', () => {
server.flushStatusPersistSync()
const path = join(userDataPath, 'agent-hooks', 'last-status.json')
expect(existsSync(path)).toBe(true)
const file = readFileSync(path, 'utf8')
expect(hasSavedStatus(path)).toBe(true)
const file = readSavedStatus(path)
// Process-local sequencing cannot remain ordering authority after restart.
expect(file).not.toContain('observation')
expect(JSON.parse(file).entries[PANE].observation).toBeUndefined()
@@ -1,86 +0,0 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import * as fs from 'node:fs'
import * as fsPromises from 'node:fs/promises'
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { AgentHookServer, _internals } from '../../agent-hook-server'
import { buildBody, PANE, postHookEvent } from '../server.test-fixtures'
import { AtomicSnapshotWriter } from '../../../../persistence/atomic-snapshot-writer'
type FsModule = typeof fs
type FsPromisesModule = typeof fsPromises
vi.mock('../../../../telemetry/client', () => ({ track: vi.fn() }))
vi.mock('../../../../telemetry/cohort-classifier', () => ({
getCohortAtEmit: vi.fn(() => ({ nth_repo_added: 2 }))
}))
vi.mock('node:fs', async (importOriginal) => {
const actual = await importOriginal<FsModule>()
return { ...actual, renameSync: vi.fn(actual.renameSync) }
})
vi.mock('node:fs/promises', async (importOriginal) => {
const actual = await importOriginal<FsPromisesModule>()
return { ...actual, rename: vi.fn(actual.rename) }
})
describe('last-status persistence stays off the main thread', () => {
let userDataPath: string
let server: AgentHookServer
const lastStatusPath = (): string => join(userDataPath, 'agent-hooks', 'last-status.json')
const readPrompt = (): unknown =>
JSON.parse(readFileSync(lastStatusPath(), 'utf8')).entries[PANE]?.payload?.prompt
// Exclude synchronous endpoint startup writes from status-persistence checks.
const statusRenameSyncCalls = (): number =>
vi.mocked(fs.renameSync).mock.calls.filter(([, to]) => String(to) === lastStatusPath()).length
const prompt = (text: string): Promise<Response> =>
postHookEvent(server, buildBody({ hook_event_name: 'UserPromptSubmit', prompt: text }))
beforeEach(async () => {
_internals.resetCachesForTests()
userDataPath = mkdtempSync(join(tmpdir(), 'orca-persist-nonblocking-'))
server = new AgentHookServer()
await server.start({ env: 'production', userDataPath })
})
afterEach(() => {
server.stop()
vi.mocked(fsPromises.rename).mockReset()
vi.restoreAllMocks()
rmSync(userDataPath, { recursive: true, force: true })
})
it('keeps answering queued hooks and preserves the quit flush while a rename is held open', async () => {
const actualRename = await vi.importActual<FsPromisesModule>('node:fs/promises')
let releaseRename: () => void = () => {}
const renameHeld = new Promise<void>((resolve) => {
vi.mocked(fsPromises.rename).mockImplementationOnce(async (from, to) => {
resolve()
await new Promise<void>((release) => {
releaseRename = release
})
return actualRename.rename(from, to)
})
})
vi.mocked(fsPromises.rename).mockImplementation(actualRename.rename)
const write = vi.spyOn(AtomicSnapshotWriter.prototype, 'write')
const writeSync = vi.spyOn(AtomicSnapshotWriter.prototype, 'writeSync')
expect((await prompt('first')).status).toBe(204)
await renameHeld
expect((await prompt('while renaming')).status).toBe(204)
expect((await prompt('queued while renaming')).status).toBe(204)
expect(writeSync).not.toHaveBeenCalled()
expect(statusRenameSyncCalls()).toBe(0)
// An older async rename must not undo the synchronous quit flush.
server.flushStatusPersistSync()
expect(writeSync).toHaveBeenCalledTimes(1)
expect(readPrompt()).toBe('queued while renaming')
releaseRename()
await write.mock.results[0]?.value
expect(readPrompt()).toBe('queued while renaming')
})
})
@@ -1,34 +1,23 @@
import { holdSavedStatusLock, readSavedStatus } from './last-status.test-fixtures'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { AgentHookServer, _internals } from '../../agent-hook-server'
import { STATUS_PERSIST_MIN_INTERVAL_MS } from '../../server-constants'
import { buildBody, PANE, postHookEvent } from '../server.test-fixtures'
import { AtomicSnapshotWriter } from '../../../../persistence/atomic-snapshot-writer'
vi.mock('../../../../telemetry/client', () => ({ track: vi.fn() }))
vi.mock('../../../../telemetry/cohort-classifier', () => ({
getCohortAtEmit: vi.fn(() => ({ nth_repo_added: 2 }))
}))
class ProbeServer extends AgentHookServer {
persistTimerArmed(): boolean {
return this.statusPersistTimer !== null
}
persistDueAt(): number {
return this.statusPersistDueAt
}
}
describe('last-status persist throttle', () => {
describe('saved agent status cadence', () => {
let userDataPath: string
let server: ProbeServer
let server: AgentHookServer
const lastStatusPath = (): string => join(userDataPath, 'agent-hooks', 'last-status.json')
const readPrompt = (): unknown =>
JSON.parse(readFileSync(lastStatusPath(), 'utf8')).entries[PANE]?.payload?.prompt
JSON.parse(readSavedStatus(lastStatusPath())).entries[PANE]?.payload?.prompt
const prompt = (text: string, launchToken?: string): Promise<Response> =>
postHookEvent(
server,
@@ -46,7 +35,7 @@ describe('last-status persist throttle', () => {
toFake: ['setTimeout', 'clearTimeout', 'Date']
})
userDataPath = mkdtempSync(join(tmpdir(), 'orca-persist-throttle-'))
server = new ProbeServer()
server = new AgentHookServer()
await server.start({ env: 'production', userDataPath })
})
@@ -57,113 +46,45 @@ describe('last-status persist throttle', () => {
rmSync(userDataPath, { recursive: true, force: true })
})
it('caps a steady stream at one write per window without pushing the timer back', async () => {
const write = vi.spyOn(AtomicSnapshotWriter.prototype, 'write')
let unchangedWhileArmed = 0
for (let i = 0; i < 60; i++) {
const armedBefore = server.persistTimerArmed()
const dueBefore = server.persistDueAt()
await prompt(`stream ${i}`)
if (armedBefore && server.persistTimerArmed()) {
expect(server.persistDueAt()).toBe(dueBefore)
unchangedWhileArmed++
}
if (server.persistTimerArmed()) {
expect(server.persistDueAt() - Date.now()).toBeLessThanOrEqual(
STATUS_PERSIST_MIN_INTERVAL_MS
)
}
await vi.advanceTimersByTimeAsync(500)
}
await vi.advanceTimersByTimeAsync(STATUS_PERSIST_MIN_INTERVAL_MS)
expect(write).toHaveBeenCalledTimes(7)
expect(unchangedWhileArmed).toBeGreaterThan(40)
await Promise.all(write.mock.results.map((result) => result.value))
expect(readPrompt()).toBe('stream 59')
expect(server.persistTimerArmed()).toBe(false)
})
it('writes an authority change immediately instead of waiting for the window', async () => {
const write = vi.spyOn(AtomicSnapshotWriter.prototype, 'write')
it('saves an authority change at once while a routine update waits for the window', async () => {
await prompt('first')
await vi.advanceTimersByTimeAsync(1)
expect(write).toHaveBeenCalledTimes(1)
expect(readPrompt()).toBe('first')
await vi.advanceTimersByTimeAsync(500)
await prompt('second')
expect(server.persistTimerArmed()).toBe(true)
await vi.advanceTimersByTimeAsync(1)
expect(readPrompt()).toBe('first')
await prompt('third', 'launch-token-new')
await vi.advanceTimersByTimeAsync(1)
expect(write).toHaveBeenCalledTimes(2)
await write.mock.results[1]?.value
expect(JSON.parse(readFileSync(lastStatusPath(), 'utf8')).entries[PANE]).toMatchObject({
launchTokenHash: expect.any(String)
expect(JSON.parse(readSavedStatus(lastStatusPath())).entries[PANE]).toMatchObject({
launchTokenHash: expect.any(String),
payload: expect.objectContaining({ prompt: 'third' })
})
})
it('flushStatusPersistSync cancels the armed timer and writes synchronously', async () => {
const write = vi.spyOn(AtomicSnapshotWriter.prototype, 'write')
const writeSync = vi.spyOn(AtomicSnapshotWriter.prototype, 'writeSync')
it('saves the latest status at stop and a restart keeps it', async () => {
await prompt('first')
await vi.advanceTimersByTimeAsync(1)
await write.mock.results[0]?.value
await prompt('second')
expect(server.persistTimerArmed()).toBe(true)
server.flushStatusPersistSync()
expect(server.persistTimerArmed()).toBe(false)
expect(writeSync).toHaveBeenCalledTimes(1)
expect(readPrompt()).toBe('second')
await vi.advanceTimersByTimeAsync(STATUS_PERSIST_MIN_INTERVAL_MS * 2)
expect(write).toHaveBeenCalledTimes(1)
})
it('stop then start leaves no armed timer and never overwrites the file with stale state', async () => {
const write = vi.spyOn(AtomicSnapshotWriter.prototype, 'write')
await prompt('first')
await vi.advanceTimersByTimeAsync(1)
await write.mock.results[0]?.value
await prompt('final before stop')
expect(server.persistTimerArmed()).toBe(true)
server.stop()
const afterStop = readFileSync(lastStatusPath(), 'utf8')
expect(readPrompt()).toBe('final before stop')
await server.start({ env: 'production', userDataPath })
expect(server.persistTimerArmed()).toBe(false)
await vi.advanceTimersByTimeAsync(STATUS_PERSIST_MIN_INTERVAL_MS * 2)
expect(write).toHaveBeenCalledTimes(1)
expect(readFileSync(lastStatusPath(), 'utf8')).toBe(afterStop)
await server.start({ env: 'production', userDataPath })
expect(server.getStatusSnapshot()[0]?.prompt).toBe('final before stop')
})
it('warns once for a failing async write and retries in the next window', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const write = vi
.spyOn(AtomicSnapshotWriter.prototype, 'write')
.mockRejectedValueOnce(new Error('disk full'))
.mockRejectedValueOnce(new Error('disk full'))
const persistWarnings = (): number =>
warn.mock.calls.filter(([message]) =>
String(message).includes('failed to write last-status file')
).length
it('saves an authority change within seconds once a held lock clears', async () => {
await prompt('first')
await vi.advanceTimersByTimeAsync(1)
expect(write).toHaveBeenCalledTimes(1)
expect(persistWarnings()).toBe(1)
expect(server.persistTimerArmed()).toBe(true)
const release = holdSavedStatusLock(lastStatusPath())
vi.spyOn(console, 'warn').mockImplementation(() => {})
await prompt('new owner', 'launch-token-new')
await vi.advanceTimersByTimeAsync(1)
release()
await vi.advanceTimersByTimeAsync(STATUS_PERSIST_MIN_INTERVAL_MS)
expect(write).toHaveBeenCalledTimes(2)
expect(persistWarnings()).toBe(1)
await vi.advanceTimersByTimeAsync(STATUS_PERSIST_MIN_INTERVAL_MS)
expect(write).toHaveBeenCalledTimes(3)
await write.mock.results[2]?.value
expect(readPrompt()).toBe('first')
expect(persistWarnings()).toBe(1)
await vi.advanceTimersByTimeAsync(2_000)
expect(readPrompt()).toBe('new owner')
})
})
@@ -1,3 +1,4 @@
import { readSavedStatus } from '../persistence/last-status.test-fixtures'
import { describe, expect, it } from 'vitest'
import {
appendFileSync,
@@ -138,7 +139,7 @@ describe('agent hook spool', () => {
expect(snapshot).toHaveLength(1)
expect(snapshot[0]!.subagents).toBeUndefined()
restarted.flushStatusPersistSync()
expect(readFileSync(restarted.lastStatusPath!, 'utf8')).not.toContain('isReplay')
expect(readSavedStatus(restarted.lastStatusPath!)).not.toContain('isReplay')
} finally {
restarted.stop()
}
@@ -1,6 +1,7 @@
import { readSavedStatus } from '../persistence/last-status.test-fixtures'
import { makeStructuredAgentStatusSubject } from '../../../../../shared/agent-status/contracts/agent-status-subject'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { mkdtempSync, readFileSync, rmSync, writeFileSync, mkdirSync } from 'node:fs'
import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import type { AgentSessionStatusSummary } from '../../../../../shared/agent-session/agent-session-wire'
@@ -215,9 +216,9 @@ describe('structured rows and last-status.json', () => {
server.stop()
}
const file = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) as {
const file: {
entries: Record<string, unknown>
}
} = JSON.parse(readSavedStatus(lastStatusPath()))
expect(Object.keys(file.entries)).toEqual([PANE])
const restored = new AgentHookServer()
+30 -10
View File
@@ -1,4 +1,4 @@
import { mkdirSync } from 'node:fs'
import { mkdirSync, rmSync } from 'node:fs'
import { dirname } from 'node:path'
import SyncDatabase from './sync-database'
import { removeTreeSync } from '../../shared/windows-transient-lock-removal'
@@ -14,8 +14,15 @@ export type VersionedCacheDatabaseSchema = {
directoryMode?: number
// Applies only to a new file; rows near half a default page waste the other half.
pageSize?: number
// Main-thread callers pass a short wait so a locked file fails fast instead of freezing.
busyTimeoutMs?: number
// Main-thread callers pass 'fail-fast' so a rebuild blocked by a held file throws at once
// (callers retry later) instead of sleeping through Windows unlock retries.
removal?: CacheDatabaseRemoval
}
export type CacheDatabaseRemoval = 'retry' | 'fail-fast'
/** Rebuild unreadable/stale cache schemas: source files remain authoritative, so replacement costs only a rescan instead of stranding the feature. */
export function openVersionedCacheDatabase(
path: string,
@@ -32,21 +39,21 @@ export function openVersionedCacheDatabase(
throw error
}
// One retry only: a second failure on a file we just created is not corruption.
removeVersionedCacheDatabase(path)
removeVersionedCacheDatabase(path, schema.removal)
return openExisting(path, schema)
}
}
function openExisting(path: string, schema: VersionedCacheDatabaseSchema): SyncDatabase {
// Null closed handles so cleanup cannot mask the real failure with ERR_INVALID_STATE.
let db: SyncDatabase | null = openWithPragmas(path, schema.pageSize)
let db: SyncDatabase | null = openWithPragmas(path, schema)
try {
if (isStaleSchema(db, schema.version)) {
// Unlink stale caches; dropping multi-GB tables can exceed scanner readiness deadlines.
db.close()
db = null
removeVersionedCacheDatabase(path)
db = openWithPragmas(path, schema.pageSize)
removeVersionedCacheDatabase(path, schema.removal)
db = openWithPragmas(path, schema)
}
db.exec(schema.sql)
schema.afterSchema?.(db)
@@ -65,7 +72,7 @@ function openExisting(path: string, schema: VersionedCacheDatabaseSchema): SyncD
const UNUSABLE_DATABASE =
/SQLITE_CORRUPT|SQLITE_NOTADB|file is not a database|database disk image is malformed/i
function isUnusableDatabaseError(error: unknown): boolean {
export function isUnusableDatabaseError(error: unknown): boolean {
if (!(error instanceof Error)) {
return false
}
@@ -76,11 +83,17 @@ function isUnusableDatabaseError(error: unknown): boolean {
)
}
function openWithPragmas(path: string, pageSize: number | undefined): SyncDatabase {
function openWithPragmas(
path: string,
{
pageSize,
busyTimeoutMs = 5000
}: Pick<VersionedCacheDatabaseSchema, 'pageSize' | 'busyTimeoutMs'>
): SyncDatabase {
const db = new SyncDatabase(path)
try {
// Set busy timeout before WAL pragmas so another process's lock can clear.
db.pragma('busy_timeout = 5000')
db.pragma(`busy_timeout = ${Math.max(0, Math.trunc(busyTimeoutMs))}`)
if (pageSize !== undefined) {
db.pragma(`page_size = ${pageSize}`)
}
@@ -97,12 +110,19 @@ function openWithPragmas(path: string, pageSize: number | undefined): SyncDataba
}
}
export function removeVersionedCacheDatabase(path: string): void {
export function removeVersionedCacheDatabase(
path: string,
removal: CacheDatabaseRemoval = 'retry'
): void {
if (path === ':memory:') {
return
}
for (const suffix of ['', '-wal', '-shm', '-journal']) {
removeTreeSync(`${path}${suffix}`)
if (removal === 'fail-fast') {
rmSync(`${path}${suffix}`, { recursive: true, force: true })
} else {
removeTreeSync(`${path}${suffix}`)
}
}
}
@@ -24,7 +24,11 @@ export const OUT_OF_STORE_DISPOSITIONS: readonly (WslMoveFieldDisposition & {
newManifestSection: true,
note: 'Blob; re-keyed by the Linux worktree id on the host.'
},
{ store: 'agent-hooks/last-status.json', disposition: 'rederive', item: 51 },
{
store: 'agent-hooks/last-status.db (was last-status.json)',
disposition: 'rederive',
item: 51
},
{
store: 'agent-session-journal.db',
disposition: 'not-applicable',