diff --git a/config/flaky-test-pattern-baseline.txt b/config/flaky-test-pattern-baseline.txt index 0162760916f..f94a7a2bf68 100644 --- a/config/flaky-test-pattern-baseline.txt +++ b/config/flaky-test-pattern-baseline.txt @@ -2,7 +2,6 @@ # This is a RATCHET: counts may only go DOWN. Run `pnpm lint:ast-grep:flaky-tests` to list them. # Prune after a fix: pnpm check:flaky-test-pattern-ratchet --prune 1 test-no-in-test-import-actual:src/cli/runtime/launch-serve-replaced-app-restart.test.ts -1 test-no-in-test-import-actual:src/main/agent-hooks/server/testing/persistence/server-status-persist-nonblocking.test.ts 1 test-no-in-test-import-actual:src/main/bitbucket/credential-connection.test.ts 4 test-no-in-test-import-actual:src/main/bitbucket/credential-store.test.ts 1 test-no-in-test-import-actual:src/main/bitbucket/status-no-decrypt.test.ts diff --git a/config/main-sync-fs-baseline.txt b/config/main-sync-fs-baseline.txt index f64f48d6883..ed9feeb4b41 100644 --- a/config/main-sync-fs-baseline.txt +++ b/config/main-sync-fs-baseline.txt @@ -8,7 +8,8 @@ 11 src/main/agent-hooks/installation/configuration/hooks-json-write.ts 6 src/main/agent-hooks/installation/scripts/locked-executable-rename.ts 9 src/main/agent-hooks/installation/scripts/managed-hook-script-write.ts -1 src/main/agent-hooks/server/server-hydration.ts +2 src/main/agent-hooks/server/last-status-legacy-file.ts +1 src/main/agent-hooks/server/last-status-saved-rows.ts 1 src/main/agent-hooks/wsl-relay/wsl-hook-relay-deps.ts 3 src/main/agent-hooks/wsl-relay/wsl-hook-relay-launch.ts 1 src/main/agent-launch/trust/agent-trust-ownership-ledger.ts @@ -501,7 +502,7 @@ 2 src/main/speech/stt-worker-paths.ts 2 src/main/sqlite/harden-database-files.ts 1 src/main/sqlite/sync-database.ts -1 src/main/sqlite/versioned-cache-database.ts +2 src/main/sqlite/versioned-cache-database.ts 1 src/main/ssh/authentication/ssh-auth-resolution.ts 2 src/main/ssh/config/ssh-config-alias-claim.ts 5 src/main/ssh/config/ssh-config-include-expander.ts diff --git a/src/main/agent-hooks/server/last-status-legacy-file.ts b/src/main/agent-hooks/server/last-status-legacy-file.ts new file mode 100644 index 00000000000..4917a8704e7 --- /dev/null +++ b/src/main/agent-hooks/server/last-status-legacy-file.ts @@ -0,0 +1,106 @@ +import { createHash } from 'node:crypto' +import { readFileSync, rmSync } from 'node:fs' + +/** + * The JSON file older builds read and write. This build only imports it: the file is removed + * once rows holding its admitted contents have committed, and only if it still has the bytes + * that were imported, so a file another build rewrote meanwhile survives for the next start. + */ +export class LegacyLastStatusFile { + private readHash: string | undefined + private removableHash: string | undefined + /** Whether the removable bytes were imported (recorded in history) or merely rejected. */ + private removableImported = false + + constructor(readonly path: string) {} + + /** Parsed contents and their hash, or undefined when absent or unreadable. */ + read(): { document: unknown; hash: string } | undefined { + const raw = this.readRaw() + if (raw === undefined) { + return undefined + } + const hash = hashContent(raw) + try { + const document: unknown = JSON.parse(raw) + this.readHash = hash + return { document, hash } + } catch { + console.warn('[agent-hooks] last-status file is not valid JSON; ignoring') + // Unreadable either way; dropping it keeps it from shadowing the rows on every start. + this.removableHash = hash + this.removableImported = false + return undefined + } + } + + /** Hydration admitted the contents read last; they may go once rows hold them. */ + confirmRead(): void { + if (this.readHash !== undefined) { + this.markImported(this.readHash) + } + } + + /** Hydration rejected the contents read last: remove them, without recording an import. */ + discardRead(): void { + if (this.readHash !== undefined) { + this.removableHash = this.readHash + this.removableImported = false + } + } + + /** Contents imported now, or earlier when the file could not be removed. */ + markImported(hash: string): void { + this.removableHash = hash + this.removableImported = true + } + + /** Hash to record as imported with the rows that hold it. */ + get pendingImportHash(): string | undefined { + return this.removableImported ? this.removableHash : undefined + } + + /** Call only after the rows holding the imported contents have committed. */ + removeIfUnchanged(): void { + const expected = this.removableHash + if (expected === undefined) { + return + } + const raw = this.readRaw() + if (raw !== undefined && hashContent(raw) !== expected) { + // Rewritten by another build since import: the next start imports it instead. + this.removableHash = undefined + return + } + try { + if (raw !== undefined) { + rmSync(this.path, { force: true }) + } + this.removableHash = undefined + } catch (err) { + console.warn('[agent-hooks] failed to remove migrated last-status file:', err) + } + } + + reset(): void { + this.readHash = undefined + this.removableHash = undefined + this.removableImported = false + } + + private readRaw(): string | undefined { + try { + return readFileSync(this.path, 'utf8') + } catch (err) { + // Why: missing file is normal (migrated or first launch); other errors degrade to the rows. + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') { + console.warn('[agent-hooks] failed to read last-status file:', err) + } + return undefined + } + } +} + +export function hashContent(content: string): string { + return createHash('sha256').update(content).digest('base64') +} diff --git a/src/main/agent-hooks/server/last-status-saved-rows.ts b/src/main/agent-hooks/server/last-status-saved-rows.ts new file mode 100644 index 00000000000..86a1fc6e721 --- /dev/null +++ b/src/main/agent-hooks/server/last-status-saved-rows.ts @@ -0,0 +1,139 @@ +import { existsSync } from 'node:fs' + +import SyncDatabase from '../../sqlite/sync-database' +import { isSqliteCorruption } from '../../sqlite/sqlite-read-failure' +import { removeVersionedCacheDatabase } from '../../sqlite/versioned-cache-database' +import { LAST_STATUS_FILE_VERSION } from './server-constants' +import type { LastStatusFile } from './server-types' +import { hashContent } from './last-status-legacy-file' + +export type RowKind = 'entry' | 'commitment' | 'exits' +export type StoredRow = { kind: RowKind; key: string; payload: string } + +export const IMPORTED_LEGACY_HASH = 'imported_legacy_hash' +// Follows LAST_STATUS_FILE_VERSION: a mismatched cache rebuilds empty, as the file did. +export const LAST_STATUS_SCHEMA_VERSION = LAST_STATUS_FILE_VERSION + +export const rowId = (kind: RowKind, key: string): string => `${kind}\u0000${key}` + +export type SavedRows = + | { kind: 'rows'; rows: StoredRow[]; importedLegacyHash: string | undefined } + /** Nothing usable is saved: no file, another schema, or a damaged file (now removed). */ + | { kind: 'empty' } + /** Saved rows exist but cannot be read now (usually another connection's lock). */ + | { kind: 'unavailable'; error: unknown } + +/** + * Read the saved rows without writing: a read-only connection needs no write lock, so a + * writer elsewhere cannot block startup the way schema initialization would. + */ +export function readSavedRows(databasePath: string, busyTimeoutMs: number): SavedRows { + if (!existsSync(databasePath)) { + return { kind: 'empty' } + } + let db: SyncDatabase | null = null + try { + db = new SyncDatabase(databasePath, { readonly: true, fileMustExist: true }) + db.pragma(`busy_timeout = ${busyTimeoutMs}`) + db.exec('BEGIN') + try { + const hasMeta = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'rows')") + .all() + if (hasMeta.length !== 2) { + return { kind: 'empty' } + } + const version = db.prepare("SELECT value FROM meta WHERE key = 'schema_version'").get() + if (version?.value !== String(LAST_STATUS_SCHEMA_VERSION)) { + return { kind: 'empty' } + } + const marker = db.prepare('SELECT value FROM meta WHERE key = ?').get(IMPORTED_LEGACY_HASH) + return { + kind: 'rows', + rows: readRows(db), + importedLegacyHash: typeof marker?.value === 'string' ? marker.value : undefined + } + } finally { + if (db.isTransaction) { + db.exec('COMMIT') + } + } + } catch (error) { + if (isSqliteCorruption(error)) { + db?.close() + db = null + // Damaged rows hold nothing recoverable; the next write rebuilds the cache. + try { + removeVersionedCacheDatabase(databasePath, 'fail-fast') + } catch (removeError) { + // A held file stays until a later start; this session reads nothing from it. + return { kind: 'unavailable', error: removeError } + } + return { kind: 'empty' } + } + return { kind: 'unavailable', error } + } finally { + db?.close() + } +} + +export function readRows(db: SyncDatabase): StoredRow[] { + return db + .prepare('SELECT kind, key, payload FROM rows') + .all() + .flatMap((row) => + isRowKind(row.kind) && typeof row.key === 'string' && typeof row.payload === 'string' + ? [{ kind: row.kind, key: row.key, payload: row.payload }] + : [] + ) +} + +export function readDataVersion(db: SyncDatabase): unknown { + return db.prepare('PRAGMA data_version').get()?.data_version +} + +export function hashRows(rows: readonly StoredRow[]): Map { + return new Map(rows.map((row) => [rowId(row.kind, row.key), hashContent(row.payload)])) +} + +export function* rowsOf(file: LastStatusFile): Generator { + for (const [key, value] of Object.entries(file.entries)) { + yield { kind: 'entry', key, payload: JSON.stringify(value) } + } + for (const [key, value] of Object.entries(file.authorityCommitments ?? {})) { + yield { kind: 'commitment', key, payload: JSON.stringify(value) } + } + if (file.agentExits && file.agentExits.length > 0) { + yield { kind: 'exits', key: '', payload: JSON.stringify(file.agentExits) } + } +} + +export function assemble(rows: readonly StoredRow[]): Required { + const file: Required = { + version: LAST_STATUS_FILE_VERSION, + entries: {}, + authorityCommitments: {}, + agentExits: [] + } + for (const row of rows) { + let value: unknown + try { + value = JSON.parse(row.payload) + } catch { + // An unreadable row is dropped here; it is not in the saved document, so a save deletes it. + continue + } + if (row.kind === 'entry') { + Object.assign(file.entries, { [row.key]: value }) + } else if (row.kind === 'commitment') { + Object.assign(file.authorityCommitments, { [row.key]: value }) + } else if (Array.isArray(value)) { + file.agentExits = value + } + } + return file +} + +function isRowKind(kind: unknown): kind is RowKind { + return kind === 'entry' || kind === 'commitment' || kind === 'exits' +} diff --git a/src/main/agent-hooks/server/last-status-store.ts b/src/main/agent-hooks/server/last-status-store.ts new file mode 100644 index 00000000000..28ec892d72c --- /dev/null +++ b/src/main/agent-hooks/server/last-status-store.ts @@ -0,0 +1,218 @@ +import { dirname, join } from 'node:path' + +import type SyncDatabase from '../../sqlite/sync-database' +import { hardenSqliteDatabaseFiles } from '../../sqlite/harden-database-files' +import { isSqliteCorruption } from '../../sqlite/sqlite-read-failure' +import { + openVersionedCacheDatabase, + removeVersionedCacheDatabase +} from '../../sqlite/versioned-cache-database' +import type { LastStatusFile } from './server-types' +import { hashContent, LegacyLastStatusFile } from './last-status-legacy-file' +import { + assemble, + hashRows, + IMPORTED_LEGACY_HASH, + LAST_STATUS_SCHEMA_VERSION, + readDataVersion, + readRows, + readSavedRows, + rowId, + rowsOf, + type StoredRow +} from './last-status-saved-rows' + +/** + * Restart cache for agent status: one row per pane, per authority commitment, plus the exit + * list. A save rewrites only rows whose content changed, where the whole-file JSON it + * replaces was rewritten on every save. It runs on the main thread, so it never waits on + * another connection's lock beyond the caller's small bound; callers retry later instead. + */ +const SCHEMA = { + version: LAST_STATUS_SCHEMA_VERSION, + sql: ` +CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS rows( + kind TEXT NOT NULL, + key TEXT NOT NULL, + payload TEXT NOT NULL, + PRIMARY KEY(kind, key) +) WITHOUT ROWID; +`, + // Rows hold prompts and assistant messages; keep the directory user-only. + directoryMode: 0o700, + // The main thread must not sleep through Windows unlock retries; saves retry instead. + removal: 'fail-fast' as const +} + +export type SavedStatus = { document: unknown; source: 'legacy' | 'rows' } + +export function lastStatusDatabasePath(legacyJsonPath: string): string { + return join(dirname(legacyJsonPath), 'last-status.db') +} + +export class LastStatusStore { + private db: SyncDatabase | null = null + /** Hashes of the rows on disk as this connection last saw them, labelled by data_version. */ + private baseline: Map | undefined + private dataVersion: unknown + private savedRows: StoredRow[] | undefined + /** + * Set when the saved rows (or the import receipt) could not be read at startup. The session + * then neither writes nor deletes, so an incomplete view cannot overwrite rows it never + * read, and the legacy file stays for a later start. This also means authority commitments + * are not persisted for this session, as when the JSON file was unwritable before. + */ + private disabled = false + private readonly legacy: LegacyLastStatusFile + private readonly databasePath: string + + constructor(legacyJsonPath: string) { + this.legacy = new LegacyLastStatusFile(legacyJsonPath) + this.databasePath = lastStatusDatabasePath(legacyJsonPath) + } + + /** True for a session whose saved rows could not be read; it saves nothing. */ + get isDisabled(): boolean { + return this.disabled + } + + /** The saved document (unvalidated) and where it came from, or undefined when none. */ + read(busyTimeoutMs: number): SavedStatus | undefined { + const legacy = this.legacy.read() + const saved = readSavedRows(this.databasePath, busyTimeoutMs) + if (saved.kind === 'unavailable') { + this.disabled = true + throw saved.error + } + this.savedRows = saved.kind === 'rows' ? saved.rows : [] + if (legacy) { + // Only an older build writes the file, so it is newer than the rows, unless this build + // imported exactly these bytes before and could not remove them. + if (saved.kind !== 'rows' || saved.importedLegacyHash !== legacy.hash) { + return { document: legacy.document, source: 'legacy' } + } + this.legacy.markImported(legacy.hash) + } + return this.readRows() + } + + /** The saved rows from the last read, for when hydration rejects a legacy file. */ + readRows(): SavedStatus | undefined { + const rows = this.savedRows ?? [] + return rows.length > 0 ? { document: assemble(rows), source: 'rows' } : undefined + } + + /** Hydration admitted the legacy file read last; it may go once rows hold it. */ + confirmLegacyImport(): void { + this.legacy.confirmRead() + } + + /** Hydration rejected the legacy file; drop it after the next commit and keep the rows. */ + rejectLegacy(): void { + this.legacy.discardRead() + } + + /** Writes changed rows in one transaction; throws on lock or I/O failure for a later retry. */ + write(file: LastStatusFile, busyTimeoutMs: number): boolean { + if (this.disabled) { + return false + } + const next = new Map() + const rows: (StoredRow & { hash: string })[] = [] + for (const row of rowsOf(file)) { + const hash = hashContent(row.payload) + next.set(rowId(row.kind, row.key), hash) + rows.push({ ...row, hash }) + } + const db = this.open(busyTimeoutMs) + let wrote = false + try { + db.exec('BEGIN IMMEDIATE') + // Sampled under the write lock, so no other commit lands before COMMIT, and this + // connection's own commit leaves it unchanged. A changed version means stale hashes. + const version = readDataVersion(db) + const baseline = + this.baseline !== undefined && version === this.dataVersion + ? this.baseline + : hashRows(readRows(db)) + const upsert = db.prepare( + 'INSERT INTO rows(kind, key, payload) VALUES (?, ?, ?) ON CONFLICT(kind, key) DO UPDATE SET payload = excluded.payload' + ) + for (const row of rows) { + const id = rowId(row.kind, row.key) + if (baseline.get(id) !== row.hash) { + upsert.run(row.kind, row.key, row.payload) + wrote = true + } + } + const remove = db.prepare('DELETE FROM rows WHERE kind = ? AND key = ?') + for (const id of baseline.keys()) { + if (next.has(id)) { + continue + } + const [kind, key] = id.split('\u0000') + remove.run(kind, key) + wrote = true + } + const importHash = this.legacy.pendingImportHash + if (importHash !== undefined) { + db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)').run( + IMPORTED_LEGACY_HASH, + importHash + ) + } + db.exec(wrote || importHash !== undefined ? 'COMMIT' : 'ROLLBACK') + this.dataVersion = version + this.baseline = next + } catch (error) { + if (db.isTransaction) { + db.exec('ROLLBACK') + } + this.handleFailure(error) + throw error + } + this.legacy.removeIfUnchanged() + return wrote + } + + close(): void { + this.db?.close() + this.db = null + this.baseline = undefined + this.dataVersion = undefined + this.savedRows = undefined + this.disabled = false + this.legacy.reset() + } + + private open(busyTimeoutMs: number): SyncDatabase { + if (this.db) { + this.db.pragma(`busy_timeout = ${busyTimeoutMs}`) + return this.db + } + const db = openVersionedCacheDatabase(this.databasePath, { ...SCHEMA, busyTimeoutMs }) + hardenSqliteDatabaseFiles(this.databasePath) + this.db = db + // data_version is per connection; the first write reloads the hashes under its lock. + this.baseline = undefined + this.dataVersion = undefined + return db + } + + private handleFailure(error: unknown): void { + if (!isSqliteCorruption(error)) { + return + } + // A damaged cache is rebuilt empty on the next attempt, as the helper does at open. + this.db?.close() + this.db = null + this.baseline = undefined + this.dataVersion = undefined + try { + removeVersionedCacheDatabase(this.databasePath, 'fail-fast') + } catch { + // Held by another process (Windows); the next save retries the rebuild. + } + } +} diff --git a/src/main/agent-hooks/server/server-constants.ts b/src/main/agent-hooks/server/server-constants.ts index 602c59866ac..3e0d3f17737 100644 --- a/src/main/agent-hooks/server/server-constants.ts +++ b/src/main/agent-hooks/server/server-constants.ts @@ -7,8 +7,14 @@ export const CODEX_SUBAGENT_POLL_MS = 1_000 // Version 1 lacked ordering clocks; mismatched caches hydrate empty. export const LAST_STATUS_FILE_VERSION = 2 -// Throttle ordinary writes; authority changes bypass the window and quit flushes synchronously. -export const STATUS_PERSIST_MIN_INTERVAL_MS = 5_000 +// Saved status only seeds the "restored, unconfirmed" view after a restart, so routine saves can +// wait minutes; authority changes bypass the window and quit, hook-disable and stop flush now. +export const STATUS_PERSIST_MIN_INTERVAL_MS = 3 * 60_000 +// A failed save (often another connection's lock) retries on its own short backoff. +export const STATUS_PERSIST_RETRY_MIN_MS = 1_000 +export const STATUS_PERSIST_RETRY_MAX_MS = 30_000 +// Routine saves never wait on a lock; quit-time flushes wait at most this long. +export const STATUS_FLUSH_BUSY_TIMEOUT_MS = 200 export const AGENT_PROMPT_SENT_AGENT_KINDS = new Set(AGENT_KIND_VALUES) // Expire orphaned PTY rows so old work cannot resurrect on hydration. diff --git a/src/main/agent-hooks/server/server-hydration.ts b/src/main/agent-hooks/server/server-hydration.ts index 24bbedab96d..ebabcfb7ad3 100644 --- a/src/main/agent-hooks/server/server-hydration.ts +++ b/src/main/agent-hooks/server/server-hydration.ts @@ -1,5 +1,3 @@ -import { readFileSync } from 'node:fs' - import { admitLegacyAgentStatus, clearLegacyAgentStatuses @@ -11,7 +9,8 @@ import { import { seedCodexStateFromSnapshot } from '../../../shared/agent-hook-listener/providers/codex-state' import { AGENT_STATUS_PERSISTED_HYDRATION_MODE } from '../../../shared/agent-status/transport/agent-status-legacy-adapter' import { AGENT_STATUS_HYDRATE_MAX_AGE_MS as HYDRATE_MAX_AGE_MS } from '../../../shared/agent-status/contracts/agent-status-freshness' -import { LAST_STATUS_FILE_VERSION } from './server-constants' +import { LAST_STATUS_FILE_VERSION, STATUS_FLUSH_BUSY_TIMEOUT_MS } from './server-constants' +import type { SavedStatus } from './last-status-store' import type { EnrichedAgentHookEventPayload, LastStatusFile } from './server-types' import { authorityCommitmentsMatch, @@ -33,26 +32,43 @@ export abstract class AgentHookServerHydration extends AgentHookServerTabCleanup clearLegacyAgentStatuses(this.state) this.hydratedLaunchTokenHashByPaneKey.clear() this.persistedAuthorityCommitmentsByPaneKey.clear() - let raw: string + const store = this.getStatusStore() + let saved: SavedStatus | undefined try { - raw = readFileSync(this.lastStatusFilePath, 'utf8') + saved = store?.read(STATUS_FLUSH_BUSY_TIMEOUT_MS) } catch (err) { - // Why: missing file is normal (first launch); other errors degrade to empty hydration + one warn. - if ((err as NodeJS.ErrnoException).code !== 'ENOENT') { - console.warn('[agent-hooks] failed to read last-status file:', err) + // Status is a restart cache; an unreadable store degrades to empty hydration + one warn. + // The store then stays read-only for this session, so rows it never read survive. + console.warn('[agent-hooks] failed to read saved agent status:', err) + return + } + if (!store || !saved) { + return + } + if (saved.source === 'legacy') { + if (this.admitSavedStatus(saved.document)) { + // Only an admitted file is recorded as imported, once rows holding it commit. + store.confirmLegacyImport() + } else { + // A rejected file must not stand in for the rows: drop it and restore those instead. + store.rejectLegacy() + const rows = store.readRows() + if (rows) { + this.admitSavedStatus(rows.document) + } } - return - } - let parsed: unknown - try { - parsed = JSON.parse(raw) - } catch { - console.warn('[agent-hooks] last-status file is not valid JSON; ignoring') - return + } else { + this.admitSavedStatus(saved.document) } + // Repairs and a migrated legacy file are saved now; an unchanged store writes nothing. + this.runStatusPersist() + } + + /** False when the document is rejected whole; nothing from it is admitted then. */ + private admitSavedStatus(parsed: unknown): boolean { if (typeof parsed !== 'object' || parsed === null) { console.warn('[agent-hooks] last-status file is not an object; ignoring') - return + return false } const file = parsed as Partial if (file.version !== LAST_STATUS_FILE_VERSION) { @@ -61,12 +77,12 @@ export abstract class AgentHookServerHydration extends AgentHookServerTabCleanup file.version )} != ${LAST_STATUS_FILE_VERSION}); ignoring` ) - return + return false } const entries = file.entries if (typeof entries !== 'object' || entries === null) { console.warn('[agent-hooks] last-status file entries missing or wrong shape; ignoring') - return + return false } let hydrated = 0 let dropped = 0 @@ -163,12 +179,7 @@ export abstract class AgentHookServerHydration extends AgentHookServerTabCleanup `[agent-hooks] last-status hydrate dropped ${dropped} entries (kept ${hydrated})` ) } - if (dropped > 0 || prunedLegacyClaudeSubagents > 0 || scrubbedLegacyLaunchTokens > 0) { - this.runStatusPersist() - } else if (hydrated > 0) { - // Reuse raw bytes for dedup only after lossless hydration. - this.primeStatusPersistBaseline(raw) - } + return true } listUnconfirmedRestoredStatuses(): EnrichedAgentHookEventPayload[] { diff --git a/src/main/agent-hooks/server/server-persistence.ts b/src/main/agent-hooks/server/server-persistence.ts index c3767e2ffb8..5e2751dcf30 100644 --- a/src/main/agent-hooks/server/server-persistence.ts +++ b/src/main/agent-hooks/server/server-persistence.ts @@ -2,7 +2,13 @@ import { createHash } from 'node:crypto' import { isValidPaneKey } from './server-status-identity' import { AGENT_STATUS_HYDRATE_MAX_AGE_MS as HYDRATE_MAX_AGE_MS } from '../../../shared/agent-status/contracts/agent-status-freshness' -import { LAST_STATUS_FILE_VERSION, STATUS_PERSIST_MIN_INTERVAL_MS } from './server-constants' +import { + LAST_STATUS_FILE_VERSION, + STATUS_FLUSH_BUSY_TIMEOUT_MS, + STATUS_PERSIST_MIN_INTERVAL_MS, + STATUS_PERSIST_RETRY_MAX_MS, + STATUS_PERSIST_RETRY_MIN_MS +} from './server-constants' import type { EnrichedAgentHookEventPayload, LastStatusFile, @@ -12,19 +18,25 @@ import type { import { authorityCommitmentsMatch } from './server-persistence-validation' import { AgentHookServerHydration } from './server-hydration' import { pruneAgentExits, serializeAgentExits } from './agent-exit-evidence' -import { AtomicSnapshotWriter } from '../../persistence/atomic-snapshot-writer' +import { LastStatusStore } from './last-status-store' export abstract class AgentHookServerPersistence extends AgentHookServerHydration { - private statusSnapshotWriter: AtomicSnapshotWriter | null = null - private statusSnapshotWriterPath: string | null = null + private statusStore: LastStatusStore | null = null + private statusStorePath: string | null = null private statusPersistFailing = false + private statusPersistRetryDelayMs = 0 + private statusPersistRetryAt = 0 - protected serializeStatusFile(): string { + protected buildStatusFile(): LastStatusFile { const entries: Record = {} const authorityCommitments: Record = {} const conflictedCommitments = new Set() + // Restart drops anything older than the TTL, so saving it would only carry dead rows forward. + const ttlCutoff = Date.now() - HYDRATE_MAX_AGE_MS for (const [paneKey, commitment] of this.persistedAuthorityCommitmentsByPaneKey) { - authorityCommitments[paneKey] = { ...commitment } + if (commitment.observedAt >= ttlCutoff) { + authorityCommitments[paneKey] = { ...commitment } + } } for (const [paneKey, payload] of this.state.lastStatusByPaneKey) { // Why: never persist invalid keys (matches the hydrate-path invariant). @@ -33,7 +45,7 @@ export abstract class AgentHookServerPersistence extends AgentHookServerHydratio } const enrichedPayload = payload as EnrichedAgentHookEventPayload // Journal projections must not compete with unconfirmed copies hydrated from this cache. - if (enrichedPayload.structuredHost) { + if (enrichedPayload.structuredHost || enrichedPayload.receivedAt < ttlCutoff) { continue } const { @@ -77,17 +89,20 @@ export abstract class AgentHookServerPersistence extends AgentHookServerHydratio } } // Panes without successor agents would otherwise retain exit evidence forever. - pruneAgentExits(this.agentExitedAtByPaneKey, Date.now() - HYDRATE_MAX_AGE_MS, (paneKey) => + pruneAgentExits(this.agentExitedAtByPaneKey, ttlCutoff, (paneKey) => this.state.lastStatusByPaneKey.has(paneKey) ) - const agentExits = serializeAgentExits(this.agentExitedAtByPaneKey) - const file: LastStatusFile = { + // Runtime keeps an old exit for a live row; the saved copy keeps it only beside a saved row. + const agentExits = serializeAgentExits(this.agentExitedAtByPaneKey).filter( + (exit) => + exit.at >= ttlCutoff || exit.paneKeys.some((paneKey) => Object.hasOwn(entries, paneKey)) + ) + return { version: LAST_STATUS_FILE_VERSION, entries, authorityCommitments, ...(agentExits.length > 0 ? { agentExits } : {}) } - return JSON.stringify(file) } // Leading-edge throttling prevents event streams from postponing writes indefinitely (#26720). @@ -99,12 +114,14 @@ export abstract class AgentHookServerPersistence extends AgentHookServerHydratio // Persist authority changes immediately to preserve spool replay filtering after a crash. const urgent = this.currentAuthorityFingerprint() !== this.lastPersistedAuthorityFingerprint // A wall-clock rollback must not defer persistence indefinitely. - const dueAt = urgent + const throttledAt = urgent ? now : Math.min( this.lastStatusPersistStartedAt + STATUS_PERSIST_MIN_INTERVAL_MS, now + STATUS_PERSIST_MIN_INTERVAL_MS ) + // After a failed save every attempt, urgent or not, waits for the short retry backoff. + const dueAt = this.statusPersistRetryAt > now ? this.statusPersistRetryAt : throttledAt if (this.statusPersistTimer) { if (dueAt >= this.statusPersistDueAt) { return @@ -114,7 +131,7 @@ export abstract class AgentHookServerPersistence extends AgentHookServerHydratio this.statusPersistDueAt = Math.max(dueAt, now) this.statusPersistTimer = setTimeout(() => { this.statusPersistTimer = null - this.startStatusPersist() + this.runStatusPersist() }, this.statusPersistDueAt - now) // Quit flushes synchronously; this timer must not keep the runtime alive. if (typeof this.statusPersistTimer.unref === 'function') { @@ -122,60 +139,71 @@ export abstract class AgentHookServerPersistence extends AgentHookServerHydratio } } - flushStatusPersistSync(): void { + /** Quit, stop and hook-disable: one bounded lock wait, then the failure is reported. */ + flushStatusPersistSync(): boolean { this.cancelStatusPersistTimer() if (!this.lastStatusFilePath) { - return + return true } - this.runStatusPersist() + return this.runStatusPersist(STATUS_FLUSH_BUSY_TIMEOUT_MS) } - /** Quit, hook-disable and hydration repair need synchronous writes; routine updates are async. */ - protected runStatusPersist(): void { - const writer = this.getStatusSnapshotWriter() - if (!writer) { - return + /** Writes only changed rows; a failure (including a held lock) retries with backoff. */ + protected runStatusPersist(busyTimeoutMs = 0): boolean { + const store = this.getStatusStore() + if (!store) { + return true + } + if (store.isDisabled) { + // Nothing reaches disk this session; flush callers must not read that as saved. + return false } this.lastStatusPersistStartedAt = Date.now() - this.lastPersistedAuthorityFingerprint = this.currentAuthorityFingerprint() + const fingerprint = this.currentAuthorityFingerprint() try { - writer.writeSync(() => this.serializeStatusFile()) - this.statusPersistFailing = false + store.write(this.buildStatusFile(), busyTimeoutMs) } catch (err) { this.reportStatusPersistFailure(err) + this.statusPersistRetryDelayMs = Math.min( + STATUS_PERSIST_RETRY_MAX_MS, + Math.max(STATUS_PERSIST_RETRY_MIN_MS, this.statusPersistRetryDelayMs * 2) + ) + this.statusPersistRetryAt = Date.now() + this.statusPersistRetryDelayMs + // Retry even if no later mutation arrives. + this.scheduleStatusPersist() + return false } - } - - protected primeStatusPersistBaseline(onDiskJson: string): void { - this.getStatusSnapshotWriter()?.primeCommittedContent(onDiskJson) - this.lastPersistedAuthorityFingerprint = this.currentAuthorityFingerprint() + // Only a committed save can stand in for the authority it carries. + this.lastPersistedAuthorityFingerprint = fingerprint + this.statusPersistFailing = false + this.statusPersistRetryDelayMs = 0 + this.statusPersistRetryAt = 0 + return true } protected closeStatusSnapshotWriter(): void { this.cancelStatusPersistTimer() - this.statusSnapshotWriter?.close() - this.statusSnapshotWriter = null - this.statusSnapshotWriterPath = null + this.statusStore?.close() + this.statusStore = null + this.statusStorePath = null this.lastStatusPersistStartedAt = Number.NEGATIVE_INFINITY this.lastPersistedAuthorityFingerprint = null this.statusPersistFailing = false + this.statusPersistRetryDelayMs = 0 + this.statusPersistRetryAt = 0 } - protected getStatusSnapshotWriter(): AtomicSnapshotWriter | null { + protected getStatusStore(): LastStatusStore | null { const filePath = this.lastStatusFilePath if (!filePath) { return null } - if (!this.statusSnapshotWriter || this.statusSnapshotWriterPath !== filePath) { - this.statusSnapshotWriter?.close() - this.statusSnapshotWriter = new AtomicSnapshotWriter(() => filePath, { - fileMode: 0o600, - directoryMode: 0o700, - skipUnchanged: true - }) - this.statusSnapshotWriterPath = filePath + if (!this.statusStore || this.statusStorePath !== filePath) { + this.statusStore?.close() + this.statusStore = new LastStatusStore(filePath) + this.statusStorePath = filePath } - return this.statusSnapshotWriter + return this.statusStore } /** Revision counters and live tokens avoid per-event hashing or serialization. */ @@ -191,30 +219,6 @@ export abstract class AgentHookServerPersistence extends AgentHookServerHydratio return fingerprint } - private startStatusPersist(): void { - const writer = this.getStatusSnapshotWriter() - if (!writer) { - return - } - this.lastStatusPersistStartedAt = Date.now() - this.lastPersistedAuthorityFingerprint = this.currentAuthorityFingerprint() - writer - .write(() => this.serializeStatusFile()) - .then( - () => { - this.statusPersistFailing = false - }, - (err: unknown) => { - if (this.statusSnapshotWriter !== writer) { - return - } - this.reportStatusPersistFailure(err) - // Retry even if no later mutation arrives. - this.scheduleStatusPersist() - } - ) - } - private cancelStatusPersistTimer(): void { if (this.statusPersistTimer) { clearTimeout(this.statusPersistTimer) @@ -224,7 +228,7 @@ export abstract class AgentHookServerPersistence extends AgentHookServerHydratio private reportStatusPersistFailure(err: unknown): void { if (!this.statusPersistFailing) { - console.warn('[agent-hooks] failed to write last-status file:', err) + console.warn('[agent-hooks] failed to save agent status:', err) } this.statusPersistFailing = true } diff --git a/src/main/agent-hooks/server/server-state.ts b/src/main/agent-hooks/server/server-state.ts index 7d785558d47..50540e4d621 100644 --- a/src/main/agent-hooks/server/server-state.ts +++ b/src/main/agent-hooks/server/server-state.ts @@ -56,6 +56,7 @@ import type { } from './server-types' import { RevisionedMap } from './revisioned-map' import { forgetAgentExit, type AgentExitEvidence } from './agent-exit-evidence' +import type { LastStatusStore } from './last-status-store' export abstract class AgentHookServerState { protected canWriteLegacyStatusRow(entry: AgentHookEventPayload): boolean { @@ -308,10 +309,9 @@ export abstract class AgentHookServerState { payload: AgentHookEventPayload | EnrichedAgentHookEventPayload, launchTokenHashOverride?: string ): AgentHookAuthorityEvidence | null - protected abstract serializeStatusFile(): string protected abstract scheduleStatusPersist(): void - protected abstract runStatusPersist(): void - protected abstract primeStatusPersistBaseline(onDiskJson: string): void + protected abstract runStatusPersist(): boolean + protected abstract getStatusStore(): LastStatusStore | null abstract _getStateForTests(): HookListenerState abstract _resetPromptSentDedupeForTests(): void diff --git a/src/main/agent-hooks/server/testing/accepted-replies/server-accepted-reply.test.ts b/src/main/agent-hooks/server/testing/accepted-replies/server-accepted-reply.test.ts index 268762da77f..9b3bb13694b 100644 --- a/src/main/agent-hooks/server/testing/accepted-replies/server-accepted-reply.test.ts +++ b/src/main/agent-hooks/server/testing/accepted-replies/server-accepted-reply.test.ts @@ -1,5 +1,6 @@ +import { readSavedStatus } from '../persistence/last-status.test-fixtures' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { AgentHookServer, _internals } from '../../agent-hook-server' @@ -58,7 +59,7 @@ describe('accepted lead reply', () => { expect(replyForCurrentRow()).toBe(REPLY) server.flushStatusPersistSync() for (const published of [ - readFileSync(join(userDataPath, 'agent-hooks', 'last-status.json'), 'utf8'), + readSavedStatus(join(userDataPath, 'agent-hooks', 'last-status.json')), JSON.stringify(server.getStatusSnapshot()) ]) { expect(published).not.toContain('MIDDLE-MARKER') diff --git a/src/main/agent-hooks/server/testing/authority/server-reminted-pane-key.test.ts b/src/main/agent-hooks/server/testing/authority/server-reminted-pane-key.test.ts index b3eb282c93a..3b580b0f963 100644 --- a/src/main/agent-hooks/server/testing/authority/server-reminted-pane-key.test.ts +++ b/src/main/agent-hooks/server/testing/authority/server-reminted-pane-key.test.ts @@ -1,4 +1,5 @@ -import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { readSavedStatus, hasSavedStatus } from '../persistence/last-status.test-fixtures' +import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -80,10 +81,10 @@ describe('reminted $$ pane keys on the OMP hook pipeline', () => { }) ]) server.flushStatusPersistSync() - expect(existsSync(lastStatusPath())).toBe(true) - const file = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) as { + expect(hasSavedStatus(lastStatusPath())).toBe(true) + const file: { entries: Record - } + } = JSON.parse(readSavedStatus(lastStatusPath())) expect(file.entries[PANE]?.payload.state).toBe('done') expect(file.entries[REMINTED]).toBeUndefined() } finally { diff --git a/src/main/agent-hooks/server/testing/authority/server-removed-worktree-foreign-authority.test.ts b/src/main/agent-hooks/server/testing/authority/server-removed-worktree-foreign-authority.test.ts index 141e9f6f6fa..cbc55586b5c 100644 --- a/src/main/agent-hooks/server/testing/authority/server-removed-worktree-foreign-authority.test.ts +++ b/src/main/agent-hooks/server/testing/authority/server-removed-worktree-foreign-authority.test.ts @@ -1,5 +1,6 @@ +import { readSavedStatus } from '../persistence/last-status.test-fixtures' import { createHash } from 'node:crypto' -import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' @@ -72,7 +73,7 @@ describe('removed-worktree foreign authority', () => { expect(server.getCurrentAuthorityObservations()).toEqual([]) server.flushStatusPersistSync() const file = JSON.parse( - readFileSync(join(userDataPath, 'agent-hooks', 'last-status.json'), 'utf8') + readSavedStatus(join(userDataPath, 'agent-hooks', 'last-status.json')) ) expect(file.authorityCommitments[PANE]).toBeUndefined() } finally { @@ -141,7 +142,7 @@ describe('removed-worktree foreign authority', () => { ) server.flushStatusPersistSync() const file = JSON.parse( - readFileSync(join(userDataPath, 'agent-hooks', 'last-status.json'), 'utf8') + readSavedStatus(join(userDataPath, 'agent-hooks', 'last-status.json')) ) expect(file.authorityCommitments[PANE]).toMatchObject({ worktreeId: KEPT, @@ -219,7 +220,7 @@ describe('removed-worktree foreign authority', () => { } server.flushStatusPersistSync() const file = JSON.parse( - readFileSync(join(userDataPath, 'agent-hooks', 'last-status.json'), 'utf8') + readSavedStatus(join(userDataPath, 'agent-hooks', 'last-status.json')) ) expect(file.authorityCommitments[PANE]).toMatchObject({ worktreeId: KEPT, diff --git a/src/main/agent-hooks/server/testing/authority/server-removed-worktree-status.test.ts b/src/main/agent-hooks/server/testing/authority/server-removed-worktree-status.test.ts index fa24ce40be1..25f7c5d74f4 100644 --- a/src/main/agent-hooks/server/testing/authority/server-removed-worktree-status.test.ts +++ b/src/main/agent-hooks/server/testing/authority/server-removed-worktree-status.test.ts @@ -1,5 +1,6 @@ +import { readSavedStatus } from '../persistence/last-status.test-fixtures' import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { AgentHookServer, _internals } from '../../agent-hook-server' @@ -72,7 +73,7 @@ describe('AgentHookServer removed-worktree retirement', () => { server.clearStatusEntriesForConnection('idle@box') const persisted = () => { server.flushStatusPersistSync() - const file = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) + const file = JSON.parse(readSavedStatus(lastStatusPath())) return { entries: Object.keys(file.entries).sort(), commitments: Object.keys(file.authorityCommitments ?? {}) @@ -156,7 +157,7 @@ describe('AgentHookServer removed-worktree retirement', () => { reportLocally('done') server.flushStatusPersistSync() - const file = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) + const file = JSON.parse(readSavedStatus(lastStatusPath())) if (localWorktree === REMOVED) { expect(file.authorityCommitments?.[pane]).toMatchObject({ worktreeId: KEPT }) expect(file.entries[pane]).toBeUndefined() diff --git a/src/main/agent-hooks/server/testing/http/server-startup-prompt-control.test.ts b/src/main/agent-hooks/server/testing/http/server-startup-prompt-control.test.ts index e2bdd770fb6..db4c6ce33b4 100644 --- a/src/main/agent-hooks/server/testing/http/server-startup-prompt-control.test.ts +++ b/src/main/agent-hooks/server/testing/http/server-startup-prompt-control.test.ts @@ -1,4 +1,5 @@ -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { readSavedStatus, hasSavedStatus } from '../persistence/last-status.test-fixtures' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { describe, expect, it, vi } from 'vitest' @@ -28,8 +29,8 @@ describe('startup prompt control with status hooks disabled', () => { throw new Error('missing status persistence path') } server.stop() - expect(existsSync(statusPath)).toBe(true) - expect(JSON.parse(readFileSync(statusPath, 'utf8')).entries[PANE]).toMatchObject({ + expect(hasSavedStatus(statusPath)).toBe(true) + expect(JSON.parse(readSavedStatus(statusPath)).entries[PANE]).toMatchObject({ paneKey: PANE, worktreeId: 'folder-1', payload: { state: 'done', prompt: 'terminal status survives quit', agentType: 'opencode' } diff --git a/src/main/agent-hooks/server/testing/identity/server-claude-session-continuation.test.ts b/src/main/agent-hooks/server/testing/identity/server-claude-session-continuation.test.ts index 9b35c902e0c..4cdf640a915 100644 --- a/src/main/agent-hooks/server/testing/identity/server-claude-session-continuation.test.ts +++ b/src/main/agent-hooks/server/testing/identity/server-claude-session-continuation.test.ts @@ -1,4 +1,5 @@ -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { readSavedStatus } from '../persistence/last-status.test-fixtures' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -104,7 +105,7 @@ describe('Claude session continued into a daemon job', () => { expect(after?.prompt).toBe(before?.prompt) server.flushStatusPersistSync() const persisted = JSON.parse( - readFileSync(join(root, 'orca', 'agent-hooks', 'last-status.json'), 'utf8') + readSavedStatus(join(root, 'orca', 'agent-hooks', 'last-status.json')) ) expect(persisted.entries[PANE].providerSession.id).toBe(FORK) }) diff --git a/src/main/agent-hooks/server/testing/identity/server-hook-session-binding.test.ts b/src/main/agent-hooks/server/testing/identity/server-hook-session-binding.test.ts index 6c37cd1f9a6..072fd90e027 100644 --- a/src/main/agent-hooks/server/testing/identity/server-hook-session-binding.test.ts +++ b/src/main/agent-hooks/server/testing/identity/server-hook-session-binding.test.ts @@ -1,5 +1,6 @@ +import { readSavedStatus } from '../persistence/last-status.test-fixtures' import type { BrowserWindow } from 'electron' -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, expect, it, vi } from 'vitest' @@ -257,8 +258,9 @@ it.each([257, 512])( server.stop() // Exact legacy representation: a helper ID with no positive observation of its own. const path = join(dir, 'agent-hooks', 'last-status.json') - const saved = JSON.parse(readFileSync(path, 'utf8')) + const saved = JSON.parse(readSavedStatus(path)) saved.entries[PANE].agentPresence.session = id + // An older build's file wins over the saved rows on the next start. writeFileSync(path, JSON.stringify(saved)) await restart() await tool('different-session') diff --git a/src/main/agent-hooks/server/testing/identity/server-opencode-writer-observation.test.ts b/src/main/agent-hooks/server/testing/identity/server-opencode-writer-observation.test.ts index bf9b8721ca5..ff1f3b0170c 100644 --- a/src/main/agent-hooks/server/testing/identity/server-opencode-writer-observation.test.ts +++ b/src/main/agent-hooks/server/testing/identity/server-opencode-writer-observation.test.ts @@ -15,12 +15,6 @@ vi.mock('../../../../../shared/agent-status/process/agent-process-presence-probe probeAgentProcessPresence: async () => 'unverifiable' })) -class WriterObservationTestServer extends AgentHookServer { - persistedWriterState(): string { - return this.serializeStatusFile() - } -} - const servers: AgentHookServer[] = [] afterEach(() => { for (const server of servers.splice(0)) { @@ -28,8 +22,8 @@ afterEach(() => { } }) -async function createServer(): Promise { - const server = new WriterObservationTestServer() +async function createServer(): Promise { + const server = new AgentHookServer() servers.push(server) await server.start({ env: 'production' }) return server @@ -120,7 +114,6 @@ describe('OpenCode writer observation', () => { expect(JSON.stringify(value)).not.toContain('pane-owned') expect(Object.getOwnPropertySymbols(structuredClone(value).providerSession ?? {})).toEqual([]) } - expect(server.persistedWriterState()).not.toContain('pane-owned') for (const extra of [{}, { opencodeSharedServer: 1 }]) { await post(extra) expect(observation()?.topology).not.toBe('pane-owned') diff --git a/src/main/agent-hooks/server/testing/main-agent/server-turn-started-at.test.ts b/src/main/agent-hooks/server/testing/main-agent/server-turn-started-at.test.ts index 34e6a034bdc..c315a61c31f 100644 --- a/src/main/agent-hooks/server/testing/main-agent/server-turn-started-at.test.ts +++ b/src/main/agent-hooks/server/testing/main-agent/server-turn-started-at.test.ts @@ -1,5 +1,6 @@ +import { readSavedStatus } from '../persistence/last-status.test-fixtures' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { AgentHookServer, _internals } from '../../agent-hook-server' @@ -124,9 +125,7 @@ describe('the host-stamped turn start', () => { expect(pushed.at(-1)).toBe(started) server.flushStatusPersistSync() server.stop() - const file = JSON.parse( - readFileSync(join(userDataPath, 'agent-hooks', 'last-status.json'), 'utf8') - ) + const file = JSON.parse(readSavedStatus(join(userDataPath, 'agent-hooks', 'last-status.json'))) expect(file.entries[PANE].turnStartedAt).toBe(started) const restarted = await startServer() diff --git a/src/main/agent-hooks/server/testing/persistence/last-status.test-fixtures.ts b/src/main/agent-hooks/server/testing/persistence/last-status.test-fixtures.ts new file mode 100644 index 00000000000..077bc6cf3c9 --- /dev/null +++ b/src/main/agent-hooks/server/testing/persistence/last-status.test-fixtures.ts @@ -0,0 +1,57 @@ +import { existsSync, mkdirSync } from 'node:fs' +import { dirname } from 'node:path' + +import SyncDatabase from '../../../../sqlite/sync-database' +import { lastStatusDatabasePath } from '../../last-status-store' +import { LAST_STATUS_FILE_VERSION } from '../../server-constants' + +/** What a restart restores, in the last-status file shape, read straight from the saved rows. */ +export function readSavedStatus(lastStatusPath: string): string { + const path = lastStatusDatabasePath(lastStatusPath) + const db = new SyncDatabase(path, { readonly: true, fileMustExist: true }) + try { + const entries: Record = {} + const authorityCommitments: Record = {} + const file: { + version: number + entries: typeof entries + authorityCommitments: typeof authorityCommitments + agentExits?: unknown + } = { + version: LAST_STATUS_FILE_VERSION, + entries, + authorityCommitments + } + for (const row of db.prepare('SELECT kind, key, payload FROM rows').all()) { + const value: unknown = JSON.parse(String(row.payload)) + if (row.kind === 'entry') { + entries[String(row.key)] = value + } else if (row.kind === 'commitment') { + authorityCommitments[String(row.key)] = value + } else { + file.agentExits = value + } + } + return JSON.stringify(file) + } finally { + db.close() + } +} + +export function hasSavedStatus(lastStatusPath: string): boolean { + if (!existsSync(lastStatusDatabasePath(lastStatusPath))) { + return false + } + return Object.keys(JSON.parse(readSavedStatus(lastStatusPath)).entries).length > 0 +} + +/** Another connection holding the write lock, as a second process would. */ +export function holdSavedStatusLock(lastStatusPath: string): () => void { + mkdirSync(dirname(lastStatusPath), { recursive: true }) + const db = new SyncDatabase(lastStatusDatabasePath(lastStatusPath)) + db.exec('BEGIN EXCLUSIVE') + return () => { + db.exec('ROLLBACK') + db.close() + } +} diff --git a/src/main/agent-hooks/server/testing/persistence/server-last-status-hydrate-validation.test.ts b/src/main/agent-hooks/server/testing/persistence/server-last-status-hydrate-validation.test.ts index 874d124818b..b3701652b41 100644 --- a/src/main/agent-hooks/server/testing/persistence/server-last-status-hydrate-validation.test.ts +++ b/src/main/agent-hooks/server/testing/persistence/server-last-status-hydrate-validation.test.ts @@ -1,14 +1,6 @@ +import { readSavedStatus, hasSavedStatus } from './last-status.test-fixtures' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - rmSync, - statSync, - utimesSync, - writeFileSync -} from 'node:fs' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { AgentHookServer, _internals } from '../../agent-hook-server' @@ -19,8 +11,7 @@ import { recentTs, PANE, GOOD_PANE, - LEAF_2, - LEAF_5 + LEAF_2 } from '../server.test-fixtures' const { getCohortAtEmitMock, trackMock } = vi.hoisted(() => ({ @@ -124,42 +115,18 @@ describe('Last-status persistence', () => { buildBody({ hook_event_name: 'UserPromptSubmit', prompt: 'about to drop' }) ) server.flushStatusPersistSync() - let parsed = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) + let parsed = JSON.parse(readSavedStatus(lastStatusPath())) expect(parsed.entries[PANE]).toBeTruthy() server.clearPaneState(PANE) server.flushStatusPersistSync() - parsed = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) + parsed = JSON.parse(readSavedStatus(lastStatusPath())) expect(parsed.entries[PANE]).toBeUndefined() } finally { server.stop() } }) - it('skips a write when the serialized contents are byte-identical to the previous write', async () => { - const server = new AgentHookServer() - await server.start({ - env: 'production', - userDataPath - }) - try { - await postHookEvent( - server, - buildBody({ hook_event_name: 'UserPromptSubmit', prompt: 'first' }) - ) - server.flushStatusPersistSync() - utimesSync(lastStatusPath(), 1, 1) - const firstMtime = statSync(lastStatusPath()).mtimeMs - - server.clearPaneState(makePaneKey('non-existent', LEAF_5)) - server.flushStatusPersistSync() - const secondMtime = statSync(lastStatusPath()).mtimeMs - expect(secondMtime).toBe(firstMtime) - } finally { - server.stop() - } - }) - it('stop() flushes pending debounced writes synchronously', async () => { const server = new AgentHookServer() await server.start({ @@ -175,8 +142,8 @@ describe('Last-status persistence', () => { } finally { server.stop() } - expect(existsSync(lastStatusPath())).toBe(true) - const parsed = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) + expect(hasSavedStatus(lastStatusPath())).toBe(true) + const parsed = JSON.parse(readSavedStatus(lastStatusPath())) expect(parsed.entries[PANE]?.payload?.prompt).toBe('flush me') }) }) diff --git a/src/main/agent-hooks/server/testing/persistence/server-last-status-migration.test.ts b/src/main/agent-hooks/server/testing/persistence/server-last-status-migration.test.ts new file mode 100644 index 00000000000..d11d5b340ed --- /dev/null +++ b/src/main/agent-hooks/server/testing/persistence/server-last-status-migration.test.ts @@ -0,0 +1,94 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { AgentHookServer, _internals } from '../../agent-hook-server' +import { recentTs, PANE } from '../server.test-fixtures' +import { holdSavedStatusLock } from './last-status.test-fixtures' + +vi.mock('../../../../telemetry/client', () => ({ track: vi.fn() })) +vi.mock('../../../../telemetry/cohort-classifier', () => ({ + getCohortAtEmit: () => ({ nth_repo_added: 2 }) +})) + +describe('saved agent status from an older build', () => { + let userDataPath: string + const lastStatusPath = (): string => join(userDataPath, 'agent-hooks', 'last-status.json') + const writeLegacyFile = (prompt: string): void => { + mkdirSync(join(userDataPath, 'agent-hooks'), { recursive: true }) + writeFileSync( + lastStatusPath(), + JSON.stringify({ + version: 2, + entries: { + [PANE]: { + paneKey: PANE, + tabId: 'tab-1', + receivedAt: recentTs(), + stateStartedAt: recentTs(), + payload: { state: 'done', prompt, agentType: 'claude' } + } + } + }) + ) + } + const restoredPrompt = async (): Promise => { + const server = new AgentHookServer() + await server.start({ env: 'production', userDataPath }) + try { + return server.getStatusSnapshot()[0]?.prompt + } finally { + server.stop() + } + } + + beforeEach(() => { + _internals.resetCachesForTests() + userDataPath = mkdtempSync(join(tmpdir(), 'orca-last-status-migration-')) + }) + + afterEach(() => { + rmSync(userDataPath, { recursive: true, force: true }) + }) + + it('moves the old file into the saved rows once and keeps restoring from them', async () => { + writeLegacyFile('from the old file') + + expect(await restoredPrompt()).toBe('from the old file') + expect(existsSync(lastStatusPath())).toBe(false) + expect(await restoredPrompt()).toBe('from the old file') + }) + + it('leaves the old file alone for a session whose rows cannot be read', async () => { + writeLegacyFile('from the old file') + const release = holdSavedStatusLock(lastStatusPath()) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const server = new AgentHookServer() + await server.start({ env: 'production', userDataPath }) + release() + server.stop() + expect(existsSync(lastStatusPath())).toBe(true) + + expect(await restoredPrompt()).toBe('from the old file') + expect(existsSync(lastStatusPath())).toBe(false) + }) + + it('keeps the saved rows when an old file is rejected', async () => { + writeLegacyFile('saved in rows') + expect(await restoredPrompt()).toBe('saved in rows') + writeFileSync(lastStatusPath(), 'null') + vi.spyOn(console, 'warn').mockImplementation(() => {}) + + expect(await restoredPrompt()).toBe('saved in rows') + expect(existsSync(lastStatusPath())).toBe(false) + expect(await restoredPrompt()).toBe('saved in rows') + }) + + it('prefers a file an older build wrote after the rows were saved', async () => { + writeLegacyFile('first') + expect(await restoredPrompt()).toBe('first') + + writeLegacyFile('written by an older build later') + expect(await restoredPrompt()).toBe('written by an older build later') + }) +}) diff --git a/src/main/agent-hooks/server/testing/persistence/server-last-status-write.test.ts b/src/main/agent-hooks/server/testing/persistence/server-last-status-write.test.ts index 9386306f039..27365f808c8 100644 --- a/src/main/agent-hooks/server/testing/persistence/server-last-status-write.test.ts +++ b/src/main/agent-hooks/server/testing/persistence/server-last-status-write.test.ts @@ -1,6 +1,7 @@ +import { readSavedStatus, hasSavedStatus } from './last-status.test-fixtures' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { createHash } from 'node:crypto' -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { AgentHookServer, _internals } from '../../agent-hook-server' @@ -67,8 +68,8 @@ describe('Last-status persistence', () => { ) ) server.flushStatusPersistSync() - expect(existsSync(lastStatusPath())).toBe(true) - const file = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) + expect(hasSavedStatus(lastStatusPath())).toBe(true) + const file = JSON.parse(readSavedStatus(lastStatusPath())) expect(file.version).toBe(2) expect(file.entries[PANE]).toMatchObject({ paneKey: PANE, @@ -92,7 +93,7 @@ describe('Last-status persistence', () => { state: 'done', stateStartedAt: expect.any(Number) }) - expect(readFileSync(lastStatusPath(), 'utf8')).not.toContain('launch-bearer-must-not-persist') + expect(readSavedStatus(lastStatusPath())).not.toContain('launch-bearer-must-not-persist') } finally { server.stop() } @@ -135,7 +136,7 @@ describe('Last-status persistence', () => { ]) expect(server.getStatusSnapshotForPane(PANE)[0]?.launchToken).toBeUndefined() - const persisted = readFileSync(lastStatusPath(), 'utf8') + const persisted = readSavedStatus(lastStatusPath()) expect(persisted).not.toContain(launchToken) expect(JSON.parse(persisted).entries[PANE]).toMatchObject({ launchTokenHash }) } finally { @@ -192,7 +193,7 @@ describe('Last-status persistence', () => { first.flushStatusPersistSync() first.stop() - const afterClear = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) + const afterClear = JSON.parse(readSavedStatus(lastStatusPath())) expect(afterClear.entries).toEqual({}) expect(afterClear.authorityCommitments[PANE]).toMatchObject({ paneKey: PANE, @@ -263,7 +264,7 @@ describe('Last-status persistence', () => { expect(trackMock).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) firstServer.flushStatusPersistSync() - const file = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) + const file = JSON.parse(readSavedStatus(lastStatusPath())) expect(file.entries[PANE]).toMatchObject({ providerSessionOnly: true, providerSession: { diff --git a/src/main/agent-hooks/server/testing/persistence/server-observation-provenance.test.ts b/src/main/agent-hooks/server/testing/persistence/server-observation-provenance.test.ts index 3d6f077bb6f..62bfbf2a44e 100644 --- a/src/main/agent-hooks/server/testing/persistence/server-observation-provenance.test.ts +++ b/src/main/agent-hooks/server/testing/persistence/server-observation-provenance.test.ts @@ -1,5 +1,6 @@ +import { hasSavedStatus, readSavedStatus } from './last-status.test-fixtures' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { AgentHookServer, _internals } from '../../agent-hook-server' @@ -60,8 +61,8 @@ describe('agent status observation provenance', () => { server.flushStatusPersistSync() const path = join(userDataPath, 'agent-hooks', 'last-status.json') - expect(existsSync(path)).toBe(true) - const file = readFileSync(path, 'utf8') + expect(hasSavedStatus(path)).toBe(true) + const file = readSavedStatus(path) // Process-local sequencing cannot remain ordering authority after restart. expect(file).not.toContain('observation') expect(JSON.parse(file).entries[PANE].observation).toBeUndefined() diff --git a/src/main/agent-hooks/server/testing/persistence/server-status-persist-nonblocking.test.ts b/src/main/agent-hooks/server/testing/persistence/server-status-persist-nonblocking.test.ts deleted file mode 100644 index f9a7e1c65d0..00000000000 --- a/src/main/agent-hooks/server/testing/persistence/server-status-persist-nonblocking.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import * as fs from 'node:fs' -import * as fsPromises from 'node:fs/promises' -import { mkdtempSync, readFileSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { AgentHookServer, _internals } from '../../agent-hook-server' -import { buildBody, PANE, postHookEvent } from '../server.test-fixtures' -import { AtomicSnapshotWriter } from '../../../../persistence/atomic-snapshot-writer' - -type FsModule = typeof fs -type FsPromisesModule = typeof fsPromises - -vi.mock('../../../../telemetry/client', () => ({ track: vi.fn() })) -vi.mock('../../../../telemetry/cohort-classifier', () => ({ - getCohortAtEmit: vi.fn(() => ({ nth_repo_added: 2 })) -})) -vi.mock('node:fs', async (importOriginal) => { - const actual = await importOriginal() - return { ...actual, renameSync: vi.fn(actual.renameSync) } -}) -vi.mock('node:fs/promises', async (importOriginal) => { - const actual = await importOriginal() - return { ...actual, rename: vi.fn(actual.rename) } -}) - -describe('last-status persistence stays off the main thread', () => { - let userDataPath: string - let server: AgentHookServer - - const lastStatusPath = (): string => join(userDataPath, 'agent-hooks', 'last-status.json') - const readPrompt = (): unknown => - JSON.parse(readFileSync(lastStatusPath(), 'utf8')).entries[PANE]?.payload?.prompt - // Exclude synchronous endpoint startup writes from status-persistence checks. - const statusRenameSyncCalls = (): number => - vi.mocked(fs.renameSync).mock.calls.filter(([, to]) => String(to) === lastStatusPath()).length - const prompt = (text: string): Promise => - postHookEvent(server, buildBody({ hook_event_name: 'UserPromptSubmit', prompt: text })) - - beforeEach(async () => { - _internals.resetCachesForTests() - userDataPath = mkdtempSync(join(tmpdir(), 'orca-persist-nonblocking-')) - server = new AgentHookServer() - await server.start({ env: 'production', userDataPath }) - }) - - afterEach(() => { - server.stop() - vi.mocked(fsPromises.rename).mockReset() - vi.restoreAllMocks() - rmSync(userDataPath, { recursive: true, force: true }) - }) - - it('keeps answering queued hooks and preserves the quit flush while a rename is held open', async () => { - const actualRename = await vi.importActual('node:fs/promises') - let releaseRename: () => void = () => {} - const renameHeld = new Promise((resolve) => { - vi.mocked(fsPromises.rename).mockImplementationOnce(async (from, to) => { - resolve() - await new Promise((release) => { - releaseRename = release - }) - return actualRename.rename(from, to) - }) - }) - vi.mocked(fsPromises.rename).mockImplementation(actualRename.rename) - const write = vi.spyOn(AtomicSnapshotWriter.prototype, 'write') - const writeSync = vi.spyOn(AtomicSnapshotWriter.prototype, 'writeSync') - - expect((await prompt('first')).status).toBe(204) - await renameHeld - - expect((await prompt('while renaming')).status).toBe(204) - expect((await prompt('queued while renaming')).status).toBe(204) - expect(writeSync).not.toHaveBeenCalled() - expect(statusRenameSyncCalls()).toBe(0) - - // An older async rename must not undo the synchronous quit flush. - server.flushStatusPersistSync() - expect(writeSync).toHaveBeenCalledTimes(1) - expect(readPrompt()).toBe('queued while renaming') - releaseRename() - await write.mock.results[0]?.value - expect(readPrompt()).toBe('queued while renaming') - }) -}) diff --git a/src/main/agent-hooks/server/testing/persistence/server-status-persist-throttle.test.ts b/src/main/agent-hooks/server/testing/persistence/server-status-persist-throttle.test.ts index 68a928815fc..1c418ec3b7d 100644 --- a/src/main/agent-hooks/server/testing/persistence/server-status-persist-throttle.test.ts +++ b/src/main/agent-hooks/server/testing/persistence/server-status-persist-throttle.test.ts @@ -1,34 +1,23 @@ +import { holdSavedStatusLock, readSavedStatus } from './last-status.test-fixtures' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { AgentHookServer, _internals } from '../../agent-hook-server' -import { STATUS_PERSIST_MIN_INTERVAL_MS } from '../../server-constants' import { buildBody, PANE, postHookEvent } from '../server.test-fixtures' -import { AtomicSnapshotWriter } from '../../../../persistence/atomic-snapshot-writer' vi.mock('../../../../telemetry/client', () => ({ track: vi.fn() })) vi.mock('../../../../telemetry/cohort-classifier', () => ({ getCohortAtEmit: vi.fn(() => ({ nth_repo_added: 2 })) })) -class ProbeServer extends AgentHookServer { - persistTimerArmed(): boolean { - return this.statusPersistTimer !== null - } - - persistDueAt(): number { - return this.statusPersistDueAt - } -} - -describe('last-status persist throttle', () => { +describe('saved agent status cadence', () => { let userDataPath: string - let server: ProbeServer + let server: AgentHookServer const lastStatusPath = (): string => join(userDataPath, 'agent-hooks', 'last-status.json') const readPrompt = (): unknown => - JSON.parse(readFileSync(lastStatusPath(), 'utf8')).entries[PANE]?.payload?.prompt + JSON.parse(readSavedStatus(lastStatusPath())).entries[PANE]?.payload?.prompt const prompt = (text: string, launchToken?: string): Promise => postHookEvent( server, @@ -46,7 +35,7 @@ describe('last-status persist throttle', () => { toFake: ['setTimeout', 'clearTimeout', 'Date'] }) userDataPath = mkdtempSync(join(tmpdir(), 'orca-persist-throttle-')) - server = new ProbeServer() + server = new AgentHookServer() await server.start({ env: 'production', userDataPath }) }) @@ -57,113 +46,45 @@ describe('last-status persist throttle', () => { rmSync(userDataPath, { recursive: true, force: true }) }) - it('caps a steady stream at one write per window without pushing the timer back', async () => { - const write = vi.spyOn(AtomicSnapshotWriter.prototype, 'write') - let unchangedWhileArmed = 0 - for (let i = 0; i < 60; i++) { - const armedBefore = server.persistTimerArmed() - const dueBefore = server.persistDueAt() - await prompt(`stream ${i}`) - if (armedBefore && server.persistTimerArmed()) { - expect(server.persistDueAt()).toBe(dueBefore) - unchangedWhileArmed++ - } - if (server.persistTimerArmed()) { - expect(server.persistDueAt() - Date.now()).toBeLessThanOrEqual( - STATUS_PERSIST_MIN_INTERVAL_MS - ) - } - await vi.advanceTimersByTimeAsync(500) - } - await vi.advanceTimersByTimeAsync(STATUS_PERSIST_MIN_INTERVAL_MS) - expect(write).toHaveBeenCalledTimes(7) - expect(unchangedWhileArmed).toBeGreaterThan(40) - await Promise.all(write.mock.results.map((result) => result.value)) - expect(readPrompt()).toBe('stream 59') - expect(server.persistTimerArmed()).toBe(false) - }) - - it('writes an authority change immediately instead of waiting for the window', async () => { - const write = vi.spyOn(AtomicSnapshotWriter.prototype, 'write') + it('saves an authority change at once while a routine update waits for the window', async () => { await prompt('first') await vi.advanceTimersByTimeAsync(1) - expect(write).toHaveBeenCalledTimes(1) + expect(readPrompt()).toBe('first') - await vi.advanceTimersByTimeAsync(500) await prompt('second') - expect(server.persistTimerArmed()).toBe(true) + await vi.advanceTimersByTimeAsync(1) + expect(readPrompt()).toBe('first') await prompt('third', 'launch-token-new') await vi.advanceTimersByTimeAsync(1) - expect(write).toHaveBeenCalledTimes(2) - await write.mock.results[1]?.value - expect(JSON.parse(readFileSync(lastStatusPath(), 'utf8')).entries[PANE]).toMatchObject({ - launchTokenHash: expect.any(String) + expect(JSON.parse(readSavedStatus(lastStatusPath())).entries[PANE]).toMatchObject({ + launchTokenHash: expect.any(String), + payload: expect.objectContaining({ prompt: 'third' }) }) }) - it('flushStatusPersistSync cancels the armed timer and writes synchronously', async () => { - const write = vi.spyOn(AtomicSnapshotWriter.prototype, 'write') - const writeSync = vi.spyOn(AtomicSnapshotWriter.prototype, 'writeSync') + it('saves the latest status at stop and a restart keeps it', async () => { await prompt('first') await vi.advanceTimersByTimeAsync(1) - await write.mock.results[0]?.value - await prompt('second') - expect(server.persistTimerArmed()).toBe(true) - - server.flushStatusPersistSync() - expect(server.persistTimerArmed()).toBe(false) - expect(writeSync).toHaveBeenCalledTimes(1) - expect(readPrompt()).toBe('second') - - await vi.advanceTimersByTimeAsync(STATUS_PERSIST_MIN_INTERVAL_MS * 2) - expect(write).toHaveBeenCalledTimes(1) - }) - - it('stop then start leaves no armed timer and never overwrites the file with stale state', async () => { - const write = vi.spyOn(AtomicSnapshotWriter.prototype, 'write') - await prompt('first') - await vi.advanceTimersByTimeAsync(1) - await write.mock.results[0]?.value await prompt('final before stop') - expect(server.persistTimerArmed()).toBe(true) server.stop() - const afterStop = readFileSync(lastStatusPath(), 'utf8') expect(readPrompt()).toBe('final before stop') - await server.start({ env: 'production', userDataPath }) - expect(server.persistTimerArmed()).toBe(false) - await vi.advanceTimersByTimeAsync(STATUS_PERSIST_MIN_INTERVAL_MS * 2) - expect(write).toHaveBeenCalledTimes(1) - expect(readFileSync(lastStatusPath(), 'utf8')).toBe(afterStop) + await server.start({ env: 'production', userDataPath }) + expect(server.getStatusSnapshot()[0]?.prompt).toBe('final before stop') }) - it('warns once for a failing async write and retries in the next window', async () => { - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) - const write = vi - .spyOn(AtomicSnapshotWriter.prototype, 'write') - .mockRejectedValueOnce(new Error('disk full')) - .mockRejectedValueOnce(new Error('disk full')) - const persistWarnings = (): number => - warn.mock.calls.filter(([message]) => - String(message).includes('failed to write last-status file') - ).length - + it('saves an authority change within seconds once a held lock clears', async () => { await prompt('first') await vi.advanceTimersByTimeAsync(1) - expect(write).toHaveBeenCalledTimes(1) - expect(persistWarnings()).toBe(1) - expect(server.persistTimerArmed()).toBe(true) + const release = holdSavedStatusLock(lastStatusPath()) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + await prompt('new owner', 'launch-token-new') + await vi.advanceTimersByTimeAsync(1) + release() - await vi.advanceTimersByTimeAsync(STATUS_PERSIST_MIN_INTERVAL_MS) - expect(write).toHaveBeenCalledTimes(2) - expect(persistWarnings()).toBe(1) - - await vi.advanceTimersByTimeAsync(STATUS_PERSIST_MIN_INTERVAL_MS) - expect(write).toHaveBeenCalledTimes(3) - await write.mock.results[2]?.value - expect(readPrompt()).toBe('first') - expect(persistWarnings()).toBe(1) + await vi.advanceTimersByTimeAsync(2_000) + expect(readPrompt()).toBe('new owner') }) }) diff --git a/src/main/agent-hooks/server/testing/replay/spool.test.ts b/src/main/agent-hooks/server/testing/replay/spool.test.ts index 8dd464b3d81..110d8ca0302 100644 --- a/src/main/agent-hooks/server/testing/replay/spool.test.ts +++ b/src/main/agent-hooks/server/testing/replay/spool.test.ts @@ -1,3 +1,4 @@ +import { readSavedStatus } from '../persistence/last-status.test-fixtures' import { describe, expect, it } from 'vitest' import { appendFileSync, @@ -138,7 +139,7 @@ describe('agent hook spool', () => { expect(snapshot).toHaveLength(1) expect(snapshot[0]!.subagents).toBeUndefined() restarted.flushStatusPersistSync() - expect(readFileSync(restarted.lastStatusPath!, 'utf8')).not.toContain('isReplay') + expect(readSavedStatus(restarted.lastStatusPath!)).not.toContain('isReplay') } finally { restarted.stop() } diff --git a/src/main/agent-hooks/server/testing/structured-ingress/server-ingest-structured-status.test.ts b/src/main/agent-hooks/server/testing/structured-ingress/server-ingest-structured-status.test.ts index 3473b8781db..cc49e5ec750 100644 --- a/src/main/agent-hooks/server/testing/structured-ingress/server-ingest-structured-status.test.ts +++ b/src/main/agent-hooks/server/testing/structured-ingress/server-ingest-structured-status.test.ts @@ -1,6 +1,7 @@ +import { readSavedStatus } from '../persistence/last-status.test-fixtures' import { makeStructuredAgentStatusSubject } from '../../../../../shared/agent-status/contracts/agent-status-subject' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { mkdtempSync, readFileSync, rmSync, writeFileSync, mkdirSync } from 'node:fs' +import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import type { AgentSessionStatusSummary } from '../../../../../shared/agent-session/agent-session-wire' @@ -215,9 +216,9 @@ describe('structured rows and last-status.json', () => { server.stop() } - const file = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) as { + const file: { entries: Record - } + } = JSON.parse(readSavedStatus(lastStatusPath())) expect(Object.keys(file.entries)).toEqual([PANE]) const restored = new AgentHookServer() diff --git a/src/main/sqlite/versioned-cache-database.ts b/src/main/sqlite/versioned-cache-database.ts index cc40d452e9a..5fef05e7fa8 100644 --- a/src/main/sqlite/versioned-cache-database.ts +++ b/src/main/sqlite/versioned-cache-database.ts @@ -1,4 +1,4 @@ -import { mkdirSync } from 'node:fs' +import { mkdirSync, rmSync } from 'node:fs' import { dirname } from 'node:path' import SyncDatabase from './sync-database' import { removeTreeSync } from '../../shared/windows-transient-lock-removal' @@ -14,8 +14,15 @@ export type VersionedCacheDatabaseSchema = { directoryMode?: number // Applies only to a new file; rows near half a default page waste the other half. pageSize?: number + // Main-thread callers pass a short wait so a locked file fails fast instead of freezing. + busyTimeoutMs?: number + // Main-thread callers pass 'fail-fast' so a rebuild blocked by a held file throws at once + // (callers retry later) instead of sleeping through Windows unlock retries. + removal?: CacheDatabaseRemoval } +export type CacheDatabaseRemoval = 'retry' | 'fail-fast' + /** Rebuild unreadable/stale cache schemas: source files remain authoritative, so replacement costs only a rescan instead of stranding the feature. */ export function openVersionedCacheDatabase( path: string, @@ -32,21 +39,21 @@ export function openVersionedCacheDatabase( throw error } // One retry only: a second failure on a file we just created is not corruption. - removeVersionedCacheDatabase(path) + removeVersionedCacheDatabase(path, schema.removal) return openExisting(path, schema) } } function openExisting(path: string, schema: VersionedCacheDatabaseSchema): SyncDatabase { // Null closed handles so cleanup cannot mask the real failure with ERR_INVALID_STATE. - let db: SyncDatabase | null = openWithPragmas(path, schema.pageSize) + let db: SyncDatabase | null = openWithPragmas(path, schema) try { if (isStaleSchema(db, schema.version)) { // Unlink stale caches; dropping multi-GB tables can exceed scanner readiness deadlines. db.close() db = null - removeVersionedCacheDatabase(path) - db = openWithPragmas(path, schema.pageSize) + removeVersionedCacheDatabase(path, schema.removal) + db = openWithPragmas(path, schema) } db.exec(schema.sql) schema.afterSchema?.(db) @@ -65,7 +72,7 @@ function openExisting(path: string, schema: VersionedCacheDatabaseSchema): SyncD const UNUSABLE_DATABASE = /SQLITE_CORRUPT|SQLITE_NOTADB|file is not a database|database disk image is malformed/i -function isUnusableDatabaseError(error: unknown): boolean { +export function isUnusableDatabaseError(error: unknown): boolean { if (!(error instanceof Error)) { return false } @@ -76,11 +83,17 @@ function isUnusableDatabaseError(error: unknown): boolean { ) } -function openWithPragmas(path: string, pageSize: number | undefined): SyncDatabase { +function openWithPragmas( + path: string, + { + pageSize, + busyTimeoutMs = 5000 + }: Pick +): SyncDatabase { const db = new SyncDatabase(path) try { // Set busy timeout before WAL pragmas so another process's lock can clear. - db.pragma('busy_timeout = 5000') + db.pragma(`busy_timeout = ${Math.max(0, Math.trunc(busyTimeoutMs))}`) if (pageSize !== undefined) { db.pragma(`page_size = ${pageSize}`) } @@ -97,12 +110,19 @@ function openWithPragmas(path: string, pageSize: number | undefined): SyncDataba } } -export function removeVersionedCacheDatabase(path: string): void { +export function removeVersionedCacheDatabase( + path: string, + removal: CacheDatabaseRemoval = 'retry' +): void { if (path === ':memory:') { return } for (const suffix of ['', '-wal', '-shm', '-journal']) { - removeTreeSync(`${path}${suffix}`) + if (removal === 'fail-fast') { + rmSync(`${path}${suffix}`, { recursive: true, force: true }) + } else { + removeTreeSync(`${path}${suffix}`) + } } } diff --git a/src/shared/wsl-move/wsl-move-out-of-store-dispositions.ts b/src/shared/wsl-move/wsl-move-out-of-store-dispositions.ts index f6ece2559bf..27d44c31a22 100644 --- a/src/shared/wsl-move/wsl-move-out-of-store-dispositions.ts +++ b/src/shared/wsl-move/wsl-move-out-of-store-dispositions.ts @@ -24,7 +24,11 @@ export const OUT_OF_STORE_DISPOSITIONS: readonly (WslMoveFieldDisposition & { newManifestSection: true, note: 'Blob; re-keyed by the Linux worktree id on the host.' }, - { store: 'agent-hooks/last-status.json', disposition: 'rederive', item: 51 }, + { + store: 'agent-hooks/last-status.db (was last-status.json)', + disposition: 'rederive', + item: 51 + }, { store: 'agent-session-journal.db', disposition: 'not-applicable',