diff --git a/config/main-sync-fs-baseline.txt b/config/main-sync-fs-baseline.txt index ed9feeb4b41..697cc84efe0 100644 --- a/config/main-sync-fs-baseline.txt +++ b/config/main-sync-fs-baseline.txt @@ -373,8 +373,9 @@ 2 src/main/orcad/orcad-wsl-host-claude-profile-refresh.ts 1 src/main/orcad/serve-profile-ssh-targets.ts 6 src/main/persistence/atomic-snapshot-writer.ts -6 src/main/persistence/loading-store/user-data-path.ts -3 src/main/persistence/loading-store/write-flush-barriers.ts +1 src/main/persistence/loading-store/github-cache-sidecar.ts +1 src/main/persistence/loading-store/github-cache-startup-read.ts +5 src/main/persistence/loading-store/user-data-path.ts 9 src/main/persistence/migrating-orcad-catalog/migration-journal-directory.ts 4 src/main/persistence/migrating-orcad-catalog/orcad-migration-blob-installers.ts 3 src/main/persistence/migrating-orcad-catalog/orcad-migration-blob-staging.ts diff --git a/src/main/persistence/loading-store/github-cache-database.test.ts b/src/main/persistence/loading-store/github-cache-database.test.ts new file mode 100644 index 00000000000..50d130bc061 --- /dev/null +++ b/src/main/persistence/loading-store/github-cache-database.test.ts @@ -0,0 +1,117 @@ +import { + existsSync, + mkdtempSync, + openSync, + closeSync, + rmSync, + writeSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import SyncDatabase from '../../sqlite/sync-database' +import { getGithubCacheDatabaseFile, writeGitHubCacheDatabase } from './github-cache-database' +import { readGitHubCacheDatabase } from './github-cache-startup-read' + +const directories: string[] = [] + +afterEach(() => { + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +function profileDataFile(): string { + const directory = mkdtempSync(join(tmpdir(), 'orca-github-cache-')) + directories.push(directory) + return join(directory, 'orca-data.json') +} + +const pr = (fetchedAt: number) => ({ data: null, fetchedAt }) +const sidecarPath = (dataFile: string) => join(dataFile, '..', 'orca-github-cache.json') +const save = (dataFile: string, cache: Parameters[1]): void => { + void writeGitHubCacheDatabase(dataFile, cache, { busyTimeoutMs: 0, importedLegacyFileHash: null }) +} + +describe('github cache database', () => { + it('reads back saved entries, including removals', () => { + const dataFile = profileDataFile() + save(dataFile, { pr: { 'o/r#1': pr(1), 'o/r#2': pr(2) }, issue: {} }) + save(dataFile, { pr: { 'o/r#2': pr(3) }, issue: { 'o/r#9': pr(4) } }) + + expect(readGitHubCacheDatabase(dataFile)).toEqual({ + cache: { pr: { 'o/r#2': pr(3) }, issue: { 'o/r#9': pr(4) } }, + fromSidecar: false, + sidecarHash: null + }) + }) + + it('keeps an older build sidecar until its imported rows commit', async () => { + const dataFile = profileDataFile() + save(dataFile, { pr: { 'o/r#1': pr(1) }, issue: {} }) + writeFileSync(sidecarPath(dataFile), JSON.stringify({ pr: { 'o/r#5': pr(5) }, issue: {} })) + const legacy = readGitHubCacheDatabase(dataFile) + expect(legacy.cache).toEqual({ pr: { 'o/r#5': pr(5) }, issue: {} }) + + const locker = new SyncDatabase(getGithubCacheDatabaseFile(dataFile)) + locker.exec('BEGIN IMMEDIATE') + const options = { busyTimeoutMs: 0, importedLegacyFileHash: legacy.sidecarHash } + expect(() => writeGitHubCacheDatabase(dataFile, legacy.cache!, options)).toThrow(/locked/) + expect(existsSync(sidecarPath(dataFile))).toBe(true) + locker.exec('ROLLBACK') + locker.close() + + await writeGitHubCacheDatabase(dataFile, legacy.cache!, options) + expect(existsSync(sidecarPath(dataFile))).toBe(false) + expect(readGitHubCacheDatabase(dataFile)).toEqual({ + cache: { pr: { 'o/r#5': pr(5) }, issue: {} }, + fromSidecar: false, + sidecarHash: null + }) + }) + + it('never lets an already imported sidecar shadow later saves, even while locked', async () => { + const dataFile = profileDataFile() + const sidecar = JSON.stringify({ pr: { 'o/r#5': pr(5) }, issue: {} }) + writeFileSync(sidecarPath(dataFile), sidecar) + const legacy = readGitHubCacheDatabase(dataFile) + await writeGitHubCacheDatabase(dataFile, legacy.cache!, { + busyTimeoutMs: 0, + importedLegacyFileHash: legacy.sidecarHash + }) + // The same imported file survives, as when its deletion failed. + writeFileSync(sidecarPath(dataFile), sidecar) + save(dataFile, { pr: { 'o/r#6': pr(6) }, issue: {} }) + const locker = new SyncDatabase(getGithubCacheDatabaseFile(dataFile)) + locker.exec('BEGIN IMMEDIATE') + try { + expect(readGitHubCacheDatabase(dataFile).cache).toEqual({ pr: { 'o/r#6': pr(6) }, issue: {} }) + } finally { + locker.exec('ROLLBACK') + locker.close() + } + }) + + it('rebuilds a database whose entry rows are corrupt and saves again', () => { + const dataFile = profileDataFile() + save(dataFile, { pr: { 'o/r#1': pr(1) }, issue: {} }) + const path = getGithubCacheDatabaseFile(dataFile) + const db = new SyncDatabase(path) + const root = db.prepare(`SELECT rootpage FROM sqlite_master WHERE name = 'entries'`).get() + const pageSize = db.pragma('page_size', { simple: true }) + db.close() + const fd = openSync(path, 'r+') + writeSync( + fd, + Buffer.from([0xff, 0xff, 0xff, 0xff]), + 0, + 4, + (Number(root?.rootpage) - 1) * Number(pageSize) + ) + closeSync(fd) + + save(dataFile, { pr: { 'o/r#2': pr(2) }, issue: {} }) + expect(readGitHubCacheDatabase(dataFile).cache).toEqual({ pr: { 'o/r#2': pr(2) }, issue: {} }) + }) +}) diff --git a/src/main/persistence/loading-store/github-cache-database.ts b/src/main/persistence/loading-store/github-cache-database.ts new file mode 100644 index 00000000000..711008ceb60 --- /dev/null +++ b/src/main/persistence/loading-store/github-cache-database.ts @@ -0,0 +1,170 @@ +import { dirname, join } from 'node:path' +import type { PersistedState } from '../../../shared/persisted-state-types' +import type SyncDatabase from '../../sqlite/sync-database' +import { + isUnusableDatabaseError, + openVersionedCacheDatabase, + removeVersionedCacheDatabase +} from '../../sqlite/versioned-cache-database' +import { retireGitHubCacheSidecar, sha256 } from './github-cache-sidecar' + +type GitHubCache = PersistedState['githubCache'] +type GitHubCacheKind = keyof GitHubCache + +/** + * Next-launch copy of the PR/issue caches, one row per entry. A save rewrites only entries + * whose content changed; the JSON sidecar it replaces was rewritten whole on every flush. + * Disposable: a corrupt or mismatched database rebuilds empty and the next poll refills it. + */ +const SCHEMA_SQL = ` +CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS entries( + kind TEXT NOT NULL, + key TEXT NOT NULL, + hash TEXT NOT NULL, + payload TEXT NOT NULL, + PRIMARY KEY(kind, key) +) WITHOUT ROWID; +` + +/** Main-thread callers never wait out another writer's lock; they retry later instead. */ +export const GITHUB_CACHE_ROUTINE_BUSY_TIMEOUT_MS = 25 +/** Quit and explicit barriers may wait briefly so a transient lock does not drop the save. */ +export const GITHUB_CACHE_BARRIER_BUSY_TIMEOUT_MS = 500 + +export const GITHUB_CACHE_SCHEMA_VERSION = 1 + +const KINDS: readonly GitHubCacheKind[] = ['pr', 'issue'] + +export function getGithubCacheDatabaseFile(dataFile: string): string { + return join(dirname(dataFile), 'orca-github-cache.db') +} + +export const IMPORTED_SIDECAR_KEY = 'imported_sidecar_sha256' + +/** + * Save the cache, writing only rows whose content changed. `importedLegacyFileHash` names the + * sidecar whose content `cache` already holds (imported or superseded): the commit records it, + * then the returned promise removes the sidecar if it is still that file. If removal never + * happens (quit, failure), the recorded fingerprint keeps it from shadowing these rows. + * Throws synchronously on a failed commit, including SQLITE_BUSY. + */ +export function writeGitHubCacheDatabase( + dataFile: string, + cache: GitHubCache, + options: { busyTimeoutMs: number; importedLegacyFileHash: string | null } +): Promise { + const path = getGithubCacheDatabaseFile(dataFile) + const imported = options.importedLegacyFileHash + withCacheDatabase(path, options.busyTimeoutMs, (db) => writeEntries(db, cache, imported)) + return imported === null ? Promise.resolve() : retireGitHubCacheSidecar(dataFile, imported) +} + +export function isGitHubCacheBusyError(error: unknown): boolean { + const code = error instanceof Error && 'code' in error ? String(error.code) : '' + const message = error instanceof Error ? error.message : '' + return /SQLITE_BUSY|SQLITE_LOCKED/.test(code) || /database (is|table is) locked/i.test(message) +} + +/** + * Open, run, close: no handle outlives a call (profile moves, Windows). Corruption found + * anywhere, including in the rows, rebuilds the database once; lock and permission errors + * propagate unchanged so the caller retries later. + */ +function withCacheDatabase( + path: string, + busyTimeoutMs: number, + run: (db: SyncDatabase) => T +): T { + // Fail-fast removal: a rebuild blocked by a held file throws to the caller's retry. + const schema = { + version: GITHUB_CACHE_SCHEMA_VERSION, + sql: SCHEMA_SQL, + busyTimeoutMs, + removal: 'fail-fast' as const + } + for (let attempt = 0; ; attempt++) { + let db: SyncDatabase | null = null + try { + db = openVersionedCacheDatabase(path, schema) + return run(db) + } catch (error) { + if (attempt > 0 || !isUnusableDatabaseError(error)) { + throw error + } + db?.close() + db = null + removeVersionedCacheDatabase(path, 'fail-fast') + } finally { + db?.close() + } + } +} + +export function readGitHubCacheEntries(db: SyncDatabase): GitHubCache { + const cache: GitHubCache = { pr: {}, issue: {} } + for (const row of db.prepare('SELECT kind, key, payload FROM entries').all()) { + const kind = KINDS.find((candidate) => candidate === row.kind) + if (!kind || typeof row.key !== 'string' || typeof row.payload !== 'string') { + continue + } + try { + Object.assign(cache[kind], { [row.key]: JSON.parse(row.payload) }) + } catch { + // An unreadable row is dropped; the next save rewrites or deletes it. + } + } + return cache +} + +function writeEntries(db: SyncDatabase, cache: GitHubCache, importedSidecar: string | null): void { + const upsert = db.prepare( + `INSERT INTO entries(kind, key, hash, payload) VALUES (?, ?, ?, ?) + ON CONFLICT(kind, key) DO UPDATE SET hash = excluded.hash, payload = excluded.payload` + ) + const remove = db.prepare('DELETE FROM entries WHERE kind = ? AND key = ?') + db.exec('BEGIN IMMEDIATE') + try { + // Read the baseline under the write lock, so another writer's commit cannot leave it stale. + const stored = new Map() + for (const row of db.prepare('SELECT kind, key, hash FROM entries').all()) { + if (typeof row.kind === 'string' && typeof row.key === 'string') { + stored.set(rowId(row.kind, row.key), String(row.hash)) + } + } + for (const kind of KINDS) { + for (const [key, entry] of Object.entries(cache[kind] ?? {})) { + const payload = JSON.stringify(entry) + if (payload === undefined) { + continue + } + const id = rowId(kind, key) + const hash = sha256(payload) + if (stored.get(id) !== hash) { + upsert.run(kind, key, hash, payload) + } + stored.delete(id) + } + } + for (const id of stored.keys()) { + const [kind, key] = id.split('\u0000', 2) + remove.run(kind, key) + } + if (importedSidecar !== null) { + db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)').run( + IMPORTED_SIDECAR_KEY, + importedSidecar + ) + } + db.exec('COMMIT') + } catch (error) { + if (db.isTransaction) { + db.exec('ROLLBACK') + } + throw error + } +} + +function rowId(kind: string, key: string): string { + return `${kind}\u0000${key}` +} diff --git a/src/main/persistence/loading-store/github-cache-persistence.ts b/src/main/persistence/loading-store/github-cache-persistence.ts new file mode 100644 index 00000000000..b0487845315 --- /dev/null +++ b/src/main/persistence/loading-store/github-cache-persistence.ts @@ -0,0 +1,118 @@ +import type { StoreRuntimeState } from './store-runtime-state' +import { + GITHUB_CACHE_BARRIER_BUSY_TIMEOUT_MS, + GITHUB_CACHE_ROUTINE_BUSY_TIMEOUT_MS, + isGitHubCacheBusyError, + writeGitHubCacheDatabase +} from './github-cache-database' + +type GitHubCachePersistenceRuntime = Pick< + StoreRuntimeState, + | 'dataFile' + | 'githubCacheDirty' + | 'githubCacheLegacyFileHash' + | 'githubCachePersistedAt' + | 'githubCachePersistTimer' + | 'githubCacheRetryDelayMs' + | 'profileMaintenancePending' + | 'quitFlushStarted' + | 'state' + | 'writesFrozen' +> + +/** + * The cache only seeds next launch's sidebar before the first poll, so routine saves wait this + * long; quit and explicit flush barriers save at once. A crash loses at most this much churn, + * which the next poll refetches. + */ +const GITHUB_CACHE_PERSIST_INTERVAL_MS = 3 * 60_000 +const GITHUB_CACHE_FIRST_RETRY_MS = 5_000 + +/** + * `routine` saves wait for the interval (or a pending retry) and never run while maintenance, + * finalization or a freeze owns the profile. `barrier` is an admitted maintenance or final flush: + * it saves now. Barriers are best-effort for this disposable cache: a failure (such as another + * writer's lock) is logged and retried later rather than failing quit or a profile switch, and + * the cost is stale sidebar facts until the first poll after restart. + */ +export function persistGithubCache( + runtime: GitHubCachePersistenceRuntime, + mode: 'routine' | 'barrier' +): void { + if (!runtime.githubCacheDirty) { + return + } + if (mode === 'barrier') { + writeGithubCacheNow(runtime, true) + return + } + // An armed timer is the next save: the interval or a busy retry's backoff. + if (isFenced(runtime) || runtime.githubCachePersistTimer !== null) { + return + } + const remaining = runtime.githubCachePersistedAt + GITHUB_CACHE_PERSIST_INTERVAL_MS - Date.now() + if (remaining <= 0) { + writeGithubCacheNow(runtime, false) + } else { + armGithubCacheTimer(runtime, remaining, 'interval') + } +} + +/** Maintenance, freeze or quit owns the profile; resume (if any) rearms through persistGithubCache. */ +export function suspendGithubCachePersistence(runtime: GitHubCachePersistenceRuntime): void { + if (runtime.githubCachePersistTimer !== null) { + clearTimeout(runtime.githubCachePersistTimer) + runtime.githubCachePersistTimer = null + } +} + +/** `retry` saves at its own backoff deadline; `interval` re-applies the routine cadence. */ +function armGithubCacheTimer( + runtime: GitHubCachePersistenceRuntime, + delayMs: number, + kind: 'interval' | 'retry' +): void { + runtime.githubCachePersistTimer = setTimeout(() => { + runtime.githubCachePersistTimer = null + if (kind === 'interval') { + persistGithubCache(runtime, 'routine') + } else if (runtime.githubCacheDirty && !isFenced(runtime)) { + writeGithubCacheNow(runtime, false) + } + }, delayMs) + runtime.githubCachePersistTimer.unref?.() +} + +function isFenced(runtime: GitHubCachePersistenceRuntime): boolean { + return runtime.writesFrozen || runtime.profileMaintenancePending || runtime.quitFlushStarted +} + +function writeGithubCacheNow(runtime: GitHubCachePersistenceRuntime, barrier: boolean): void { + suspendGithubCachePersistence(runtime) + try { + // Synchronous: no later mutation can interleave, so the cache is clean once this returns. + const retired = writeGitHubCacheDatabase(runtime.dataFile, runtime.state.githubCache, { + busyTimeoutMs: barrier + ? GITHUB_CACHE_BARRIER_BUSY_TIMEOUT_MS + : GITHUB_CACHE_ROUTINE_BUSY_TIMEOUT_MS, + importedLegacyFileHash: runtime.githubCacheLegacyFileHash + }) + // A sidecar left behind is harmless: the committed fingerprint retires it next launch. + void retired.catch(() => {}) + runtime.githubCacheDirty = false + runtime.githubCacheLegacyFileHash = null + runtime.githubCachePersistedAt = Date.now() + runtime.githubCacheRetryDelayMs = 0 + } catch (error) { + // Another connection holds the lock (or the disk refused): back off rather than block. + const delay = Math.min( + Math.max(GITHUB_CACHE_FIRST_RETRY_MS, runtime.githubCacheRetryDelayMs * 2), + GITHUB_CACHE_PERSIST_INTERVAL_MS + ) + runtime.githubCacheRetryDelayMs = delay + armGithubCacheTimer(runtime, delay, 'retry') + if (!isGitHubCacheBusyError(error)) { + console.warn('[persistence] Failed to write github cache snapshot:', error) + } + } +} diff --git a/src/main/persistence/loading-store/github-cache-sidecar.ts b/src/main/persistence/loading-store/github-cache-sidecar.ts new file mode 100644 index 00000000000..52024c94df0 --- /dev/null +++ b/src/main/persistence/loading-store/github-cache-sidecar.ts @@ -0,0 +1,56 @@ +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { readFile, rm } from 'node:fs/promises' +import type { PersistedState } from '../../../shared/persisted-state-types' +import { getGithubCacheFile } from './user-data-path' + +type GitHubCache = PersistedState['githubCache'] + +export function sha256(content: string | Buffer): string { + return createHash('sha256').update(content).digest('hex') +} + +/** + * The JSON sidecar older builds write, parsed and fingerprinted from one read of its bytes, so a + * concurrent replacement can never pair one file's content with another's fingerprint. + * Missing or unparsable sidecars read as null, as they always have. + */ +export function readGitHubCacheSidecar(dataFile: string): GitHubCacheSidecar | null { + let bytes: Buffer + try { + // One-shot: the synchronous profile load reads this once at startup. + bytes = readFileSync(getGithubCacheFile(dataFile)) + } catch { + return null + } + return parseGitHubCacheSidecar(bytes) +} + +/** Remove the sidecar off the save path, only if it is still the file the rows superseded. */ +export async function retireGitHubCacheSidecar(dataFile: string, hash: string): Promise { + const path = getGithubCacheFile(dataFile) + const bytes = await readFile(path).catch(() => null) + if (bytes && parseGitHubCacheSidecar(bytes)?.hash === hash) { + await rm(path, { force: true }) + } +} + +type GitHubCacheSidecar = { cache: GitHubCache; hash: string } + +function parseGitHubCacheSidecar(bytes: Buffer): GitHubCacheSidecar | null { + let parsed: unknown + try { + parsed = JSON.parse(bytes.toString('utf-8')) + } catch { + return null + } + if (!isRecord(parsed) || !isRecord(parsed.pr) || !isRecord(parsed.issue)) { + return null + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: same shape check the sidecar reader always applied; entries stay opaque cache payloads. + return { cache: parsed as GitHubCache, hash: sha256(bytes) } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/src/main/persistence/loading-store/github-cache-startup-read.ts b/src/main/persistence/loading-store/github-cache-startup-read.ts new file mode 100644 index 00000000000..8374d930576 --- /dev/null +++ b/src/main/persistence/loading-store/github-cache-startup-read.ts @@ -0,0 +1,100 @@ +import { existsSync } from 'node:fs' +import type { PersistedState } from '../../../shared/persisted-state-types' +import SyncDatabase from '../../sqlite/sync-database' +import { + isUnusableDatabaseError, + removeVersionedCacheDatabase +} from '../../sqlite/versioned-cache-database' +import { + GITHUB_CACHE_ROUTINE_BUSY_TIMEOUT_MS, + GITHUB_CACHE_SCHEMA_VERSION, + getGithubCacheDatabaseFile, + IMPORTED_SIDECAR_KEY, + readGitHubCacheEntries +} from './github-cache-database' +import { readGitHubCacheSidecar } from './github-cache-sidecar' + +type GitHubCache = PersistedState['githubCache'] + +export type StoredGitHubCache = { + /** What to hydrate, or null when nothing can be hydrated now. */ + cache: GitHubCache | null + /** `cache` came from an older build's sidecar, so it must be saved as rows. */ + fromSidecar: boolean + /** Fingerprint of a sidecar to retire once rows that supersede it commit. */ + sidecarHash: string | null +} + +type SavedRows = + | { kind: 'rows'; cache: GitHubCache; importedSidecar: string | null } + | { kind: 'empty' } + /** Rows exist but cannot be read now, usually another connection's lock. */ + | { kind: 'unavailable' } + +/** + * Startup read. An older build's sidecar is newer than the rows unless the rows record having + * imported exactly its bytes; while the rows cannot be read, that is unknown, so the sidecar is + * neither hydrated nor allowed to replace rows (the first commit supersedes it instead). + */ +export function readGitHubCacheDatabase(dataFile: string): StoredGitHubCache { + const sidecar = readGitHubCacheSidecar(dataFile) + const saved = readSavedRows(getGithubCacheDatabaseFile(dataFile)) + if (sidecar && saved.kind !== 'unavailable') { + if (saved.kind === 'empty' || saved.importedSidecar !== sidecar.hash) { + return { cache: sidecar.cache, fromSidecar: true, sidecarHash: sidecar.hash } + } + } + const rows = saved.kind === 'rows' ? saved.cache : null + const hasRows = rows !== null && Object.keys(rows.pr).length + Object.keys(rows.issue).length > 0 + return { cache: hasRows ? rows : null, fromSidecar: false, sidecarHash: sidecar?.hash ?? null } +} + +/** One read-only snapshot: no write lock, so another writer cannot block or fail startup. */ +function readSavedRows(path: string): SavedRows { + // One-shot: the synchronous profile load reads this once at startup. + if (!existsSync(path)) { + return { kind: 'empty' } + } + let db: SyncDatabase | null = null + try { + db = new SyncDatabase(path, { readonly: true, fileMustExist: true }) + db.pragma(`busy_timeout = ${GITHUB_CACHE_ROUTINE_BUSY_TIMEOUT_MS}`) + db.exec('BEGIN') + try { + const tables = db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'entries')" + ) + .all() + const version = + tables.length === 2 + ? db.prepare("SELECT value FROM meta WHERE key = 'schema_version'").get() + : undefined + if (version?.value !== String(GITHUB_CACHE_SCHEMA_VERSION)) { + // Another schema rebuilds empty at the next save, as a mismatched cache always has. + return { kind: 'empty' } + } + const marker = db.prepare('SELECT value FROM meta WHERE key = ?').get(IMPORTED_SIDECAR_KEY) + return { + kind: 'rows', + cache: readGitHubCacheEntries(db), + importedSidecar: typeof marker?.value === 'string' ? marker.value : null + } + } finally { + if (db.isTransaction) { + db.exec('COMMIT') + } + } + } catch (error) { + if (!isUnusableDatabaseError(error)) { + return { kind: 'unavailable' } + } + db?.close() + db = null + // Damaged rows hold nothing recoverable; the next save rebuilds the cache. + removeVersionedCacheDatabase(path, 'fail-fast') + return { kind: 'empty' } + } finally { + db?.close() + } +} diff --git a/src/main/persistence/loading-store/loaded-state-parsing.ts b/src/main/persistence/loading-store/loaded-state-parsing.ts index f5225f4134b..9bfe817f6d7 100644 --- a/src/main/persistence/loading-store/loaded-state-parsing.ts +++ b/src/main/persistence/loading-store/loaded-state-parsing.ts @@ -18,7 +18,7 @@ import { isLegacyOpenCodeSessionCookie, isLegacySshPtyOwnerLease } from '../leasing-ssh-ptys/secret-validation' -import { readGithubCacheSnapshot } from './user-data-path' +import { readGitHubCacheDatabase } from './github-cache-startup-read' import { retainLegacyOpenCodeGoApiKey } from './legacy-opencode-go-api-key-migration' import { gcStaleWorktreeMeta, @@ -50,6 +50,7 @@ type LoadedStateParsingOperationsRuntime = Pick< StoreRuntimeState, | 'dataFile' | 'githubCacheDirty' + | 'githubCacheLegacyFileHash' | 'loadNeedsSave' | 'originalExperimentalNativeChat' | 'protectedSecrets' @@ -287,18 +288,25 @@ export class LoadedStateParsingOperations { chatUiLaunchTipAudience: audience } - // Legacy inline caches seed the session, then move to the cache sidecar. + // Cache precedence, resolved once: an inline legacy cache, else an older build's sidecar, + // else the saved rows. Whatever wins is saved as rows, which retires any sidecar it supersedes. const legacyCache = migrated.githubCache const hasLegacyCache = Object.keys(legacyCache?.pr ?? {}).length > 0 || Object.keys(legacyCache?.issue ?? {}).length > 0 + const stored = readStoredGitHubCache(this.runtime.dataFile) if (hasLegacyCache) { this.runtime.loadNeedsSave = true // Persist the seed even when no cache poll refresh occurs this session. this.runtime.githubCacheDirty = true - } else { - migrated.githubCache = readGithubCacheSnapshot(this.runtime.dataFile) ?? migrated.githubCache + } else if (stored?.cache) { + migrated.githubCache = stored.cache + // A sidecar import, or rows that already hold it, saves so the sidecar retires. + this.runtime.githubCacheDirty ||= stored.sidecarHash !== null } + // While the rows are unreadable nothing is dirty, so no save can replace them; a later + // commit of polled state supersedes the unhydrated sidecar instead. + this.runtime.githubCacheLegacyFileHash = stored?.sidecarHash ?? null logStartupMilestone('persistence-load-done', () => ({ repos: migrated.repos.length, @@ -307,3 +315,15 @@ export class LoadedStateParsingOperations { return migrated } } + +function readStoredGitHubCache( + dataFile: string +): ReturnType | null { + try { + return readGitHubCacheDatabase(dataFile) + } catch (error) { + // A disposable cache must never block startup; the first poll refills it. + console.warn('[persistence] Failed to read github cache:', error) + return null + } +} diff --git a/src/main/persistence/loading-store/profile-preferences.ts b/src/main/persistence/loading-store/profile-preferences.ts index d7143a3e82b..4741d1a49f4 100644 --- a/src/main/persistence/loading-store/profile-preferences.ts +++ b/src/main/persistence/loading-store/profile-preferences.ts @@ -23,6 +23,7 @@ import { } from '../applying-settings/feature-interaction-recording' import type { StoreRuntimeState } from './store-runtime-state' +import { persistGithubCache } from './github-cache-persistence' import type { WriteSchedulingOperations } from './write-scheduling' import { scheduleSave } from './write-scheduling' import { bumpLocalWorktreeScanGeneration } from '../../workspaces/catalog/local-worktree-scan-generation' @@ -30,9 +31,16 @@ import { bumpLocalWorktreeScanGeneration } from '../../workspaces/catalog/local- type ProfilePreferencesRuntime = Pick< StoreRuntimeState, | 'activeViewPreference' + | 'dataFile' | 'githubCacheDirty' - | 'githubCacheGeneration' + | 'githubCachePersistedAt' + | 'githubCacheLegacyFileHash' + | 'githubCachePersistTimer' + | 'githubCacheRetryDelayMs' | 'githubCacheWriteListeners' + | 'profileMaintenancePending' + | 'quitFlushStarted' + | 'writesFrozen' | 'protectedSecrets' | 'runDurableMutation' | 'settingsChangeListeners' @@ -197,7 +205,7 @@ export class ProfilePreferences { // Cache polls write a sidecar at flush; scheduling a full profile save here amplifies every poll. this[profilePreferencesContext].runtime.state.githubCache = cache this[profilePreferencesContext].runtime.githubCacheDirty = true - this[profilePreferencesContext].runtime.githubCacheGeneration += 1 + persistGithubCache(this[profilePreferencesContext].runtime, 'routine') for (const listener of this[profilePreferencesContext].runtime.githubCacheWriteListeners) { listener() } diff --git a/src/main/persistence/loading-store/profile-state-maintenance.ts b/src/main/persistence/loading-store/profile-state-maintenance.ts index 85e3bae7d3b..3f04eba8e92 100644 --- a/src/main/persistence/loading-store/profile-state-maintenance.ts +++ b/src/main/persistence/loading-store/profile-state-maintenance.ts @@ -5,6 +5,7 @@ import type { StoreRuntimeState } from './store-runtime-state' import { drainProfileStateOperations } from './profile-state-flush-lifetime' import { flushCurrentStateAsync } from './write-flush-barriers' import { scheduleSave } from './write-scheduling' +import { persistGithubCache, suspendGithubCachePersistence } from './github-cache-persistence' export type ProfileStateMaintenanceOptions = { signal?: AbortSignal @@ -21,6 +22,7 @@ export function freezeProfileStateWrites(runtime: StoreRuntimeState): void { clearTimeout(runtime.writeTimer) runtime.writeTimer = null } + suspendGithubCachePersistence(runtime) runtime.profileStateAuthority?.close?.() } @@ -30,6 +32,7 @@ export async function freezeProfileStateWritesAsync(runtime: StoreRuntimeState): clearTimeout(runtime.writeTimer) runtime.writeTimer = null } + suspendGithubCachePersistence(runtime) try { await drainProfileStateOperations([ runtime.pendingProfileMaintenance, @@ -56,6 +59,7 @@ export function beginProfileStateMaintenance( clearTimeout(runtime.writeTimer) runtime.writeTimer = null } + suspendGithubCachePersistence(runtime) const resumePreference = runtime.activeViewPreference.pauseForMaintenance() const resumeScheduling = () => { runtime.writesFrozen = false @@ -63,6 +67,8 @@ export function beginProfileStateMaintenance( runtime.pendingProfileMaintenance = null resumePreference() scheduleSave(domains.scheduling) + // Updates admitted while paused (or a retry it skipped) resume their routine save. + persistGithubCache(runtime, 'routine') } const paused = pauseProfileState(runtime, domains, options).then((authority) => { let consumed = false @@ -162,7 +168,7 @@ async function drainProfileFileWork(runtime: StoreRuntimeState): Promise { runtime.staleProfileStateTempCleanup, runtime.pendingWrite, runtime.pendingSnapshotFileWork, - runtime.pendingGithubCacheWrite, + runtime.staleGithubCacheTempCleanup, runtime.activeViewPreference.waitForPendingWrite(), runtime.profileStateAuthority?.drainBackups?.( runtime.profileMaintenancePending || runtime.quitFlushStarted diff --git a/src/main/persistence/loading-store/store-runtime-state.ts b/src/main/persistence/loading-store/store-runtime-state.ts index 6a03d732750..52c3a1ed4eb 100644 --- a/src/main/persistence/loading-store/store-runtime-state.ts +++ b/src/main/persistence/loading-store/store-runtime-state.ts @@ -83,9 +83,12 @@ export class StoreRuntimeState { dirtyProfileStateDomains: Set | null = new Set() pendingAutomationRunsAfter: readonly AutomationRun[] | undefined githubCacheDirty = false - githubCacheGeneration = 0 + githubCachePersistedAt = Date.now() + githubCachePersistTimer: ReturnType | null = null + githubCacheRetryDelayMs = 0 + /** Fingerprint of a sidecar the loaded cache imported or supersedes; retired once rows commit. */ + githubCacheLegacyFileHash: string | null = null readonly githubCacheWriteListeners = new Set<() => void>() - pendingGithubCacheWrite: Promise | null = null readonly staleGithubCacheTempCleanup: Promise readonly staleProfileStateTempCleanup: Promise readonly gitUsernameCache = new Map() diff --git a/src/main/persistence/loading-store/user-data-path.ts b/src/main/persistence/loading-store/user-data-path.ts index 470d611f49b..bbbd5af3d1a 100644 --- a/src/main/persistence/loading-store/user-data-path.ts +++ b/src/main/persistence/loading-store/user-data-path.ts @@ -1,7 +1,6 @@ import { getAppEnvironment } from '../../../shared/app-environment' -import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs' +import { copyFileSync, existsSync, mkdirSync, rmSync } from 'node:fs' import { dirname, join, resolve } from 'node:path' -import type { PersistedState } from '../../../shared/persisted-state-types' import { hardenExistingSecureFile } from '../../../shared/secure-storage/secure-file' import { MOBILE_PAIRING_USERDATA_FILES } from '../../runtime/rpc/server/pairing/mobile-pairing-files' @@ -29,22 +28,6 @@ export function getGithubCacheFile(dataFile = getDataFile()): string { return join(dirname(dataFile), 'orca-github-cache.json') } -export function readGithubCacheSnapshot(dataFile: string): PersistedState['githubCache'] | null { - try { - const parsed = JSON.parse(readFileSync(getGithubCacheFile(dataFile), 'utf-8')) as unknown - const isPlainRecord = (value: unknown): value is Record => - typeof value === 'object' && value !== null && !Array.isArray(value) - if ( - isPlainRecord(parsed) && - isPlainRecord((parsed as { pr?: unknown }).pr) && - isPlainRecord((parsed as { issue?: unknown }).issue) - ) { - return parsed as PersistedState['githubCache'] - } - } catch {} - return null -} - /** Reuse the path captured before app.setName; late resolution can change casing and strand paired devices. */ export function getCanonicalUserDataPath(): string { if (!_userDataDir) { diff --git a/src/main/persistence/loading-store/write-flush-barriers.ts b/src/main/persistence/loading-store/write-flush-barriers.ts index bb302fcdadb..66ba9940203 100644 --- a/src/main/persistence/loading-store/write-flush-barriers.ts +++ b/src/main/persistence/loading-store/write-flush-barriers.ts @@ -1,7 +1,4 @@ -import { writeFileSync, renameSync, unlinkSync } from 'node:fs' -import { rename, rm, writeFile } from 'node:fs/promises' -import { durableWriteTempPath } from '../../filesystem/durable-file-write' -import { getGithubCacheFile } from './user-data-path' +import { persistGithubCache } from './github-cache-persistence' import type { StoreRuntimeState } from './store-runtime-state' import type { PrimaryStateWriteOperations } from './primary-state-writes' @@ -16,16 +13,17 @@ type WriteFlushBarrierOperationsRuntime = Pick< | 'dataFile' | 'firstPendingSaveAt' | 'githubCacheDirty' - | 'githubCacheGeneration' + | 'githubCachePersistedAt' + | 'githubCacheLegacyFileHash' + | 'githubCachePersistTimer' + | 'githubCacheRetryDelayMs' | 'lastDurableWriteGeneration' - | 'pendingGithubCacheWrite' | 'pendingProfileFlushes' | 'pendingProfileMaintenance' | 'profileMaintenancePending' | 'profileStateAuthority' | 'quitFlushPromise' | 'quitFlushStarted' - | 'staleGithubCacheTempCleanup' | 'staleProfileStateTempCleanup' | 'state' | 'writeGeneration' @@ -70,7 +68,7 @@ export class WriteFlushBarrierOperations { } catch (err) { console.error('[active-view] Failed to flush preference:', err) } - writeGithubCacheSnapshotSync(this) + persistGithubCache(runtime, 'routine') } flushAsync(): Promise { @@ -177,7 +175,11 @@ export async function flushCurrentStateAsync( await (final ? runtime.activeViewPreference.flushAsync() : runtime.activeViewPreference.flushPendingAsync(signal)) - await writeGithubCacheSnapshotAsync(owner, final, signal) + // Quit and explicit barriers save the cache now; best-effort flushes may wait. + persistGithubCache( + runtime, + final || requireInitialGenerationDurable ? 'barrier' : 'routine' + ) if (final || runtime.profileMaintenancePending) { await runtime.profileStateAuthority?.drainBackups?.(true) } @@ -200,83 +202,3 @@ export async function flushCurrentStateAsync( } }) } - -export async function writeGithubCacheSnapshotAsync( - owner: WriteFlushBarrierOperations, - drainToStableGeneration = true, - signal?: AbortSignal -): Promise { - const { runtime } = owner[writeFlushBarrierOperationsContext] - if (!runtime.githubCacheDirty) { - return - } - const previousWrite = runtime.pendingGithubCacheWrite ?? runtime.staleGithubCacheTempCleanup - const nextWrite = previousWrite - .then(async () => { - while (runtime.githubCacheDirty) { - if (signal?.aborted) { - throw new Error('GitHub cache flush aborted') - } - const generation = runtime.githubCacheGeneration - const cacheFile = getGithubCacheFile(runtime.dataFile) - const tmpFile = durableWriteTempPath(cacheFile) - let renamed = false - try { - await writeFile(tmpFile, JSON.stringify(runtime.state.githubCache), 'utf-8') - if (generation === runtime.githubCacheGeneration) { - await rename(tmpFile, cacheFile) - renamed = true - if (generation === runtime.githubCacheGeneration) { - runtime.githubCacheDirty = false - } - } - } finally { - if (!renamed) { - await rm(tmpFile).catch(() => {}) - } - } - if (signal?.aborted) { - throw new Error('GitHub cache flush aborted') - } - if (!drainToStableGeneration) { - break - } - } - }) - .catch((err) => { - console.warn('[persistence] Failed to write github cache snapshot:', err) - }) - .finally(() => { - if (runtime.pendingGithubCacheWrite === nextWrite) { - runtime.pendingGithubCacheWrite = null - } - }) - runtime.pendingGithubCacheWrite = nextWrite - await nextWrite -} - -export function writeGithubCacheSnapshotSync(owner: WriteFlushBarrierOperations): void { - const { runtime } = owner[writeFlushBarrierOperationsContext] - if (!runtime.githubCacheDirty) { - return - } - if (runtime.pendingGithubCacheWrite) { - void writeGithubCacheSnapshotAsync(owner) - return - } - const cacheFile = getGithubCacheFile(runtime.dataFile) - const generation = runtime.githubCacheGeneration - const tmpFile = durableWriteTempPath(cacheFile) - try { - writeFileSync(tmpFile, JSON.stringify(runtime.state.githubCache), 'utf-8') - renameSync(tmpFile, cacheFile) - if (generation === runtime.githubCacheGeneration) { - runtime.githubCacheDirty = false - } - } catch (err) { - try { - unlinkSync(tmpFile) - } catch {} - console.warn('[persistence] Failed to write github cache snapshot:', err) - } -} diff --git a/src/main/persistence/persistence-async-write-syscalls.test.ts b/src/main/persistence/persistence-async-write-syscalls.test.ts index 19e9049f3b6..499df38a519 100644 --- a/src/main/persistence/persistence-async-write-syscalls.test.ts +++ b/src/main/persistence/persistence-async-write-syscalls.test.ts @@ -152,27 +152,6 @@ describe('worker persistence avoids synchronous profile filesystem calls', () => expect(readState().ui.sidebarWidth).toBe(511) }) - it('drains mutations made while a sidecar rename is stalled', async () => { - const { store, readState } = await createWorkerMaintenanceFixture() - const started = maintenanceBarrier() - const finish = maintenanceBarrier() - fsCalls.waitAsync = (name, path) => { - if (name !== 'rename' || !path.includes('orca-github-cache.json.')) { - return - } - started.resolve() - return finish.promise - } - store.updateUI({ sidebarWidth: 611 }) - store.setGitHubCache({ pr: {}, issue: {} }) - const barrier = store.flushPendingOrThrowAsync() - await started.promise - store.updateUI({ sidebarWidth: 612 }) - finish.resolve() - await barrier - expect(readState().ui.sidebarWidth).toBe(612) - }) - it('serializes queued durable mutations and refuses a synchronous live flush', async () => { const { store, authority, readState } = await createWorkerMaintenanceFixture() const gate = pauseCommit(authority) diff --git a/src/main/persistence/persistence-flush-and-save-scheduling.test.ts b/src/main/persistence/persistence-flush-and-save-scheduling.test.ts index cec21e4fd09..163c2505dff 100644 --- a/src/main/persistence/persistence-flush-and-save-scheduling.test.ts +++ b/src/main/persistence/persistence-flush-and-save-scheduling.test.ts @@ -190,7 +190,7 @@ describe('Store', () => { }) }) - it('keeps GitHub cache sidecars scoped to explicit profile data files', async () => { + it('keeps the GitHub cache scoped to explicit profile data files', async () => { const profileADir = join(testState.dir, 'profiles', 'a') const profileBDir = join(testState.dir, 'profiles', 'b') const profileADataFile = join(profileADir, 'orca-data.json') @@ -204,12 +204,12 @@ describe('Store', () => { initDataPath() const profileAStore = createSqliteTestStore(Store, { dataFile: profileADataFile }) profileAStore.setGitHubCache({ pr: { 'o/r#a': { fetchedAt: 10 } as never }, issue: {} }) - profileAStore.flush() + await profileAStore.flushPendingOrThrowAsync() const profileBStore = createSqliteTestStore(Store, { dataFile: profileBDataFile }) expect(profileBStore.getGitHubCache().pr['o/r#a']).toBeUndefined() profileBStore.setGitHubCache({ pr: { 'o/r#b': { fetchedAt: 20 } as never }, issue: {} }) - profileBStore.flush() + await profileBStore.flushPendingOrThrowAsync() const restartedProfileA = createSqliteTestStore(Store, { dataFile: profileADataFile }) const restartedProfileB = createSqliteTestStore(Store, { dataFile: profileBDataFile }) diff --git a/src/main/startup/quit/quit-path-durable-write-blocking.test.ts b/src/main/startup/quit/quit-path-durable-write-blocking.test.ts index 340e732e023..141b1bea1a1 100644 --- a/src/main/startup/quit/quit-path-durable-write-blocking.test.ts +++ b/src/main/startup/quit/quit-path-durable-write-blocking.test.ts @@ -286,45 +286,6 @@ describe('quit-path durable writes never park the main thread', () => { expect(readState(dir).ui.sidebarWidth).toBe(200) }) - it('serializes github-cache snapshots and keeps the newest generation', async () => { - const dir = makeDir() - const store = await createStore(dir) - let releaseWrite!: () => void - const writeRelease = new Promise((resolve) => { - releaseWrite = resolve - }) - let signalWrite!: () => void - releases.push(releaseWrite) - const writeStarted = new Promise((resolve) => { - signalWrite = resolve - }) - let held = false - fsCalls.dirPrefix = dir - fsCalls.recording = true - fsCalls.waitAsync = (fn, target) => { - if (held || fn !== 'writeFile' || !target.includes('orca-github-cache.json')) { - return null - } - held = true - signalWrite() - return writeRelease - } - - store.setGitHubCache({ pr: { test: { data: null, fetchedAt: 1 } }, issue: {} }) - const finalFlush = store.flushAsync() - await writeStarted - store.setGitHubCache({ pr: { test: { data: null, fetchedAt: 2 } }, issue: {} }) - - releaseWrite() - await finalFlush - fsCalls.recording = false - - expect(JSON.parse(readFileSync(join(dir, 'orca-github-cache.json'), 'utf-8'))).toEqual({ - pr: { test: { data: null, fetchedAt: 2 } }, - issue: {} - }) - }) - it('stats.flushAsync() wins over a debounced write already in flight', async () => { const dir = makeDir() const stats = await createStatsCollector(dir)