From 3efd991ee3f7a7d474697fdf76b587f334aec53f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:17:54 -0700 Subject: [PATCH] perf(codex): bound launch session migration scans --- .../runtime-home-service.test.ts | 12 +++++ .../codex-accounts/runtime-home-service.ts | 38 +++++++++----- src/main/codex/codex-session-backfill-date.ts | 51 +++++++++++++++++++ .../codex/codex-session-backfill-marker.ts | 48 +++-------------- .../codex/codex-session-backfill-types.ts | 4 ++ src/main/codex/codex-session-backfill.test.ts | 27 ++++++++++ src/main/codex/codex-session-backfill.ts | 12 ++--- .../codex-session-migration-scheduler.test.ts | 51 +++++++++++++++++++ .../codex-session-migration-scheduler.ts | 44 +++++++++++++--- 9 files changed, 216 insertions(+), 71 deletions(-) create mode 100644 src/main/codex/codex-session-backfill-date.ts diff --git a/src/main/codex-accounts/runtime-home-service.test.ts b/src/main/codex-accounts/runtime-home-service.test.ts index 991840f0271..f629959f2cb 100644 --- a/src/main/codex-accounts/runtime-home-service.test.ts +++ b/src/main/codex-accounts/runtime-home-service.test.ts @@ -1010,6 +1010,18 @@ describe('CodexRuntimeHomeService', () => { expect(service.prepareForCodexLaunch()).toBe(getRuntimeCodexHomePath()) expect(existsSync(markerPath)).toBe(false) expect(scheduleSessionMigration).toHaveBeenCalledOnce() + expect(scheduleSessionMigration).toHaveBeenCalledWith(true) + writeFileSync( + markerPath, + `${JSON.stringify({ + version: 3, + systemSessionsRoot: join(getSystemCodexHomePath(), 'sessions'), + summary: { scannedFiles: 1 } + })}\n`, + 'utf-8' + ) + service.prepareForCodexLaunch() + expect(scheduleSessionMigration).toHaveBeenLastCalledWith(false) expect(service.prepareForRateLimitFetch()).toBe(getRuntimeCodexHomePath()) expect(service.getHostCodexHomePathsForSessionDiscovery()).toEqual([getRuntimeCodexHomePath()]) expect(existsSync(getRuntimeCodexHomePath())).toBe(true) diff --git a/src/main/codex-accounts/runtime-home-service.ts b/src/main/codex-accounts/runtime-home-service.ts index 2cdee81832a..9f3dde190e6 100644 --- a/src/main/codex-accounts/runtime-home-service.ts +++ b/src/main/codex-accounts/runtime-home-service.ts @@ -41,7 +41,6 @@ import { import { getOrcaManagedCodexHomePath, getOrcaUserDataPath, - getCodexSessionBackfillStateDirPath, getSystemCodexHomePath, resolveOrcaManagedCodexHomePath, syncCodexGlobalInstructionsIntoManagedHome, @@ -68,7 +67,11 @@ import { } from './runtime-selection' import { getDefaultWslDistro, getWslHome } from '../wsl' import { hasCustomCodexHomeOverrideForLaunch } from '../codex/codex-real-home-path' -import { invalidateCodexSessionBackfillMarker } from '../codex/codex-session-backfill-marker' +import { + hasCompletedCodexSessionBackfillMarker, + invalidateCodexSessionBackfillMarker +} from '../codex/codex-session-backfill-marker' +import { resolveCodexSessionBackfillPaths } from '../codex/codex-session-backfill' import { assertOwnedHostCodexManagedHomePath } from './host-codex-managed-home-ownership' import { codexAuthCouldBelongToManagedAccount, @@ -181,7 +184,7 @@ export class CodexRuntimeHomeService { private sharedAuthRefreshBlockedByManagedTransition = false // Why: transient auth.json read/parse failures must not deselect an account. private readonly credentialAbsenceGrace = new CodexCredentialAbsenceGrace() - private scheduleHostSystemDefaultSessionMigration: () => void = () => {} + private scheduleHostSystemDefaultSessionMigration: (fullScanRequired: boolean) => void = () => {} constructor(private readonly store: Store) { this.safeRecoverInterruptedRuntimeAuthOperation() @@ -242,7 +245,7 @@ export class CodexRuntimeHomeService { this.reconcileLegacySharedHomeForRetainedPanes() return null } - const shouldScheduleSessionMigration = + const sessionMigrationFullScanRequired = this.invalidateBackfillAfterManagedSystemDefaultLaunch(launchEnv) this.syncForCurrentSelection(target, launchEnv) syncSystemCodexResourcesIntoManagedHome() @@ -252,13 +255,15 @@ export class CodexRuntimeHomeService { {}, resolveHostCodexSessionSourceHome(this.store.getSettings()) ) - if (shouldScheduleSessionMigration) { - this.scheduleHostSystemDefaultSessionMigration() + if (sessionMigrationFullScanRequired !== null) { + this.scheduleHostSystemDefaultSessionMigration(sessionMigrationFullScanRequired) } return this.getRuntimeHomePath() } - setHostSystemDefaultSessionMigrationScheduler(schedule: () => void): void { + setHostSystemDefaultSessionMigrationScheduler( + schedule: (fullScanRequired: boolean) => void + ): void { this.scheduleHostSystemDefaultSessionMigration = schedule } @@ -269,10 +274,16 @@ export class CodexRuntimeHomeService { ) } - prepareHostSystemDefaultSessionMigrationPass(): void { - invalidateCodexSessionBackfillMarker( - join(getCodexSessionBackfillStateDirPath(), 'backfill-complete.json') + prepareHostSystemDefaultSessionMigrationPass(): boolean { + const paths = resolveCodexSessionBackfillPaths( + resolveHostCodexSessionSourceHome(this.store.getSettings()) ) + const fullScanRequired = !hasCompletedCodexSessionBackfillMarker( + paths.markerPath, + paths.systemSessionsRoot + ) + invalidateCodexSessionBackfillMarker(paths.markerPath) + return fullScanRequired } // Why: a managed HOST account runs against its own self-contained CODEX_HOME @@ -428,16 +439,15 @@ export class CodexRuntimeHomeService { private invalidateBackfillAfterManagedSystemDefaultLaunch( launchEnv?: NodeJS.ProcessEnv - ): boolean { + ): boolean | null { const settings = this.store.getSettings() if ( normalizeCodexRuntimeSelection(settings).host !== null || hasCustomCodexHomeOverrideForLaunch(launchEnv) ) { - return false + return null } - this.prepareHostSystemDefaultSessionMigrationPass() - return true + return this.prepareHostSystemDefaultSessionMigrationPass() } private startWslSessionBridgeForLaunch( diff --git a/src/main/codex/codex-session-backfill-date.ts b/src/main/codex/codex-session-backfill-date.ts new file mode 100644 index 00000000000..2b66118a8f2 --- /dev/null +++ b/src/main/codex/codex-session-backfill-date.ts @@ -0,0 +1,51 @@ +import { join } from 'node:path' +import { listCodexSessionJsonlFilesIncrementally } from './codex-session-file-listing' +import type { + CodexSessionBackfillDate, + CodexSessionBackfillOptions +} from './codex-session-backfill-types' + +export function getCodexSessionBackfillDate(date = new Date()): CodexSessionBackfillDate { + return [ + String(date.getFullYear()).padStart(4, '0'), + String(date.getMonth() + 1).padStart(2, '0'), + String(date.getDate()).padStart(2, '0') + ] +} + +export async function* listCodexSessionBackfillFilesForDates( + sessionsRoot: string, + options: CodexSessionBackfillOptions, + onDirectoryError: (directoryPath: string, error: unknown) => void | Promise +): AsyncGenerator { + const scanRoots = resolveCodexSessionBackfillDateRoots(sessionsRoot, options.scanDates) + for (const scanRoot of scanRoots) { + yield* listCodexSessionJsonlFilesIncrementally( + scanRoot, + options, + async (directoryPath, error) => { + if (directoryPath !== scanRoot || !isNotFoundError(error)) { + await onDirectoryError(directoryPath, error) + } + } + ) + } +} + +function resolveCodexSessionBackfillDateRoots( + sessionsRoot: string, + scanDates: readonly CodexSessionBackfillDate[] | undefined +): string[] { + if (!scanDates?.length) { + return [sessionsRoot] + } + return scanDates + .filter( + ([year, month, day]) => /^\d{4}$/.test(year) && /^\d{2}$/.test(month) && /^\d{2}$/.test(day) + ) + .map(([year, month, day]) => join(sessionsRoot, year, month, day)) +} + +function isNotFoundError(error: unknown): boolean { + return (error as NodeJS.ErrnoException | null)?.code === 'ENOENT' +} diff --git a/src/main/codex/codex-session-backfill-marker.ts b/src/main/codex/codex-session-backfill-marker.ts index 1f519672a06..4436c65fe98 100644 --- a/src/main/codex/codex-session-backfill-marker.ts +++ b/src/main/codex/codex-session-backfill-marker.ts @@ -1,5 +1,5 @@ -import { mkdirSync, readFileSync, readdirSync, rmSync } from 'node:fs' -import { dirname, join } from 'node:path' +import { mkdirSync, readFileSync, rmSync } from 'node:fs' +import { dirname } from 'node:path' import { writeFileAtomically } from '../codex-accounts/fs-utils' import type { CodexSessionBackfillSummary } from './codex-session-backfill-types' @@ -14,8 +14,7 @@ export function captureCodexSessionBackfillMarkerGeneration(): number { export function hasCompletedCodexSessionBackfillMarker( markerPath: string, - systemSessionsRoot: string, - managedSessionsRoot: string + systemSessionsRoot: string ): boolean { try { const parsed: unknown = JSON.parse(readFileSync(markerPath, 'utf-8')) @@ -35,49 +34,14 @@ export function hasCompletedCodexSessionBackfillMarker( if (!markerMatchesTarget) { return false } - // Why: an empty source can become populated after an early migration run. - return marker.summary?.scannedFiles !== 0 || !managedSessionsRootHasRollout(managedSessionsRoot) + // Why: an empty source can become populated after an early migration run; + // let the incremental async walk verify it without blocking the main thread. + return marker.summary?.scannedFiles !== 0 } catch { return false } } -function managedSessionsRootHasRollout(managedSessionsRoot: string): boolean { - try { - for (const year of readdirSync(managedSessionsRoot, { withFileTypes: true })) { - if (!year.isDirectory() || !/^\d{4}$/.test(year.name)) { - continue - } - const yearPath = join(managedSessionsRoot, year.name) - for (const month of readdirSync(yearPath, { withFileTypes: true })) { - if (!month.isDirectory() || !/^\d{2}$/.test(month.name)) { - continue - } - const monthPath = join(yearPath, month.name) - for (const day of readdirSync(monthPath, { withFileTypes: true })) { - if (!day.isDirectory() || !/^\d{2}$/.test(day.name)) { - continue - } - const dayPath = join(monthPath, day.name) - if ( - readdirSync(dayPath, { withFileTypes: true }).some( - (entry) => entry.isFile() && /^rollout-.+\.jsonl$/.test(entry.name) - ) - ) { - return true - } - } - } - } - return false - } catch (error) { - if ((error as NodeJS.ErrnoException | null)?.code === 'ENOENT') { - return false - } - throw error - } -} - export function writeCodexSessionBackfillMarker( markerPath: string, systemSessionsRoot: string, diff --git a/src/main/codex/codex-session-backfill-types.ts b/src/main/codex/codex-session-backfill-types.ts index d3f25b37bbf..be199808c14 100644 --- a/src/main/codex/codex-session-backfill-types.ts +++ b/src/main/codex/codex-session-backfill-types.ts @@ -24,4 +24,8 @@ export type CodexSessionBackfillPaths = { export type CodexSessionBackfillOptions = CodexSessionBridgeIncrementalOptions & { /** Polled before each target mutation; true stops with progress preserved. */ shouldStop?: () => boolean + /** Limits a launch-triggered pass to the date directories that can contain its rollouts. */ + scanDates?: readonly CodexSessionBackfillDate[] } + +export type CodexSessionBackfillDate = readonly [year: string, month: string, day: string] diff --git a/src/main/codex/codex-session-backfill.test.ts b/src/main/codex/codex-session-backfill.test.ts index bc7eca545c9..bc6f620f220 100644 --- a/src/main/codex/codex-session-backfill.test.ts +++ b/src/main/codex/codex-session-backfill.test.ts @@ -460,6 +460,33 @@ describe('backfillManagedCodexSessionsIntoSystemHome', () => { expect(summary).toMatchObject({ scannedFiles: 0 }) expect(existsSync(getSystemSessionsRoot())).toBe(false) }) + + it('bounds a launch pass to its rollout date directories', async () => { + const oldRelativePath = join('2025', '12', '31', 'rollout-old.jsonl') + const launchRelativePath = join('2026', '08', '05', 'rollout-launch.jsonl') + writeManagedSession(oldRelativePath, 'old\n') + writeManagedSession(launchRelativePath, 'launch\n') + + const summary = await backfillManagedCodexSessionsIntoSystemHome( + resolveCodexSessionBackfillPaths(), + { scanDates: [['2026', '08', '05']] } + ) + + expect(summary).toMatchObject({ scannedFiles: 1, linkedFiles: 1, failedDirectories: 0 }) + expect(existsSync(join(getSystemSessionsRoot(), launchRelativePath))).toBe(true) + expect(existsSync(join(getSystemSessionsRoot(), oldRelativePath))).toBe(false) + }) + + it('treats a not-yet-created launch date as an empty bounded pass', async () => { + writeManagedSession(join('2025', '12', '31', 'rollout-old.jsonl'), 'old\n') + + const summary = await backfillManagedCodexSessionsIntoSystemHome( + resolveCodexSessionBackfillPaths(), + { scanDates: [['2026', '08', '05']] } + ) + + expect(summary).toMatchObject({ scannedFiles: 0, linkedFiles: 0, failedDirectories: 0 }) + }) }) describe('startCodexSessionBackfillInBackground', () => { diff --git a/src/main/codex/codex-session-backfill.ts b/src/main/codex/codex-session-backfill.ts index 377f2bc2050..0d487fa7678 100644 --- a/src/main/codex/codex-session-backfill.ts +++ b/src/main/codex/codex-session-backfill.ts @@ -15,7 +15,7 @@ import { copySessionFileWithoutOverwrite, isAtomicNoReplaceUnsupportedError } from './codex-session-backfill-copy' -import { listCodexSessionJsonlFilesIncrementally } from './codex-session-file-listing' +import { listCodexSessionBackfillFilesForDates } from './codex-session-backfill-date' import { captureCodexSessionBackfillMarkerGeneration, hasCompletedCodexSessionBackfillMarker, @@ -89,13 +89,7 @@ async function runCodexSessionBackfillOncePerHost( ): Promise { const paths = resolveCodexSessionBackfillPaths(systemCodexHomePathOverride) const markerGeneration = captureCodexSessionBackfillMarkerGeneration() - if ( - hasCompletedCodexSessionBackfillMarker( - paths.markerPath, - paths.systemSessionsRoot, - paths.managedSessionsRoot - ) - ) { + if (hasCompletedCodexSessionBackfillMarker(paths.markerPath, paths.systemSessionsRoot)) { return null } const summary = await backfillManagedCodexSessionsIntoSystemHome(paths, options) @@ -141,7 +135,7 @@ export async function backfillManagedCodexSessionsIntoSystemHome( const ensuredTargetDirectories = new Set() const managedSessionsRootExists = await checkManagedSessionsRoot(paths, summary, auditPass) if (managedSessionsRootExists) { - for await (const managedSessionFilePath of listCodexSessionJsonlFilesIncrementally( + for await (const managedSessionFilePath of listCodexSessionBackfillFilesForDates( paths.managedSessionsRoot, options, async (directoryPath, error) => { diff --git a/src/main/codex/codex-session-migration-scheduler.test.ts b/src/main/codex/codex-session-migration-scheduler.test.ts index d25b349095f..43b51ba7cf3 100644 --- a/src/main/codex/codex-session-migration-scheduler.test.ts +++ b/src/main/codex/codex-session-migration-scheduler.test.ts @@ -32,6 +32,7 @@ describe('createCodexSessionMigrationScheduler', () => { }) it('schedules a delayed rerun after a shared-home launch', async () => { + vi.setSystemTime(new Date(2026, 7, 5, 10, 0, 0)) const prepareScheduledRun = vi.fn() const startBackfill = vi.fn().mockResolvedValue(null) const startIndexHeal = vi.fn().mockResolvedValue(null) @@ -53,9 +54,59 @@ describe('createCodexSessionMigrationScheduler', () => { await vi.advanceTimersByTimeAsync(1) await vi.waitFor(() => expect(startIndexHeal).toHaveBeenCalledOnce()) expect(startBackfill).toHaveBeenCalledOnce() + expect(startBackfill).toHaveBeenCalledWith( + expect.objectContaining({ scanDates: [['2026', '08', '05']] }), + undefined + ) expect(prepareScheduledRun).toHaveBeenCalledOnce() }) + it('covers both launch and run dates when a delayed pass crosses midnight', async () => { + vi.setSystemTime(new Date(2026, 7, 5, 23, 59, 59, 500)) + const startBackfill = vi.fn().mockResolvedValue(null) + const scheduler = createCodexSessionMigrationScheduler({ + isEligible: () => true, + isQuitting: () => false, + resolveSystemCodexHomePathOverride: () => undefined, + startBackfill, + startIndexHeal: vi.fn().mockResolvedValue(null), + initialDelayMs: 1_000 + }) + + scheduler.scheduleRun() + await vi.advanceTimersByTimeAsync(1_000) + + expect(startBackfill).toHaveBeenCalledWith( + expect.objectContaining({ + scanDates: [ + ['2026', '08', '05'], + ['2026', '08', '06'] + ] + }), + undefined + ) + }) + + it('keeps launch passes full when no completed baseline can cover older failures', async () => { + const startBackfill = vi.fn().mockResolvedValue(null) + const scheduler = createCodexSessionMigrationScheduler({ + isEligible: () => true, + isQuitting: () => false, + resolveSystemCodexHomePathOverride: () => undefined, + startBackfill, + startIndexHeal: vi.fn().mockResolvedValue(null), + initialDelayMs: 1_000 + }) + + scheduler.scheduleRun(true) + await vi.advanceTimersByTimeAsync(1_000) + + expect(startBackfill).toHaveBeenCalledWith( + expect.objectContaining({ scanDates: undefined }), + undefined + ) + }) + it('delays the startup run from the latest shared-home launch', async () => { const prepareScheduledRun = vi.fn() const startBackfill = vi.fn().mockResolvedValue(null) diff --git a/src/main/codex/codex-session-migration-scheduler.ts b/src/main/codex/codex-session-migration-scheduler.ts index bcb94e7e4cd..d8c7b8d5cde 100644 --- a/src/main/codex/codex-session-migration-scheduler.ts +++ b/src/main/codex/codex-session-migration-scheduler.ts @@ -1,4 +1,8 @@ -import type { CodexSessionBackfillOptions } from './codex-session-backfill-types' +import { getCodexSessionBackfillDate } from './codex-session-backfill-date' +import type { + CodexSessionBackfillDate, + CodexSessionBackfillOptions +} from './codex-session-backfill-types' type MigrationRun = ( options: CodexSessionBackfillOptions, @@ -7,7 +11,7 @@ type MigrationRun = ( export type CodexSessionMigrationScheduler = { scheduleInitialRun(): void - scheduleRun(): void + scheduleRun(fullScanRequired?: boolean): void requestRun(): void } @@ -23,16 +27,29 @@ export function createCodexSessionMigrationScheduler(args: { let scheduledTimer: ReturnType | null = null let scheduledRunGeneration = 0 let pendingScheduledRunGeneration: number | null = null + const scheduledScanDates = new Map() + const pendingScanDates = new Map() + let scheduledFullScan = false + let pendingFullScan = false let migrationTask: Promise | null = null let activeRunStopObserved = false let rerunRequested = false - const requestRun = (rerunIfActive = false, requestedGeneration?: number): void => { + const requestRun = ( + rerunIfActive = false, + requestedGeneration?: number, + requestedScanDates: readonly CodexSessionBackfillDate[] = [], + requestedFullScan = false + ): void => { if (requestedGeneration !== undefined) { pendingScheduledRunGeneration = Math.max( requestedGeneration, pendingScheduledRunGeneration ?? requestedGeneration ) + for (const scanDate of requestedScanDates) { + pendingScanDates.set(scanDate.join('-'), scanDate) + } + pendingFullScan ||= requestedFullScan } if (args.isQuitting() || !args.isEligible()) { return @@ -47,6 +64,10 @@ export function createCodexSessionMigrationScheduler(args: { // Why: an older active pass can rewrite the marker after launch invalidates it. args.prepareScheduledRun?.() } + const scanDates = + !pendingFullScan && pendingScanDates.size > 0 ? [...pendingScanDates.values()] : undefined + pendingScanDates.clear() + pendingFullScan = false activeRunStopObserved = false rerunRequested = false const shouldStop = (): boolean => { @@ -57,7 +78,7 @@ export function createCodexSessionMigrationScheduler(args: { const systemCodexHomePathOverride = args.resolveSystemCodexHomePathOverride() let stoppedBackfill = false const task = args - .startBackfill({ shouldStop }, systemCodexHomePathOverride) + .startBackfill({ shouldStop, scanDates }, systemCodexHomePathOverride) .then((result) => { stoppedBackfill = isStoppedMigrationResult(result) if (stoppedBackfill || shouldStop()) { @@ -86,8 +107,16 @@ export function createCodexSessionMigrationScheduler(args: { const armScheduledRun = (generation?: number): void => { scheduledTimer = setTimeout(() => { scheduledTimer = null + if (generation !== undefined) { + const currentDate = getCodexSessionBackfillDate() + scheduledScanDates.set(currentDate.join('-'), currentDate) + } + const scanDates = [...scheduledScanDates.values()] + scheduledScanDates.clear() + const fullScanRequired = scheduledFullScan + scheduledFullScan = false // Why: a launch can invalidate the marker while a long index-heal pass is active. - requestRun(true, generation) + requestRun(true, generation, scanDates, fullScanRequired) }, args.initialDelayMs ?? 15_000) } @@ -97,12 +126,15 @@ export function createCodexSessionMigrationScheduler(args: { armScheduledRun() } }, - scheduleRun(): void { + scheduleRun(fullScanRequired = false): void { // Why: delay from the latest launch so Codex has time to create its rollout. if (scheduledTimer) { clearTimeout(scheduledTimer) } scheduledRunGeneration += 1 + scheduledFullScan ||= fullScanRequired + const launchDate = getCodexSessionBackfillDate() + scheduledScanDates.set(launchDate.join('-'), launchDate) armScheduledRun(scheduledRunGeneration) }, requestRun: () => requestRun()