perf(codex): bound launch session migration scans

This commit is contained in:
Brennan Benson
2026-08-05 13:17:54 -07:00
parent 7123d9c2ee
commit 3efd991ee3
9 changed files with 216 additions and 71 deletions
@@ -1010,6 +1010,18 @@ describe('CodexRuntimeHomeService', () => {
expect(service.prepareForCodexLaunch()).toBe(getRuntimeCodexHomePath())
expect(existsSync(markerPath)).toBe(false)
expect(scheduleSessionMigration).toHaveBeenCalledOnce()
expect(scheduleSessionMigration).toHaveBeenCalledWith(true)
writeFileSync(
markerPath,
`${JSON.stringify({
version: 3,
systemSessionsRoot: join(getSystemCodexHomePath(), 'sessions'),
summary: { scannedFiles: 1 }
})}\n`,
'utf-8'
)
service.prepareForCodexLaunch()
expect(scheduleSessionMigration).toHaveBeenLastCalledWith(false)
expect(service.prepareForRateLimitFetch()).toBe(getRuntimeCodexHomePath())
expect(service.getHostCodexHomePathsForSessionDiscovery()).toEqual([getRuntimeCodexHomePath()])
expect(existsSync(getRuntimeCodexHomePath())).toBe(true)
+24 -14
View File
@@ -41,7 +41,6 @@ import {
import {
getOrcaManagedCodexHomePath,
getOrcaUserDataPath,
getCodexSessionBackfillStateDirPath,
getSystemCodexHomePath,
resolveOrcaManagedCodexHomePath,
syncCodexGlobalInstructionsIntoManagedHome,
@@ -68,7 +67,11 @@ import {
} from './runtime-selection'
import { getDefaultWslDistro, getWslHome } from '../wsl'
import { hasCustomCodexHomeOverrideForLaunch } from '../codex/codex-real-home-path'
import { invalidateCodexSessionBackfillMarker } from '../codex/codex-session-backfill-marker'
import {
hasCompletedCodexSessionBackfillMarker,
invalidateCodexSessionBackfillMarker
} from '../codex/codex-session-backfill-marker'
import { resolveCodexSessionBackfillPaths } from '../codex/codex-session-backfill'
import { assertOwnedHostCodexManagedHomePath } from './host-codex-managed-home-ownership'
import {
codexAuthCouldBelongToManagedAccount,
@@ -181,7 +184,7 @@ export class CodexRuntimeHomeService {
private sharedAuthRefreshBlockedByManagedTransition = false
// Why: transient auth.json read/parse failures must not deselect an account.
private readonly credentialAbsenceGrace = new CodexCredentialAbsenceGrace()
private scheduleHostSystemDefaultSessionMigration: () => void = () => {}
private scheduleHostSystemDefaultSessionMigration: (fullScanRequired: boolean) => void = () => {}
constructor(private readonly store: Store) {
this.safeRecoverInterruptedRuntimeAuthOperation()
@@ -242,7 +245,7 @@ export class CodexRuntimeHomeService {
this.reconcileLegacySharedHomeForRetainedPanes()
return null
}
const shouldScheduleSessionMigration =
const sessionMigrationFullScanRequired =
this.invalidateBackfillAfterManagedSystemDefaultLaunch(launchEnv)
this.syncForCurrentSelection(target, launchEnv)
syncSystemCodexResourcesIntoManagedHome()
@@ -252,13 +255,15 @@ export class CodexRuntimeHomeService {
{},
resolveHostCodexSessionSourceHome(this.store.getSettings())
)
if (shouldScheduleSessionMigration) {
this.scheduleHostSystemDefaultSessionMigration()
if (sessionMigrationFullScanRequired !== null) {
this.scheduleHostSystemDefaultSessionMigration(sessionMigrationFullScanRequired)
}
return this.getRuntimeHomePath()
}
setHostSystemDefaultSessionMigrationScheduler(schedule: () => void): void {
setHostSystemDefaultSessionMigrationScheduler(
schedule: (fullScanRequired: boolean) => void
): void {
this.scheduleHostSystemDefaultSessionMigration = schedule
}
@@ -269,10 +274,16 @@ export class CodexRuntimeHomeService {
)
}
prepareHostSystemDefaultSessionMigrationPass(): void {
invalidateCodexSessionBackfillMarker(
join(getCodexSessionBackfillStateDirPath(), 'backfill-complete.json')
prepareHostSystemDefaultSessionMigrationPass(): boolean {
const paths = resolveCodexSessionBackfillPaths(
resolveHostCodexSessionSourceHome(this.store.getSettings())
)
const fullScanRequired = !hasCompletedCodexSessionBackfillMarker(
paths.markerPath,
paths.systemSessionsRoot
)
invalidateCodexSessionBackfillMarker(paths.markerPath)
return fullScanRequired
}
// Why: a managed HOST account runs against its own self-contained CODEX_HOME
@@ -428,16 +439,15 @@ export class CodexRuntimeHomeService {
private invalidateBackfillAfterManagedSystemDefaultLaunch(
launchEnv?: NodeJS.ProcessEnv
): boolean {
): boolean | null {
const settings = this.store.getSettings()
if (
normalizeCodexRuntimeSelection(settings).host !== null ||
hasCustomCodexHomeOverrideForLaunch(launchEnv)
) {
return false
return null
}
this.prepareHostSystemDefaultSessionMigrationPass()
return true
return this.prepareHostSystemDefaultSessionMigrationPass()
}
private startWslSessionBridgeForLaunch(
@@ -0,0 +1,51 @@
import { join } from 'node:path'
import { listCodexSessionJsonlFilesIncrementally } from './codex-session-file-listing'
import type {
CodexSessionBackfillDate,
CodexSessionBackfillOptions
} from './codex-session-backfill-types'
export function getCodexSessionBackfillDate(date = new Date()): CodexSessionBackfillDate {
return [
String(date.getFullYear()).padStart(4, '0'),
String(date.getMonth() + 1).padStart(2, '0'),
String(date.getDate()).padStart(2, '0')
]
}
export async function* listCodexSessionBackfillFilesForDates(
sessionsRoot: string,
options: CodexSessionBackfillOptions,
onDirectoryError: (directoryPath: string, error: unknown) => void | Promise<void>
): AsyncGenerator<string> {
const scanRoots = resolveCodexSessionBackfillDateRoots(sessionsRoot, options.scanDates)
for (const scanRoot of scanRoots) {
yield* listCodexSessionJsonlFilesIncrementally(
scanRoot,
options,
async (directoryPath, error) => {
if (directoryPath !== scanRoot || !isNotFoundError(error)) {
await onDirectoryError(directoryPath, error)
}
}
)
}
}
function resolveCodexSessionBackfillDateRoots(
sessionsRoot: string,
scanDates: readonly CodexSessionBackfillDate[] | undefined
): string[] {
if (!scanDates?.length) {
return [sessionsRoot]
}
return scanDates
.filter(
([year, month, day]) => /^\d{4}$/.test(year) && /^\d{2}$/.test(month) && /^\d{2}$/.test(day)
)
.map(([year, month, day]) => join(sessionsRoot, year, month, day))
}
function isNotFoundError(error: unknown): boolean {
return (error as NodeJS.ErrnoException | null)?.code === 'ENOENT'
}
@@ -1,5 +1,5 @@
import { mkdirSync, readFileSync, readdirSync, rmSync } from 'node:fs'
import { dirname, join } from 'node:path'
import { mkdirSync, readFileSync, rmSync } from 'node:fs'
import { dirname } from 'node:path'
import { writeFileAtomically } from '../codex-accounts/fs-utils'
import type { CodexSessionBackfillSummary } from './codex-session-backfill-types'
@@ -14,8 +14,7 @@ export function captureCodexSessionBackfillMarkerGeneration(): number {
export function hasCompletedCodexSessionBackfillMarker(
markerPath: string,
systemSessionsRoot: string,
managedSessionsRoot: string
systemSessionsRoot: string
): boolean {
try {
const parsed: unknown = JSON.parse(readFileSync(markerPath, 'utf-8'))
@@ -35,49 +34,14 @@ export function hasCompletedCodexSessionBackfillMarker(
if (!markerMatchesTarget) {
return false
}
// Why: an empty source can become populated after an early migration run.
return marker.summary?.scannedFiles !== 0 || !managedSessionsRootHasRollout(managedSessionsRoot)
// Why: an empty source can become populated after an early migration run;
// let the incremental async walk verify it without blocking the main thread.
return marker.summary?.scannedFiles !== 0
} catch {
return false
}
}
function managedSessionsRootHasRollout(managedSessionsRoot: string): boolean {
try {
for (const year of readdirSync(managedSessionsRoot, { withFileTypes: true })) {
if (!year.isDirectory() || !/^\d{4}$/.test(year.name)) {
continue
}
const yearPath = join(managedSessionsRoot, year.name)
for (const month of readdirSync(yearPath, { withFileTypes: true })) {
if (!month.isDirectory() || !/^\d{2}$/.test(month.name)) {
continue
}
const monthPath = join(yearPath, month.name)
for (const day of readdirSync(monthPath, { withFileTypes: true })) {
if (!day.isDirectory() || !/^\d{2}$/.test(day.name)) {
continue
}
const dayPath = join(monthPath, day.name)
if (
readdirSync(dayPath, { withFileTypes: true }).some(
(entry) => entry.isFile() && /^rollout-.+\.jsonl$/.test(entry.name)
)
) {
return true
}
}
}
}
return false
} catch (error) {
if ((error as NodeJS.ErrnoException | null)?.code === 'ENOENT') {
return false
}
throw error
}
}
export function writeCodexSessionBackfillMarker(
markerPath: string,
systemSessionsRoot: string,
@@ -24,4 +24,8 @@ export type CodexSessionBackfillPaths = {
export type CodexSessionBackfillOptions = CodexSessionBridgeIncrementalOptions & {
/** Polled before each target mutation; true stops with progress preserved. */
shouldStop?: () => boolean
/** Limits a launch-triggered pass to the date directories that can contain its rollouts. */
scanDates?: readonly CodexSessionBackfillDate[]
}
export type CodexSessionBackfillDate = readonly [year: string, month: string, day: string]
@@ -460,6 +460,33 @@ describe('backfillManagedCodexSessionsIntoSystemHome', () => {
expect(summary).toMatchObject({ scannedFiles: 0 })
expect(existsSync(getSystemSessionsRoot())).toBe(false)
})
it('bounds a launch pass to its rollout date directories', async () => {
const oldRelativePath = join('2025', '12', '31', 'rollout-old.jsonl')
const launchRelativePath = join('2026', '08', '05', 'rollout-launch.jsonl')
writeManagedSession(oldRelativePath, 'old\n')
writeManagedSession(launchRelativePath, 'launch\n')
const summary = await backfillManagedCodexSessionsIntoSystemHome(
resolveCodexSessionBackfillPaths(),
{ scanDates: [['2026', '08', '05']] }
)
expect(summary).toMatchObject({ scannedFiles: 1, linkedFiles: 1, failedDirectories: 0 })
expect(existsSync(join(getSystemSessionsRoot(), launchRelativePath))).toBe(true)
expect(existsSync(join(getSystemSessionsRoot(), oldRelativePath))).toBe(false)
})
it('treats a not-yet-created launch date as an empty bounded pass', async () => {
writeManagedSession(join('2025', '12', '31', 'rollout-old.jsonl'), 'old\n')
const summary = await backfillManagedCodexSessionsIntoSystemHome(
resolveCodexSessionBackfillPaths(),
{ scanDates: [['2026', '08', '05']] }
)
expect(summary).toMatchObject({ scannedFiles: 0, linkedFiles: 0, failedDirectories: 0 })
})
})
describe('startCodexSessionBackfillInBackground', () => {
+3 -9
View File
@@ -15,7 +15,7 @@ import {
copySessionFileWithoutOverwrite,
isAtomicNoReplaceUnsupportedError
} from './codex-session-backfill-copy'
import { listCodexSessionJsonlFilesIncrementally } from './codex-session-file-listing'
import { listCodexSessionBackfillFilesForDates } from './codex-session-backfill-date'
import {
captureCodexSessionBackfillMarkerGeneration,
hasCompletedCodexSessionBackfillMarker,
@@ -89,13 +89,7 @@ async function runCodexSessionBackfillOncePerHost(
): Promise<CodexSessionBackfillSummary | null> {
const paths = resolveCodexSessionBackfillPaths(systemCodexHomePathOverride)
const markerGeneration = captureCodexSessionBackfillMarkerGeneration()
if (
hasCompletedCodexSessionBackfillMarker(
paths.markerPath,
paths.systemSessionsRoot,
paths.managedSessionsRoot
)
) {
if (hasCompletedCodexSessionBackfillMarker(paths.markerPath, paths.systemSessionsRoot)) {
return null
}
const summary = await backfillManagedCodexSessionsIntoSystemHome(paths, options)
@@ -141,7 +135,7 @@ export async function backfillManagedCodexSessionsIntoSystemHome(
const ensuredTargetDirectories = new Set<string>()
const managedSessionsRootExists = await checkManagedSessionsRoot(paths, summary, auditPass)
if (managedSessionsRootExists) {
for await (const managedSessionFilePath of listCodexSessionJsonlFilesIncrementally(
for await (const managedSessionFilePath of listCodexSessionBackfillFilesForDates(
paths.managedSessionsRoot,
options,
async (directoryPath, error) => {
@@ -32,6 +32,7 @@ describe('createCodexSessionMigrationScheduler', () => {
})
it('schedules a delayed rerun after a shared-home launch', async () => {
vi.setSystemTime(new Date(2026, 7, 5, 10, 0, 0))
const prepareScheduledRun = vi.fn()
const startBackfill = vi.fn().mockResolvedValue(null)
const startIndexHeal = vi.fn().mockResolvedValue(null)
@@ -53,9 +54,59 @@ describe('createCodexSessionMigrationScheduler', () => {
await vi.advanceTimersByTimeAsync(1)
await vi.waitFor(() => expect(startIndexHeal).toHaveBeenCalledOnce())
expect(startBackfill).toHaveBeenCalledOnce()
expect(startBackfill).toHaveBeenCalledWith(
expect.objectContaining({ scanDates: [['2026', '08', '05']] }),
undefined
)
expect(prepareScheduledRun).toHaveBeenCalledOnce()
})
it('covers both launch and run dates when a delayed pass crosses midnight', async () => {
vi.setSystemTime(new Date(2026, 7, 5, 23, 59, 59, 500))
const startBackfill = vi.fn().mockResolvedValue(null)
const scheduler = createCodexSessionMigrationScheduler({
isEligible: () => true,
isQuitting: () => false,
resolveSystemCodexHomePathOverride: () => undefined,
startBackfill,
startIndexHeal: vi.fn().mockResolvedValue(null),
initialDelayMs: 1_000
})
scheduler.scheduleRun()
await vi.advanceTimersByTimeAsync(1_000)
expect(startBackfill).toHaveBeenCalledWith(
expect.objectContaining({
scanDates: [
['2026', '08', '05'],
['2026', '08', '06']
]
}),
undefined
)
})
it('keeps launch passes full when no completed baseline can cover older failures', async () => {
const startBackfill = vi.fn().mockResolvedValue(null)
const scheduler = createCodexSessionMigrationScheduler({
isEligible: () => true,
isQuitting: () => false,
resolveSystemCodexHomePathOverride: () => undefined,
startBackfill,
startIndexHeal: vi.fn().mockResolvedValue(null),
initialDelayMs: 1_000
})
scheduler.scheduleRun(true)
await vi.advanceTimersByTimeAsync(1_000)
expect(startBackfill).toHaveBeenCalledWith(
expect.objectContaining({ scanDates: undefined }),
undefined
)
})
it('delays the startup run from the latest shared-home launch', async () => {
const prepareScheduledRun = vi.fn()
const startBackfill = vi.fn().mockResolvedValue(null)
@@ -1,4 +1,8 @@
import type { CodexSessionBackfillOptions } from './codex-session-backfill-types'
import { getCodexSessionBackfillDate } from './codex-session-backfill-date'
import type {
CodexSessionBackfillDate,
CodexSessionBackfillOptions
} from './codex-session-backfill-types'
type MigrationRun = (
options: CodexSessionBackfillOptions,
@@ -7,7 +11,7 @@ type MigrationRun = (
export type CodexSessionMigrationScheduler = {
scheduleInitialRun(): void
scheduleRun(): void
scheduleRun(fullScanRequired?: boolean): void
requestRun(): void
}
@@ -23,16 +27,29 @@ export function createCodexSessionMigrationScheduler(args: {
let scheduledTimer: ReturnType<typeof setTimeout> | null = null
let scheduledRunGeneration = 0
let pendingScheduledRunGeneration: number | null = null
const scheduledScanDates = new Map<string, CodexSessionBackfillDate>()
const pendingScanDates = new Map<string, CodexSessionBackfillDate>()
let scheduledFullScan = false
let pendingFullScan = false
let migrationTask: Promise<void> | null = null
let activeRunStopObserved = false
let rerunRequested = false
const requestRun = (rerunIfActive = false, requestedGeneration?: number): void => {
const requestRun = (
rerunIfActive = false,
requestedGeneration?: number,
requestedScanDates: readonly CodexSessionBackfillDate[] = [],
requestedFullScan = false
): void => {
if (requestedGeneration !== undefined) {
pendingScheduledRunGeneration = Math.max(
requestedGeneration,
pendingScheduledRunGeneration ?? requestedGeneration
)
for (const scanDate of requestedScanDates) {
pendingScanDates.set(scanDate.join('-'), scanDate)
}
pendingFullScan ||= requestedFullScan
}
if (args.isQuitting() || !args.isEligible()) {
return
@@ -47,6 +64,10 @@ export function createCodexSessionMigrationScheduler(args: {
// Why: an older active pass can rewrite the marker after launch invalidates it.
args.prepareScheduledRun?.()
}
const scanDates =
!pendingFullScan && pendingScanDates.size > 0 ? [...pendingScanDates.values()] : undefined
pendingScanDates.clear()
pendingFullScan = false
activeRunStopObserved = false
rerunRequested = false
const shouldStop = (): boolean => {
@@ -57,7 +78,7 @@ export function createCodexSessionMigrationScheduler(args: {
const systemCodexHomePathOverride = args.resolveSystemCodexHomePathOverride()
let stoppedBackfill = false
const task = args
.startBackfill({ shouldStop }, systemCodexHomePathOverride)
.startBackfill({ shouldStop, scanDates }, systemCodexHomePathOverride)
.then((result) => {
stoppedBackfill = isStoppedMigrationResult(result)
if (stoppedBackfill || shouldStop()) {
@@ -86,8 +107,16 @@ export function createCodexSessionMigrationScheduler(args: {
const armScheduledRun = (generation?: number): void => {
scheduledTimer = setTimeout(() => {
scheduledTimer = null
if (generation !== undefined) {
const currentDate = getCodexSessionBackfillDate()
scheduledScanDates.set(currentDate.join('-'), currentDate)
}
const scanDates = [...scheduledScanDates.values()]
scheduledScanDates.clear()
const fullScanRequired = scheduledFullScan
scheduledFullScan = false
// Why: a launch can invalidate the marker while a long index-heal pass is active.
requestRun(true, generation)
requestRun(true, generation, scanDates, fullScanRequired)
}, args.initialDelayMs ?? 15_000)
}
@@ -97,12 +126,15 @@ export function createCodexSessionMigrationScheduler(args: {
armScheduledRun()
}
},
scheduleRun(): void {
scheduleRun(fullScanRequired = false): void {
// Why: delay from the latest launch so Codex has time to create its rollout.
if (scheduledTimer) {
clearTimeout(scheduledTimer)
}
scheduledRunGeneration += 1
scheduledFullScan ||= fullScanRequired
const launchDate = getCodexSessionBackfillDate()
scheduledScanDates.set(launchDate.join('-'), launchDate)
armScheduledRun(scheduledRunGeneration)
},
requestRun: () => requestRun()