diff --git a/src/main/codex/codex-session-backfill.test.ts b/src/main/codex/codex-session-backfill.test.ts index ef400a1360d..8d725a56531 100644 --- a/src/main/codex/codex-session-backfill.test.ts +++ b/src/main/codex/codex-session-backfill.test.ts @@ -4,12 +4,14 @@ import { lstatSync, mkdirSync, mkdtempSync, + readdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' import type * as NodeFs from 'node:fs' +import type * as NodeFsPromises from 'node:fs/promises' import type * as NodeOs from 'node:os' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' @@ -21,7 +23,8 @@ const { homedirMock } = vi.hoisted(() => ({ const { fsMockState } = vi.hoisted(() => ({ fsMockState: { failLink: false, - failCopy: false + failCopy: false, + failDirectoryPath: null as string | null } })) @@ -39,6 +42,9 @@ vi.mock('node:fs', async () => { }, copyFileSync: (...args: Parameters) => { if (fsMockState.failCopy) { + // Simulate a copy that fails after opening its destination, which is + // the dangerous case for resumability rather than a preflight error. + actual.writeFileSync(args[1], 'partial copy\n', 'utf-8') const error = new Error('EACCES: copy disabled for test') as NodeJS.ErrnoException error.code = 'EACCES' throw error @@ -48,6 +54,21 @@ vi.mock('node:fs', async () => { } }) +vi.mock('node:fs/promises', async () => { + const actual = await vi.importActual('node:fs/promises') + return { + ...actual, + opendir: (...args: Parameters) => { + if (args[0] === fsMockState.failDirectoryPath) { + const error = new Error('EACCES: directory unreadable') as NodeJS.ErrnoException + error.code = 'EACCES' + throw error + } + return actual.opendir(...args) + } + } +}) + vi.mock('node:os', async () => { const actual = await vi.importActual('node:os') return { @@ -99,6 +120,7 @@ function readAuditActions(): string[] { beforeEach(() => { fsMockState.failLink = false fsMockState.failCopy = false + fsMockState.failDirectoryPath = null fakeHomeDir = mkdtempSync(join(tmpdir(), 'orca-codex-backfill-home-')) userDataDir = mkdtempSync(join(tmpdir(), 'orca-codex-backfill-user-data-')) previousUserDataPath = process.env.ORCA_USER_DATA_PATH @@ -140,6 +162,32 @@ describe('backfillManagedCodexSessionsIntoSystemHome', () => { expect(readAuditActions()).toEqual(['hardlink', 'hardlink', 'run-summary']) }) + it('only backfills rollout files in the exact YYYY/MM/DD layout', async () => { + writeManagedSession(join('2026', '05', '26', 'rollout-valid ü.jsonl'), 'valid\n') + writeManagedSession(join('2026', '05', '26', 'session-index.jsonl'), 'not a rollout\n') + writeManagedSession(join('2026', '5', '26', 'rollout-wrong-month.jsonl'), 'wrong month\n') + writeManagedSession(join('scratch', 'rollout-too-shallow.jsonl'), 'too shallow\n') + writeManagedSession(join('2026', '05', '26', 'nested', 'rollout-too-deep.jsonl'), 'too deep\n') + + const summary = await backfillManagedCodexSessionsIntoSystemHome( + resolveCodexSessionBackfillPaths() + ) + + expect(summary).toMatchObject({ + scannedFiles: 5, + linkedFiles: 1, + skippedUnexpectedFiles: 4, + failedFiles: 0 + }) + expect( + existsSync(join(getSystemSessionsRoot(), '2026', '05', '26', 'rollout-valid ü.jsonl')) + ).toBe(true) + expect( + existsSync(join(getSystemSessionsRoot(), '2026', '05', '26', 'session-index.jsonl')) + ).toBe(false) + expect(existsSync(join(getSystemSessionsRoot(), 'scratch'))).toBe(false) + }) + it('never overwrites an existing target file, even with different contents', async () => { writeManagedSession(join('2026', '05', '26', 'rollout-a.jsonl'), 'managed contents\n') const collidingPath = join(getSystemSessionsRoot(), '2026', '05', '26', 'rollout-a.jsonl') @@ -192,9 +240,8 @@ describe('backfillManagedCodexSessionsIntoSystemHome', () => { // Why: the session walker skips symlink dirents, so bridge-created links // (which point back into the user's own home) never reach the copier. expect(summary).toMatchObject({ linkedFiles: 0, copiedFiles: 0, failedFiles: 0 }) - expect(existsSync(join(getSystemSessionsRoot(), '2026', '05', '26', 'rollout-a.jsonl'))).toBe( - false - ) + const targetPath = join(getSystemSessionsRoot(), '2026', '05', '26', 'rollout-a.jsonl') + expect(existsSync(targetPath)).toBe(false) }) it('is idempotent: a second run links nothing new and changes nothing', async () => { @@ -236,6 +283,9 @@ describe('backfillManagedCodexSessionsIntoSystemHome', () => { ) expect(summary).toMatchObject({ failedFiles: 1, linkedFiles: 0, copiedFiles: 0 }) + const targetPath = join(getSystemSessionsRoot(), '2026', '05', '26', 'rollout-a.jsonl') + expect(existsSync(targetPath)).toBe(false) + expect(readdirSync(dirname(targetPath))).toEqual([]) expect(readAuditActions()).toEqual(['failed', 'run-summary']) }) @@ -274,11 +324,36 @@ describe('startCodexSessionBackfillInBackground', () => { const first = await startCodexSessionBackfillInBackground() expect(first).toMatchObject({ failedFiles: 1 }) expect(existsSync(getMarkerPath())).toBe(false) + const targetPath = join(getSystemSessionsRoot(), '2026', '05', '26', 'rollout-a.jsonl') + expect(existsSync(targetPath)).toBe(false) fsMockState.failLink = false fsMockState.failCopy = false const second = await startCodexSessionBackfillInBackground() expect(second).toMatchObject({ linkedFiles: 1, failedFiles: 0 }) + expect(readFileSync(targetPath, 'utf-8')).toBe('{"id":"a"}\n') + expect(existsSync(getMarkerPath())).toBe(true) + }) + + it('leaves the marker unset when a directory cannot be scanned', async () => { + writeManagedSession(join('2026', '05', '26', 'rollout-readable.jsonl'), 'readable\n') + const unreadableDirectory = dirname( + writeManagedSession(join('2026', '06', '01', 'rollout-unreadable.jsonl'), 'unreadable\n') + ) + fsMockState.failDirectoryPath = unreadableDirectory + + const first = await startCodexSessionBackfillInBackground({ yieldMs: 0 }) + + expect(first).toMatchObject({ failedDirectories: 1 }) + expect(existsSync(getMarkerPath())).toBe(false) + expect(readAuditActions()).toContain('scan-failed') + + fsMockState.failDirectoryPath = null + const second = await startCodexSessionBackfillInBackground({ yieldMs: 0 }) + expect(second).toMatchObject({ failedDirectories: 0, failedFiles: 0 }) + expect( + existsSync(join(getSystemSessionsRoot(), '2026', '06', '01', 'rollout-unreadable.jsonl')) + ).toBe(true) expect(existsSync(getMarkerPath())).toBe(true) }) @@ -294,4 +369,18 @@ describe('startCodexSessionBackfillInBackground', () => { ) expect(existsSync(getSystemSessionsRoot())).toBe(false) }) + + it('re-runs when the configured real Codex home changes', async () => { + writeManagedSession(join('2026', '05', '26', 'rollout-a.jsonl'), '{"id":"a"}\n') + await startCodexSessionBackfillInBackground() + const customHome = join(fakeHomeDir, 'custom Codex ü') + + const moved = await startCodexSessionBackfillInBackground({}, customHome) + + expect(moved).toMatchObject({ linkedFiles: 1, failedFiles: 0 }) + expect(existsSync(join(customHome, 'sessions', '2026', '05', '26', 'rollout-a.jsonl'))).toBe( + true + ) + expect(await startCodexSessionBackfillInBackground({}, customHome)).toBeNull() + }) }) diff --git a/src/main/codex/codex-session-backfill.ts b/src/main/codex/codex-session-backfill.ts index 9018a61928d..0a2080c9dd8 100644 --- a/src/main/codex/codex-session-backfill.ts +++ b/src/main/codex/codex-session-backfill.ts @@ -6,9 +6,12 @@ import { linkSync, lstatSync, mkdirSync, - readFileSync + readFileSync, + rmSync, + writeFileSync } from 'node:fs' -import { dirname, join, relative } from 'node:path' +import { randomUUID } from 'node:crypto' +import { dirname, join, relative, sep } from 'node:path' import { writeFileAtomically } from '../codex-accounts/fs-utils' import { getCodexSessionBackfillStateDirPath, @@ -27,7 +30,9 @@ export type CodexSessionBackfillSummary = { linkedFiles: number copiedFiles: number skippedExistingFiles: number + skippedUnexpectedFiles: number skippedSymlinkFiles: number + failedDirectories: number failedFiles: number } @@ -93,14 +98,14 @@ async function runCodexSessionBackfillOncePerHost( systemCodexHomePathOverride?: string ): Promise { const paths = resolveCodexSessionBackfillPaths(systemCodexHomePathOverride) - if (hasCompletedBackfillMarker(paths.markerPath)) { + if (hasCompletedBackfillMarker(paths.markerPath, paths.systemSessionsRoot)) { return null } const summary = await backfillManagedCodexSessionsIntoSystemHome(paths, options) // Why: per-file failures (locked or unreadable files) leave the marker unset // so the next startup retries; skip-existing keeps those retries cheap. - if (summary.failedFiles === 0) { - writeBackfillMarker(paths.markerPath, summary) + if (summary.failedFiles === 0 && summary.failedDirectories === 0) { + writeBackfillMarker(paths.markerPath, paths.systemSessionsRoot, summary) } return summary } @@ -121,15 +126,31 @@ export async function backfillManagedCodexSessionsIntoSystemHome( linkedFiles: 0, copiedFiles: 0, skippedExistingFiles: 0, + skippedUnexpectedFiles: 0, skippedSymlinkFiles: 0, + failedDirectories: 0, failedFiles: 0 } if (existsSync(paths.managedSessionsRoot)) { for await (const managedSessionFilePath of listCodexSessionJsonlFilesIncrementally( paths.managedSessionsRoot, - options + options, + (directoryPath, error) => { + // Why: a partial walk must remain retryable; otherwise an unreadable + // date directory would be silently omitted behind a completion marker. + summary.failedDirectories += 1 + appendAuditRecord(paths.auditLogPath, { + action: 'scan-failed', + source: directoryPath, + error: describeError(error) + }) + } )) { summary.scannedFiles += 1 + if (!isCodexRolloutPath(paths.managedSessionsRoot, managedSessionFilePath)) { + summary.skippedUnexpectedFiles += 1 + continue + } backfillOneManagedSessionFile(paths, managedSessionFilePath, summary) } } @@ -137,6 +158,20 @@ export async function backfillManagedCodexSessionsIntoSystemHome( return summary } +function isCodexRolloutPath(sessionsRoot: string, filePath: string): boolean { + const pathParts = relative(sessionsRoot, filePath).split(sep) + if (pathParts.length !== 4) { + return false + } + const [year, month, day, fileName] = pathParts + return ( + /^\d{4}$/.test(year) && + /^\d{2}$/.test(month) && + /^\d{2}$/.test(day) && + /^rollout-.+\.jsonl$/.test(fileName) + ) +} + function backfillOneManagedSessionFile( paths: CodexSessionBackfillPaths, managedSessionFilePath: string, @@ -170,9 +205,9 @@ function backfillOneManagedSessionFile( return } try { - // Why: hardlinks fail across volumes; COPYFILE_EXCL keeps the - // never-overwrite contract even if the target appeared mid-run. - copyFileSync(managedSessionFilePath, systemSessionFilePath, constants.COPYFILE_EXCL) + // Why: cross-volume copies are staged so failures cannot strand a + // truncated rollout, then installed without overwriting collisions. + copySessionFileWithoutOverwrite(managedSessionFilePath, systemSessionFilePath) summary.copiedFiles += 1 appendAuditRecord(paths.auditLogPath, { action: 'copy', @@ -196,6 +231,36 @@ function backfillOneManagedSessionFile( } } +function copySessionFileWithoutOverwrite(sourcePath: string, targetPath: string): void { + const temporaryPath = join(dirname(targetPath), `.orca-backfill-${randomUUID()}.tmp`) + // Why: stage cross-volume copies away from the rollout filename so a failed + // copy cannot strand a truncated session that a later retry would skip. + writeFileSync(temporaryPath, '', { encoding: 'utf-8', flag: 'wx', mode: 0o600 }) + try { + copyFileSync(sourcePath, temporaryPath) + try { + // Why: this same-volume hardlink atomically installs the staged copy + // without risking a collision overwrite after an EXDEV fallback. + linkSync(temporaryPath, targetPath) + } catch (installLinkError) { + if (isExistsError(installLinkError)) { + throw installLinkError + } + // Some target filesystems do not support hardlinks at all. COPYFILE_EXCL + // preserves the collision contract while retaining the staged snapshot. + copyFileSync(temporaryPath, targetPath, constants.COPYFILE_EXCL) + } + } finally { + try { + rmSync(temporaryPath, { force: true }) + } catch (error) { + // Why: cleanup trouble must not misreport a successfully installed + // rollout as a copy failure; the .tmp file is ignored by Codex. + console.warn('[codex-session-backfill] Failed to remove staged copy:', temporaryPath, error) + } + } +} + function isSymbolicLink(filePath: string): boolean { try { return lstatSync(filePath).isSymbolicLink() @@ -239,24 +304,39 @@ function appendAuditRecord(auditLogPath: string, record: Record } } -function hasCompletedBackfillMarker(markerPath: string): boolean { +function hasCompletedBackfillMarker(markerPath: string, systemSessionsRoot: string): boolean { try { const parsed: unknown = JSON.parse(readFileSync(markerPath, 'utf-8')) if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { return false } - return (parsed as { version?: unknown }).version === CODEX_SESSION_BACKFILL_MARKER_VERSION + const marker = parsed as { version?: unknown; systemSessionsRoot?: unknown } + // Why: changing the configured real Codex home must backfill the new + // target instead of honoring a marker written for a different history. + return ( + marker.version === CODEX_SESSION_BACKFILL_MARKER_VERSION && + marker.systemSessionsRoot === systemSessionsRoot + ) } catch { return false } } -function writeBackfillMarker(markerPath: string, summary: CodexSessionBackfillSummary): void { +function writeBackfillMarker( + markerPath: string, + systemSessionsRoot: string, + summary: CodexSessionBackfillSummary +): void { mkdirSync(dirname(markerPath), { recursive: true }) writeFileAtomically( markerPath, `${JSON.stringify( - { version: CODEX_SESSION_BACKFILL_MARKER_VERSION, completedAt: Date.now(), summary }, + { + version: CODEX_SESSION_BACKFILL_MARKER_VERSION, + systemSessionsRoot, + completedAt: Date.now(), + summary + }, null, 2 )}\n` diff --git a/src/main/codex/codex-session-file-listing.ts b/src/main/codex/codex-session-file-listing.ts index c6eeb9b1f77..92676b38052 100644 --- a/src/main/codex/codex-session-file-listing.ts +++ b/src/main/codex/codex-session-file-listing.ts @@ -56,7 +56,8 @@ function appendSessionFilePaths(target: string[], source: readonly string[]): vo */ export async function* listCodexSessionJsonlFilesIncrementally( rootPath: string, - options: CodexSessionBridgeIncrementalOptions + options: CodexSessionBridgeIncrementalOptions, + onDirectoryError?: (directoryPath: string, error: unknown) => void ): AsyncGenerator { const batchSize = Math.max(1, options.batchSize ?? INCREMENTAL_BRIDGE_BATCH_SIZE) const yieldMs = Math.max(0, options.yieldMs ?? INCREMENTAL_BRIDGE_YIELD_MS) @@ -84,6 +85,7 @@ export async function* listCodexSessionJsonlFilesIncrementally( } } } catch (error) { + onDirectoryError?.(currentDirectory, error) console.warn('[codex-session-bridge] Failed to list system Codex sessions:', error) } }