Files
archived-orca/src/main/codex-usage/codex-rollout-file-parse.ts
T

204 lines
7.1 KiB
TypeScript

import { basename } from 'node:path'
import {
readJsonlFileSnapshot,
type JsonlFileSnapshot
} from '../usage/source-input/jsonl/jsonl-file-snapshot'
import { readJsonlLinesFromOffset } from '../usage/source-input/jsonl/jsonl-line-offsets'
import {
openJsonlFileReader,
validateJsonlFileReader,
jsonlPhysicalFileId
} from '../usage/source-input/jsonl/jsonl-file-checkpoint'
import { attributeCodexUsageEvent } from './codex-usage-event-attribution'
import type { UsageWorktreeResolver } from '../usage/worktree-attribution/usage-worktree-resolver'
import { parseCodexUsageRecord, type CodexUsageParseContext } from './codex-usage-record-parser'
import { codexUsageAggregation } from './codex-usage-aggregation'
import {
buildCodexRolloutResumeState,
resolveCodexRolloutResume
} from './codex-rollout-resume-state'
import type {
CodexUsageDailyAggregate,
CodexUsageParseResumeState,
CodexUsagePersistedFile,
CodexUsageProcessedFile,
CodexUsageSession
} from './types'
const { finalizeSessions, mergeSessions, mergeDailyAggregates, sortDailyAggregates } =
codexUsageAggregation
export type CodexRolloutParseOptions = {
/** Suffix-only parse for a diverged legacy copied-session bridge. */
legacySourceSkipBytes?: number
canClaimEventKey?: (eventKey: string) => boolean
commitEventKey?: (eventKey: string) => void
/** Resume point verified by the caller, with the cached projection to extend. */
resume?: { state: CodexUsageParseResumeState; previous: CodexUsagePersistedFile }
}
export async function getProcessedFileInfo(filePath: string): Promise<CodexUsageProcessedFile> {
const fileStat = await readJsonlFileSnapshot(filePath)
return processedFileInfo(filePath, fileStat)
}
function processedFileInfo(filePath: string, fileStat: JsonlFileSnapshot): CodexUsageProcessedFile {
return {
path: filePath,
mtimeMs: fileStat.mtimeMs,
size: fileStat.size,
ctimeMs: fileStat.ctimeMs,
physicalFileId: jsonlPhysicalFileId(fileStat)
}
}
function mergeRolloutProjections(
previous: CodexUsagePersistedFile,
appended: { sessions: CodexUsageSession[]; dailyAggregates: CodexUsageDailyAggregate[] }
): { sessions: CodexUsageSession[]; dailyAggregates: CodexUsageDailyAggregate[] } {
const sessionsById = new Map<string, CodexUsageSession>()
mergeSessions(sessionsById, previous.sessions)
mergeSessions(sessionsById, appended.sessions)
const dailyByKey = new Map<string, CodexUsageDailyAggregate>()
mergeDailyAggregates(dailyByKey, previous.dailyAggregates)
mergeDailyAggregates(dailyByKey, appended.dailyAggregates)
return {
sessions: finalizeSessions(sessionsById),
dailyAggregates: sortDailyAggregates(dailyByKey)
}
}
function createParseContext(
filePath: string,
options: CodexRolloutParseOptions
): CodexUsageParseContext {
const resume = options.resume?.state
if (resume) {
return {
sessionId: resume.sessionId,
sessionCwd: resume.sessionCwd,
currentCwd: resume.currentCwd,
currentModel: resume.currentModel,
currentTurnId: resume.currentTurnId,
previousTotals: resume.previousTotals,
totalOnlyBaselinePending: false
}
}
return {
sessionId: basename(filePath, '.jsonl'),
sessionCwd: null,
currentCwd: null,
currentModel: null,
previousTotals: null,
// A copied suffix lacks prefix context; its first total-only snapshot establishes the baseline.
totalOnlyBaselinePending: (options.legacySourceSkipBytes ?? 0) > 0
}
}
export async function parseCodexUsageFile(
filePath: string,
resolveWorktree: UsageWorktreeResolver,
options: CodexRolloutParseOptions = {}
): Promise<CodexUsagePersistedFile> {
let parseOptions = options
for (;;) {
const reader = await openJsonlFileReader(filePath)
try {
// Verification, parsing and checkpointing must all read the same opened file.
if (
parseOptions.resume &&
(await resolveCodexRolloutResume(filePath, parseOptions.resume.previous, reader)) === null
) {
parseOptions = { ...parseOptions, resume: undefined }
continue
}
const legacySourceSkipBytes = parseOptions.legacySourceSkipBytes ?? 0
const startOffset = parseOptions.resume?.state.parsedBytes ?? legacySourceSkipBytes
const context = createParseContext(filePath, parseOptions)
const accumulator = codexUsageAggregation.createAccumulator()
const ownedEventKeys = new Set<string>()
let hasDeferredClaims = false
let parsedBytes = startOffset
let resumeContext = context
let partialTailProducedEvent = false
for await (const { line, endOffset, terminated } of readJsonlLinesFromOffset(
filePath,
startOffset,
reader
)) {
if (!terminated) {
// The next scan rereads this tail, including its context changes.
resumeContext = { ...context }
}
const parsed = parseCodexUsageRecord(line, context)
if (terminated) {
parsedBytes = endOffset
} else if (parsed) {
partialTailProducedEvent = true
}
if (!parsed) {
continue
}
// Deferred records still advance cumulative totals without retaining their copies.
if (parseOptions.canClaimEventKey && !parseOptions.canClaimEventKey(parsed.eventKey)) {
hasDeferredClaims = true
continue
}
ownedEventKeys.add(parsed.eventKey)
const attributed = await attributeCodexUsageEvent(parsed, resolveWorktree)
if (attributed) {
accumulator.add(attributed)
}
}
// A counted unterminated row would be counted again from the committed offset.
const resumeStateSuppressed = partialTailProducedEvent || legacySourceSkipBytes > 0
const parseResumeState = resumeStateSuppressed
? null
: await buildCodexRolloutResumeState(
filePath,
parsedBytes,
resumeContext,
parseOptions.resume?.state.headDigest ?? null,
reader
)
if (
!(await validateJsonlFileReader(reader, parseOptions.resume?.state)) ||
(parseOptions.resume && !resumeStateSuppressed && parseResumeState === null)
) {
parseOptions = { ...parseOptions, resume: undefined }
continue
}
await reader.handle.close()
// Discarded reads cannot claim keys that would hide records in another file.
for (const eventKey of ownedEventKeys) {
parseOptions.commitEventKey?.(eventKey)
}
const processedFile = processedFileInfo(filePath, reader.stats)
const appended = accumulator.finalize()
const previous = parseOptions.resume?.previous
const committedKeys = previous ? new Set(previous.ownedEventKeys) : ownedEventKeys
if (previous) {
for (const key of ownedEventKeys) {
committedKeys.add(key)
}
}
return {
...processedFile,
...(previous ? mergeRolloutProjections(previous, appended) : appended),
ownedEventKeys: [...committedKeys],
hasDeferredClaims: previous?.hasDeferredClaims || hasDeferredClaims,
parseResumeState
}
} finally {
await reader.handle.close()
}
}
}