Reduce memory used to count Codex usage and avoid partial totals (#26966)

This commit is contained in:
Neil
2026-10-09 20:23:25 -07:00
committed by GitHub
parent f1a09aa9c2
commit 440cde0f74
27 changed files with 5384 additions and 323 deletions
@@ -16,6 +16,10 @@ on:
# without these a rename lands green and only breaks the next nightly.
- 'src/main/sqlite/sync-database.test.ts'
- 'src/main/runtime/orchestration/db/row-column-lists.test.ts'
- 'src/main/codex-usage/codex-rollout-file-lifetime.test.ts'
- 'src/main/codex-usage/scanner-incremental-append.test.ts'
- 'src/main/usage/jsonl-file-reader.test.ts'
- 'src/main/usage/jsonl-file-snapshot.test.ts'
- 'src/relay/fs-path-metadata-symlink-concurrency.test.ts'
- 'src/renderer/src/components/editor/rich-markdown-list-tokenizers.test.ts'
- 'src/renderer/src/components/editor/rich-markdown-lowlight-cache.test.ts'
+3 -2
View File
@@ -16,8 +16,9 @@ manual review. A warning identifies repeated setup, not proof of visible lag.
`pnpm test:perf:contracts` runs the explicit selection in
`vitest.performance.config.ts`: SQLite statement reuse and schema parity, relay
filesystem concurrency, tokenizer rejection, highlighting cache, queued
cancellation, terminal backing-memory retention and detector fixtures. Missing
filesystem concurrency, incremental Codex scans and reader lifetimes, pinned JSONL
readers and snapshots, tokenizer rejection, highlighting cache, queued cancellation,
terminal backing-memory retention and detector fixtures. Missing
listed files fail configuration loading. Tests run serially, without retries,
and inherit the full suite's setup and forced-GC support. This makes existing
regression coverage easy to run and attribute; it does not create new workload
@@ -0,0 +1,300 @@
import assert from 'node:assert/strict'
import { createHash } from 'node:crypto'
import { appendFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { availableParallelism, cpus, release, tmpdir, totalmem } from 'node:os'
import { dirname, join, relative, resolve, sep } from 'node:path'
import { performance } from 'node:perf_hooks'
import { pathToFileURL } from 'node:url'
const root = resolve(import.meta.dirname, '../..')
const digest = (value) => createHash('sha256').update(value).digest('hex')
const args = process.argv.slice(2)
function argument(name, fallback) {
const index = args.indexOf(name)
return index === -1 ? fallback : args[index + 1]
}
function positiveInteger(value, label) {
const parsed = Number(value)
if (!Number.isInteger(parsed) || parsed <= 0) {
throw new Error(`${label} must be a positive integer`)
}
return parsed
}
function memorySnapshot(phase, started) {
globalThis.gc()
globalThis.gc()
return { phase, elapsedMs: performance.now() - started, ...process.memoryUsage() }
}
async function runWorker() {
assert.equal(typeof globalThis.gc, 'function', 'Run workers with --expose-gc')
const parser = await import(pathToFileURL(argument('--bundle')).href)
const count = positiveInteger(argument('--events'), 'events')
const stride = positiveInteger(argument('--stride'), 'stride')
const intervalMs = positiveInteger(argument('--sample-ms'), 'sample-ms')
const arm = argument('--arm')
let claimQueries = 0
let committedKeys = 0
const canClaim = () => claimQueries++ % stride === 0
const options =
arm === 'baseline'
? { claimEventKey: canClaim }
: { canClaimEventKey: canClaim, commitEventKey: () => committedKeys++ }
const started = performance.now()
const baseline = memorySnapshot('before', started)
const samples = []
const timer = setInterval(() => samples.push(memorySnapshot('during', started)), intervalMs)
let result
try {
result = await parser.parseCodexUsageFile(argument('--fixture'), () => null, options)
} finally {
clearInterval(timer)
}
globalThis.__orcaMemoryResult = result
const final = memorySnapshot('after', started)
const duringSampleCount = samples.length
samples.push(final)
const expected = Math.ceil(count / stride)
const correctness = {
claimQueries,
committedKeys: arm === 'baseline' ? null : committedKeys,
ownedKeys: result.ownedEventKeys.length,
countedEvents: result.sessions.reduce((sum, session) => sum + session.eventCount, 0),
inputTokens: result.sessions.reduce((sum, session) => sum + session.totalInputTokens, 0),
outputTokens: result.sessions.reduce((sum, session) => sum + session.totalOutputTokens, 0),
totalTokens: result.sessions.reduce((sum, session) => sum + session.totalTokens, 0),
hasDeferredClaims: result.hasDeferredClaims,
projectionSha256: digest(
JSON.stringify({
sessions: result.sessions,
dailyAggregates: result.dailyAggregates,
ownedEventKeys: result.ownedEventKeys,
hasDeferredClaims: result.hasDeferredClaims
})
)
}
assert.equal(correctness.claimQueries, count)
assert.equal(correctness.ownedKeys, expected)
assert.equal(correctness.countedEvents, expected)
assert.equal(correctness.inputTokens, expected * 100)
assert.equal(correctness.outputTokens, expected * 10)
assert.equal(correctness.totalTokens, expected * 110)
assert.equal(correctness.hasDeferredClaims, stride > 1 && count > 1)
if (arm === 'current') {
assert.equal(committedKeys, expected)
}
process.stdout.write(
`${JSON.stringify({
arm,
stride,
duringSampleCount,
finalOnlyMeasurement: duringSampleCount === 0,
baseline,
samples,
sampledRetainedPeakDeltaBytes:
Math.max(...samples.map((sample) => sample.heapUsed)) - baseline.heapUsed,
finalRetainedDeltaBytes: final.heapUsed - baseline.heapUsed,
correctness
})}\n`
)
delete globalThis.__orcaMemoryResult
}
function writeFixture(filePath, eventCount) {
const cwd = join('benchmark', 'workspace')
writeFileSync(
filePath,
`${JSON.stringify({ type: 'session_meta', payload: { id: 'session', cwd } })}\n`
)
let batch = ''
for (let index = 0; index < eventCount; index++) {
batch += `${JSON.stringify({
timestamp: new Date(1791547200000 + index * 1000).toISOString(),
type: 'event_msg',
payload: {
type: 'token_count',
info: {
model: 'gpt-5-codex',
last_token_usage: { input_tokens: 100, output_tokens: 10, total_tokens: 110 }
}
}
})}\n`
if ((index + 1) % 5000 === 0) {
appendFileSync(filePath, batch)
batch = ''
}
}
if (batch) {
appendFileSync(filePath, batch)
}
}
async function runComparison() {
const { build, version: esbuildVersion } = await import('esbuild')
const { runProcessSync, describeProcessFailure } = await import('./script-child-process.mjs')
const { buildCounterbalancedSchedule } = await import('./counterbalanced-benchmark-schedule.mjs')
const eventCount = positiveInteger(argument('--events', '100000'), 'events')
const pairCount = positiveInteger(argument('--pairs', '2'), 'pairs')
const intervalMs = positiveInteger(argument('--sample-ms', '35'), 'sample-ms')
const schedule = buildCounterbalancedSchedule(pairCount, 'baseline', 'current')
const execute = (program, commandArgs) => {
const result = runProcessSync({
program,
args: commandArgs,
cwd: root,
env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' },
timeoutMs: 120000,
maxOutputBytes: 16 * 1024 * 1024
})
if (result.code !== 0 || result.timedOut || result.outputTruncated) {
throw new Error(describeProcessFailure(result))
}
return result.stdout
}
const baselineCommit = execute('git', [
'rev-parse',
'--verify',
`${argument('--baseline-ref', 'HEAD')}^{commit}`
]).trim()
const temporary = mkdtempSync(join(tmpdir(), 'orca-codex-parser-memory-'))
try {
const sources = {}
const bundles = {}
for (const arm of ['baseline', 'current']) {
const sourceHashes = new Map()
const bundlePath = join(temporary, `${arm}.mjs`)
await build({
entryPoints: [join(root, 'src/main/codex-usage/codex-rollout-file-parse.ts')],
bundle: true,
platform: 'node',
target: 'node20',
format: 'esm',
outfile: bundlePath,
logLevel: 'silent',
plugins: [
{
name: 'record-parser-source-graph',
setup(builder) {
builder.onLoad({ filter: /\.[cm]?[jt]sx?$/ }, ({ path }) => {
const sourcePath = relative(root, path).split(sep).join('/')
if (!sourcePath.startsWith('src/')) {
return undefined
}
const contents =
arm === 'baseline'
? execute('git', ['show', `${baselineCommit}:${sourcePath}`])
: readFileSync(path, 'utf8')
sourceHashes.set(sourcePath, digest(contents))
return {
contents,
resolveDir: dirname(path),
loader: sourcePath.endsWith('.tsx') ? 'tsx' : 'ts'
}
})
}
}
]
})
bundles[arm] = bundlePath
sources[arm] = {
origin: arm === 'baseline' ? baselineCommit : 'worktree at bundle build',
files: Object.fromEntries(
[...sourceHashes].sort(([left], [right]) => left.localeCompare(right))
),
bundleSha256: digest(readFileSync(bundlePath))
}
}
const fixturePath = join(temporary, 'rollout.jsonl')
writeFixture(fixturePath, eventCount)
const fixtureBytes = readFileSync(fixturePath)
const measurements = []
for (let round = 0; round < schedule.length; round++) {
for (const stride of [1, 10]) {
const paired = []
for (const arm of schedule[round]) {
const measurement = JSON.parse(
execute(process.execPath, [
'--expose-gc',
import.meta.filename,
'--worker',
'--bundle',
bundles[arm],
'--fixture',
fixturePath,
'--events',
String(eventCount),
'--stride',
String(stride),
'--sample-ms',
String(intervalMs),
'--arm',
arm
])
)
paired.push(measurement)
measurements.push({ round: round + 1, order: schedule[round], ...measurement })
}
assert.equal(paired[0].correctness.projectionSha256, paired[1].correctness.projectionSha256)
}
}
const artifact = {
schemaVersion: 1,
recordedAt: new Date().toISOString(),
command: [
'node',
relative(root, import.meta.filename)
.split(sep)
.join('/'),
...args
].join(' '),
environment: {
node: process.version,
v8: process.versions.v8,
platform: process.platform,
architecture: process.arch,
osRelease: release(),
cpuModel: cpus()[0]?.model ?? null,
availableParallelism: availableParallelism(),
totalMemoryBytes: totalmem(),
esbuild: esbuildVersion,
timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone
},
instrumentation: {
metric: 'sampled retained heapUsed after two forced full GCs; not absolute peak memory',
sampleIntervalMs: intervalMs,
isolation: 'one fresh Node process for each arm, scenario and round',
finalResult: 'held by a global reference through the final sample',
observerMemory: 'retained snapshot records contribute small duration-dependent overhead',
baseline: 'all bundled project source modules read from the pinned baseline commit',
current: 'all bundled project source modules read and hashed once before any worker runs',
timing: 'elapsed values include forced GC and are not a speed benchmark'
},
instrumentationSha256: Object.fromEntries(
[
import.meta.filename,
join(import.meta.dirname, 'script-child-process.mjs'),
join(import.meta.dirname, 'counterbalanced-benchmark-schedule.mjs')
].map((path) => [relative(root, path).split(sep).join('/'), digest(readFileSync(path))])
),
baselineCommit,
sources,
fixture: { events: eventCount, bytes: fixtureBytes.length, sha256: digest(fixtureBytes) },
schedule,
measurements
}
const output = `${JSON.stringify(artifact, null, 2)}\n`
const outputPath = argument('--output')
if (outputPath) {
writeFileSync(resolve(root, outputPath), output)
} else {
process.stdout.write(output)
}
} finally {
rmSync(temporary, { recursive: true, force: true })
}
}
await (args.includes('--worker') ? runWorker() : runComparison())
+4
View File
@@ -7,6 +7,10 @@ import { nodeRuntimePool } from './scripts/vitest-node-runtime-pool'
const contracts = [
'src/main/sqlite/sync-database.test.ts',
'src/main/runtime/orchestration/db/row-column-lists.test.ts',
'src/main/codex-usage/codex-rollout-file-lifetime.test.ts',
'src/main/codex-usage/scanner-incremental-append.test.ts',
'src/main/usage/jsonl-file-reader.test.ts',
'src/main/usage/jsonl-file-snapshot.test.ts',
'src/relay/fs-path-metadata-symlink-concurrency.test.ts',
'src/renderer/src/components/editor/rich-markdown-list-tokenizers.test.ts',
'src/renderer/src/components/editor/rich-markdown-lowlight-cache.test.ts',
@@ -0,0 +1,63 @@
{
"status": "historical exploratory parser-retention comparison, not the final pinned-source benchmark",
"baselineCommit": "51e7181850b022bae2d41091fe1229accef4bd50",
"sourceProvenance": {
"baseline": "Only codex-rollout-file-parse.ts was loaded with git show from this commit; other bundled project dependencies came from the worktree.",
"current": "Actual worktree parser and project dependencies at the time of the exploratory run.",
"sourceHashes": null,
"limitation": "Source hashes and individual heap snapshots were not saved during this run. Later hashes cannot establish the earlier source. Use the reproducible full-graph comparison for final evidence."
},
"environment": {
"node": "v24.20.0",
"v8": "13.6.233.17-node.53",
"platform": "darwin",
"architecture": "arm64",
"timeZone": "America/Los_Angeles"
},
"instrumentation": {
"fixture": "100,000 event_msg/token_count rows with distinct one-second timestamps, model gpt-5-codex, last usage input 100/output 10/total 110, and one session_meta row; written in batches of 5,000.",
"clockOrigin": 1791547200000,
"metric": "heapUsed after two forced full GCs before parse, every 35 ms during parse, and after parse",
"isolation": "Each measurement had its own async function scope; result held on globalThis through the final sample, removed at function end, with setImmediate before the next baseline sample.",
"ownership": "baseline claimEventKey and current canClaimEventKey accept queryIndex++ % stride === 0; current commitEventKey is a no-op",
"interpretation": "Sampled retained memory, not absolute process peak. Timings are not a speed benchmark."
},
"measurements": [
{
"arm": "baseline",
"stride": 1,
"samples": 9,
"claimedKeys": 100000,
"countedEvents": 100000,
"sampledRetainedPeakDeltaMiB": 45.45518493652344,
"finalRetainedDeltaMiB": 6.44708251953125
},
{
"arm": "current",
"stride": 1,
"samples": 10,
"claimedKeys": 100000,
"countedEvents": 100000,
"sampledRetainedPeakDeltaMiB": 7.5893402099609375,
"finalRetainedDeltaMiB": 6.328620910644531
},
{
"arm": "baseline",
"stride": 10,
"samples": 4,
"claimedKeys": 10000,
"countedEvents": 10000,
"sampledRetainedPeakDeltaMiB": 3.7055206298828125,
"finalRetainedDeltaMiB": 0.6778564453125
},
{
"arm": "current",
"stride": 10,
"samples": 5,
"claimedKeys": 10000,
"countedEvents": 10000,
"sampledRetainedPeakDeltaMiB": 0.91009521484375,
"finalRetainedDeltaMiB": 0.66552734375
}
]
}
@@ -0,0 +1,978 @@
{
"schemaVersion": 1,
"recordedAt": "2026-10-10T00:27:00.815Z",
"command": "node config/scripts/codex-usage-parser-memory-benchmark.mjs --baseline-ref 51e7181850b022bae2d41091fe1229accef4bd50 --events 100000 --pairs 2 --sample-ms 35 --output docs/bug-reproductions/usage-scan-append/codex-parser-memory-final.json",
"environment": {
"node": "v24.20.0",
"v8": "13.6.233.17-node.53",
"platform": "darwin",
"architecture": "arm64",
"osRelease": "25.3.0",
"cpuModel": "Apple M4 Max",
"availableParallelism": 16,
"totalMemoryBytes": 137438953472,
"esbuild": "0.28.2",
"timeZone": "America/Los_Angeles"
},
"instrumentation": {
"metric": "sampled retained heapUsed after two forced full GCs; not absolute peak memory",
"sampleIntervalMs": 35,
"isolation": "one fresh Node process for each arm, scenario and round",
"finalResult": "held by a global reference through the final sample",
"observerMemory": "retained snapshot records contribute small duration-dependent overhead",
"baseline": "all bundled project source modules read from the pinned baseline commit",
"current": "all bundled project source modules read and hashed once before any worker runs",
"timing": "elapsed values include forced GC and are not a speed benchmark"
},
"instrumentationSha256": {
"config/scripts/codex-usage-parser-memory-benchmark.mjs": "434e9818f8d8b132603912b104560a000e9e2635a2564cf65ae7e87cff483a1a",
"config/scripts/script-child-process.mjs": "a7441428bf7b5c9a897b6a3c057c50f46acca24c4f218a4b849de5aec53c4173",
"config/scripts/counterbalanced-benchmark-schedule.mjs": "bc35b40e954fa7ddec0225ed375bd3b113c2e05b8554bc50734d14f89987294d"
},
"baselineCommit": "51e7181850b022bae2d41091fe1229accef4bd50",
"sources": {
"baseline": {
"origin": "51e7181850b022bae2d41091fe1229accef4bd50",
"files": {
"src/main/codex-usage/codex-model-pricing.ts": "05416b11916778dc2c69edfccfb277b0a2fe6ef7e88e0d8c9bdf030f2dc7ea1d",
"src/main/codex-usage/codex-rollout-file-parse.ts": "9ef58c665491ea709166d9580913bb1791a1748435cb2af1a5bb41dfe47df792",
"src/main/codex-usage/codex-rollout-resume-state.ts": "de254c8fbca043c311350adf5466d5e510c4894b9c0d9e1a086a204a8c949f6f",
"src/main/codex-usage/codex-usage-aggregation.ts": "82774021e25bfcf413dbc2faad8d0cb92310a3ddeac87ba640abe4254f19d082",
"src/main/codex-usage/codex-usage-event-attribution.ts": "1b3c37111bb5ff9a28fb9a97d3e40d0c76026b6b5be2d55a0c512b68d5944b02",
"src/main/codex-usage/codex-usage-record-parser.ts": "b73ecd62b6ed20ce9cd7e21ca8e78e8a3f7d9a635311f2d8d56f9d64042a1661",
"src/main/codex-usage/codex-usage-token-delta.ts": "fb3e64b17eef3d8e23fafbcb2ce4a1ebfe44cab1091faa979d8c0269f294d300",
"src/main/usage/jsonl-line-offsets.ts": "ea7f58e248e8c332bfe6d18294261b1941816eb38f079e9a2c0c9b15c06820c5",
"src/main/usage/usage-calendar-range.ts": "fe4c4eb09aaefcae6d2698e113e134663826c3ed9ca3ef960f3047a95686b1dd",
"src/main/usage/usage-event-aggregation.ts": "68caf65824d32c55b706fd78ee90d583439f4b86445862cd75d10f3f4ef122d1",
"src/main/usage/usage-event-attribution.ts": "e7a909437d836c34939e2709ede99794f81cc8746c6c55b1bbdd36bf352e7664",
"src/main/usage/usage-path-comparison.ts": "07be51caa36e43962fac4cb66c1e0bfd4700f58aeb76eae32cc524fff88625d8",
"src/main/usage/usage-record-coercion.ts": "1b7f9fd8a0c01fbe25f82171d9ceefcbc717e87df3e321fc9514c59c9cfbd7da",
"src/main/usage/usage-rollup-merge.ts": "67cc062058439d659f77a8b47ce1b0ad3e2c323a83d2332aaa75e399de98b433",
"src/main/usage/usage-rollup-records.ts": "eff570d174439c5786175081ffe3ac2aedd960ebfc7e317f9f3cfc4091f27dab",
"src/main/usage/usage-session-breakdown-index.ts": "51725b3480ed078506beeec3105bd95402836e10478bd681cdff95fed080f81d"
},
"bundleSha256": "aaf0b5266d4511752229a998db9789c583ca3cd2c0b7bcc4afe03c465ef6e564"
},
"current": {
"origin": "worktree at bundle build",
"files": {
"src/main/codex-usage/codex-model-pricing.ts": "05416b11916778dc2c69edfccfb277b0a2fe6ef7e88e0d8c9bdf030f2dc7ea1d",
"src/main/codex-usage/codex-rollout-file-parse.ts": "f39a10c1dce47a71d46c0948d05df30aa6dcc044c81b777b6a013be6f23e9c7a",
"src/main/codex-usage/codex-rollout-resume-state.ts": "aa028d2155b7744c0d961c5649537faa20c39bfe404dea8d9c4c1464d11a0e39",
"src/main/codex-usage/codex-usage-aggregation.ts": "82774021e25bfcf413dbc2faad8d0cb92310a3ddeac87ba640abe4254f19d082",
"src/main/codex-usage/codex-usage-event-attribution.ts": "1b3c37111bb5ff9a28fb9a97d3e40d0c76026b6b5be2d55a0c512b68d5944b02",
"src/main/codex-usage/codex-usage-record-parser.ts": "b73ecd62b6ed20ce9cd7e21ca8e78e8a3f7d9a635311f2d8d56f9d64042a1661",
"src/main/codex-usage/codex-usage-token-delta.ts": "fb3e64b17eef3d8e23fafbcb2ce4a1ebfe44cab1091faa979d8c0269f294d300",
"src/main/usage/jsonl-file-checkpoint.ts": "b7f307564989dabaf1dfda095a26c7bd9f0683e0efcabe79a010c7b58fa1a2d4",
"src/main/usage/jsonl-file-snapshot.ts": "b1b8869db92c08c762160acba7570aabff25879649cb6ffa3826912ee35deffe",
"src/main/usage/jsonl-line-offsets.ts": "f3574adfad4549c6330e2ac52760c91be5a8cbb9bf8e1beb440e3226087074db",
"src/main/usage/usage-calendar-range.ts": "fe4c4eb09aaefcae6d2698e113e134663826c3ed9ca3ef960f3047a95686b1dd",
"src/main/usage/usage-event-aggregation.ts": "31adda2e8e0a32756416fb49b2c3e02c707703bfe76434aabc95c61a7efd180b",
"src/main/usage/usage-event-attribution.ts": "e7a909437d836c34939e2709ede99794f81cc8746c6c55b1bbdd36bf352e7664",
"src/main/usage/usage-path-comparison.ts": "07be51caa36e43962fac4cb66c1e0bfd4700f58aeb76eae32cc524fff88625d8",
"src/main/usage/usage-record-coercion.ts": "1b7f9fd8a0c01fbe25f82171d9ceefcbc717e87df3e321fc9514c59c9cfbd7da",
"src/main/usage/usage-rollup-merge.ts": "67cc062058439d659f77a8b47ce1b0ad3e2c323a83d2332aaa75e399de98b433",
"src/main/usage/usage-rollup-records.ts": "eff570d174439c5786175081ffe3ac2aedd960ebfc7e317f9f3cfc4091f27dab",
"src/main/usage/usage-session-breakdown-index.ts": "51725b3480ed078506beeec3105bd95402836e10478bd681cdff95fed080f81d"
},
"bundleSha256": "9f8b81ddae88e3d4fdd48175a53f68011d9606bfe169d6f5660238b3db74abaf"
}
},
"fixture": {
"events": 100000,
"bytes": 20200079,
"sha256": "fa36fd10e36653020c0997890c0c9d9e75f40ee4040adf3a8d79735c4fbe20d5"
},
"schedule": [
["baseline", "current"],
["current", "baseline"]
],
"measurements": [
{
"round": 1,
"order": ["baseline", "current"],
"arm": "baseline",
"stride": 1,
"duringSampleCount": 9,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 2.961416,
"rss": 54771712,
"heapTotal": 7667712,
"heapUsed": 4923944,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 43.085458,
"rss": 79740928,
"heapTotal": 28196864,
"heapUsed": 10333960,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 80.171208,
"rss": 100352000,
"heapTotal": 50823168,
"heapUsed": 16436440,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 116.55525,
"rss": 109264896,
"heapTotal": 56590336,
"heapUsed": 22257384,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 153.371208,
"rss": 129040384,
"heapTotal": 95125504,
"heapUsed": 27179344,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 191.0125,
"rss": 146718720,
"heapTotal": 100581376,
"heapUsed": 32420792,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 223.885208,
"rss": 154877952,
"heapTotal": 105824256,
"heapUsed": 37958976,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 261.673083,
"rss": 166674432,
"heapTotal": 178733056,
"heapUsed": 43593472,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 302.397916,
"rss": 184074240,
"heapTotal": 183451648,
"heapUsed": 48113072,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 337.490916,
"rss": 187842560,
"heapTotal": 186597376,
"heapUsed": 51628032,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "after",
"elapsedMs": 404.128916,
"rss": 197738496,
"heapTotal": 147144704,
"heapUsed": 11730912,
"external": 2455679,
"arrayBuffers": 143819
}
],
"sampledRetainedPeakDeltaBytes": 46704088,
"finalRetainedDeltaBytes": 6806968,
"correctness": {
"claimQueries": 100000,
"committedKeys": null,
"ownedKeys": 100000,
"countedEvents": 100000,
"inputTokens": 10000000,
"outputTokens": 1000000,
"totalTokens": 11000000,
"hasDeferredClaims": false,
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
}
},
{
"round": 1,
"order": ["baseline", "current"],
"arm": "current",
"stride": 1,
"duringSampleCount": 9,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 2.704749999999997,
"rss": 54919168,
"heapTotal": 7929856,
"heapUsed": 4938600,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 40.513166999999996,
"rss": 68812800,
"heapTotal": 11419648,
"heapUsed": 6306616,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 75.017375,
"rss": 75153408,
"heapTotal": 16465920,
"heapUsed": 7232808,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 109.88325,
"rss": 75939840,
"heapTotal": 16990208,
"heapUsed": 7780688,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 145.465458,
"rss": 77348864,
"heapTotal": 18432000,
"heapUsed": 9102536,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 180.921875,
"rss": 78675968,
"heapTotal": 27344896,
"heapUsed": 9704632,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 215.932708,
"rss": 85442560,
"heapTotal": 27869184,
"heapUsed": 10366048,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 250.763458,
"rss": 88440832,
"heapTotal": 29966336,
"heapUsed": 12341424,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 286.477917,
"rss": 89866240,
"heapTotal": 30752768,
"heapUsed": 12972312,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 321.256625,
"rss": 91324416,
"heapTotal": 31277056,
"heapUsed": 13643896,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "after",
"elapsedMs": 336.48325,
"rss": 92192768,
"heapTotal": 29704192,
"heapUsed": 11803368,
"external": 2455471,
"arrayBuffers": 143611
}
],
"sampledRetainedPeakDeltaBytes": 8705296,
"finalRetainedDeltaBytes": 6864768,
"correctness": {
"claimQueries": 100000,
"committedKeys": 100000,
"ownedKeys": 100000,
"countedEvents": 100000,
"inputTokens": 10000000,
"outputTokens": 1000000,
"totalTokens": 11000000,
"hasDeferredClaims": false,
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
}
},
{
"round": 1,
"order": ["baseline", "current"],
"arm": "baseline",
"stride": 10,
"duringSampleCount": 4,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 2.7109999999999985,
"rss": 54607872,
"heapTotal": 7929856,
"heapUsed": 4924000,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 41.222042,
"rss": 66764800,
"heapTotal": 11337728,
"heapUsed": 6303312,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 75.894208,
"rss": 71761920,
"heapTotal": 16498688,
"heapUsed": 7411304,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 111.375667,
"rss": 75366400,
"heapTotal": 25935872,
"heapUsed": 8445088,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 146.555292,
"rss": 83542016,
"heapTotal": 27410432,
"heapUsed": 9730008,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "after",
"elapsedMs": 172.027042,
"rss": 86786048,
"heapTotal": 24182784,
"heapUsed": 5946064,
"external": 2455679,
"arrayBuffers": 143819
}
],
"sampledRetainedPeakDeltaBytes": 4806008,
"finalRetainedDeltaBytes": 1022064,
"correctness": {
"claimQueries": 100000,
"committedKeys": null,
"ownedKeys": 10000,
"countedEvents": 10000,
"inputTokens": 1000000,
"outputTokens": 100000,
"totalTokens": 1100000,
"hasDeferredClaims": true,
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
}
},
{
"round": 1,
"order": ["baseline", "current"],
"arm": "current",
"stride": 10,
"duringSampleCount": 4,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 3.0069579999999974,
"rss": 54706176,
"heapTotal": 7667712,
"heapUsed": 4938616,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 40.785916,
"rss": 65945600,
"heapTotal": 10551296,
"heapUsed": 5556160,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 75.849791,
"rss": 66551808,
"heapTotal": 10731520,
"heapUsed": 5764056,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 110.944416,
"rss": 66797568,
"heapTotal": 10993664,
"heapUsed": 5903120,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 145.28670799999998,
"rss": 68075520,
"heapTotal": 11419648,
"heapUsed": 6229832,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "after",
"elapsedMs": 172.400125,
"rss": 70189056,
"heapTotal": 11075584,
"heapUsed": 6002200,
"external": 2455471,
"arrayBuffers": 143611
}
],
"sampledRetainedPeakDeltaBytes": 1291216,
"finalRetainedDeltaBytes": 1063584,
"correctness": {
"claimQueries": 100000,
"committedKeys": 10000,
"ownedKeys": 10000,
"countedEvents": 10000,
"inputTokens": 1000000,
"outputTokens": 100000,
"totalTokens": 1100000,
"hasDeferredClaims": true,
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
}
},
{
"round": 2,
"order": ["current", "baseline"],
"arm": "current",
"stride": 1,
"duringSampleCount": 10,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 2.3475420000000007,
"rss": 54722560,
"heapTotal": 7929856,
"heapUsed": 4938624,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 40.420334,
"rss": 68485120,
"heapTotal": 11419648,
"heapUsed": 6288496,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 76.145292,
"rss": 74825728,
"heapTotal": 16465920,
"heapUsed": 7269912,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 110.49249999999999,
"rss": 75481088,
"heapTotal": 16990208,
"heapUsed": 7924696,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 145.027584,
"rss": 76890112,
"heapTotal": 18432000,
"heapUsed": 9178464,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 180.48204199999998,
"rss": 78184448,
"heapTotal": 27344896,
"heapUsed": 9725784,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 217.70899999999997,
"rss": 86163456,
"heapTotal": 27869184,
"heapUsed": 10424152,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 250.85633399999998,
"rss": 89620480,
"heapTotal": 29966336,
"heapUsed": 12375976,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 286.69275,
"rss": 91013120,
"heapTotal": 30490624,
"heapUsed": 12988984,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 354.68100000000004,
"rss": 91783168,
"heapTotal": 31014912,
"heapUsed": 13393048,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 366.923584,
"rss": 91799552,
"heapTotal": 31014912,
"heapUsed": 13405528,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "after",
"elapsedMs": 400.572084,
"rss": 93257728,
"heapTotal": 29442048,
"heapUsed": 11784552,
"external": 2455471,
"arrayBuffers": 143611
}
],
"sampledRetainedPeakDeltaBytes": 8466904,
"finalRetainedDeltaBytes": 6845928,
"correctness": {
"claimQueries": 100000,
"committedKeys": 100000,
"ownedKeys": 100000,
"countedEvents": 100000,
"inputTokens": 10000000,
"outputTokens": 1000000,
"totalTokens": 11000000,
"hasDeferredClaims": false,
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
}
},
{
"round": 2,
"order": ["current", "baseline"],
"arm": "baseline",
"stride": 1,
"duringSampleCount": 17,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 3.085000000000008,
"rss": 54870016,
"heapTotal": 7929856,
"heapUsed": 4923944,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 66.69608300000002,
"rss": 70172672,
"heapTotal": 16498688,
"heapUsed": 7281848,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 85.53883300000001,
"rss": 73515008,
"heapTotal": 17809408,
"heapUsed": 8570768,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 152.638541,
"rss": 85196800,
"heapTotal": 30031872,
"heapUsed": 12156408,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 159.173375,
"rss": 85639168,
"heapTotal": 30228480,
"heapUsed": 12357872,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 225.200083,
"rss": 104693760,
"heapTotal": 52133888,
"heapUsed": 17698272,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 297.829375,
"rss": 104857600,
"heapTotal": 52396032,
"heapUsed": 17837984,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 310.679916,
"rss": 104988672,
"heapTotal": 52396032,
"heapUsed": 17986776,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 343.32525,
"rss": 106856448,
"heapTotal": 54493184,
"heapUsed": 19932448,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 408.458833,
"rss": 111263744,
"heapTotal": 57901056,
"heapUsed": 23407968,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 420.895625,
"rss": 111493120,
"heapTotal": 58163200,
"heapUsed": 23541184,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 474.20687499999997,
"rss": 120029184,
"heapTotal": 94601216,
"heapUsed": 26919320,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 494.13695800000005,
"rss": 121274368,
"heapTotal": 96174080,
"heapUsed": 28168392,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 523.933083,
"rss": 135741440,
"heapTotal": 99794944,
"heapUsed": 32038824,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 567.489416,
"rss": 156827648,
"heapTotal": 106348544,
"heapUsed": 38424024,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 598.908,
"rss": 160595968,
"heapTotal": 110051328,
"heapUsed": 42221320,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 634.888333,
"rss": 170278912,
"heapTotal": 181878784,
"heapUsed": 46720192,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 670.023,
"rss": 185548800,
"heapTotal": 185810944,
"heapUsed": 50929816,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "after",
"elapsedMs": 735.424125,
"rss": 202817536,
"heapTotal": 147144704,
"heapUsed": 11727688,
"external": 2455679,
"arrayBuffers": 143819
}
],
"sampledRetainedPeakDeltaBytes": 46005872,
"finalRetainedDeltaBytes": 6803744,
"correctness": {
"claimQueries": 100000,
"committedKeys": null,
"ownedKeys": 100000,
"countedEvents": 100000,
"inputTokens": 10000000,
"outputTokens": 1000000,
"totalTokens": 11000000,
"hasDeferredClaims": false,
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
}
},
{
"round": 2,
"order": ["current", "baseline"],
"arm": "current",
"stride": 10,
"duringSampleCount": 5,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 2.7139999999999986,
"rss": 54902784,
"heapTotal": 7929856,
"heapUsed": 4938792,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 39.88775,
"rss": 66011136,
"heapTotal": 10813440,
"heapUsed": 5552696,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 75.154917,
"rss": 66748416,
"heapTotal": 10993664,
"heapUsed": 5767816,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 110.337708,
"rss": 67108864,
"heapTotal": 11255808,
"heapUsed": 5912424,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 147.088458,
"rss": 68534272,
"heapTotal": 11419648,
"heapUsed": 6234112,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 179.94975,
"rss": 68550656,
"heapTotal": 11681792,
"heapUsed": 6230488,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "after",
"elapsedMs": 205.10170799999997,
"rss": 70860800,
"heapTotal": 10813440,
"heapUsed": 5992168,
"external": 2455471,
"arrayBuffers": 143611
}
],
"sampledRetainedPeakDeltaBytes": 1295320,
"finalRetainedDeltaBytes": 1053376,
"correctness": {
"claimQueries": 100000,
"committedKeys": 10000,
"ownedKeys": 10000,
"countedEvents": 10000,
"inputTokens": 1000000,
"outputTokens": 100000,
"totalTokens": 1100000,
"hasDeferredClaims": true,
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
}
},
{
"round": 2,
"order": ["current", "baseline"],
"arm": "baseline",
"stride": 10,
"duringSampleCount": 4,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 3.1553749999999994,
"rss": 54804480,
"heapTotal": 7929856,
"heapUsed": 4923976,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 41.432542,
"rss": 66666496,
"heapTotal": 11337728,
"heapUsed": 6206048,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 75.186042,
"rss": 71827456,
"heapTotal": 16760832,
"heapUsed": 7309048,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 111.10141699999998,
"rss": 74629120,
"heapTotal": 17809408,
"heapUsed": 8403400,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 146.308042,
"rss": 83525632,
"heapTotal": 27148288,
"heapUsed": 9674064,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "after",
"elapsedMs": 172.545917,
"rss": 87998464,
"heapTotal": 24182784,
"heapUsed": 5945896,
"external": 2455679,
"arrayBuffers": 143819
}
],
"sampledRetainedPeakDeltaBytes": 4750088,
"finalRetainedDeltaBytes": 1021920,
"correctness": {
"claimQueries": 100000,
"committedKeys": null,
"ownedKeys": 10000,
"countedEvents": 10000,
"inputTokens": 1000000,
"outputTokens": 100000,
"totalTokens": 1100000,
"hasDeferredClaims": true,
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
}
}
]
}
@@ -0,0 +1,879 @@
{
"schemaVersion": 1,
"recordedAt": "2026-10-09T18:39:49.816Z",
"command": "node config/scripts/codex-usage-parser-memory-benchmark.mjs --baseline-ref 51e7181850b022bae2d41091fe1229accef4bd50 --events 100000 --pairs 2 --sample-ms 35 --output docs/bug-reproductions/usage-scan-append/codex-parser-memory.json",
"environment": {
"node": "v24.20.0",
"v8": "13.6.233.17-node.53",
"platform": "darwin",
"architecture": "arm64",
"osRelease": "25.3.0",
"cpuModel": "Apple M4 Max",
"availableParallelism": 16,
"totalMemoryBytes": 137438953472,
"esbuild": "0.28.2",
"timeZone": "America/Los_Angeles"
},
"instrumentation": {
"metric": "sampled retained heapUsed after two forced full GCs; not absolute peak memory",
"sampleIntervalMs": 35,
"isolation": "one fresh Node process for each arm, scenario and round",
"finalResult": "held by a global reference through the final sample",
"observerMemory": "retained snapshot records contribute small duration-dependent overhead",
"baseline": "all bundled project source modules read from the pinned baseline commit",
"current": "all bundled project source modules read and hashed once before any worker runs",
"timing": "elapsed values include forced GC and are not a speed benchmark"
},
"instrumentationSha256": {
"config/scripts/codex-usage-parser-memory-benchmark.mjs": "434e9818f8d8b132603912b104560a000e9e2635a2564cf65ae7e87cff483a1a",
"config/scripts/script-child-process.mjs": "a7441428bf7b5c9a897b6a3c057c50f46acca24c4f218a4b849de5aec53c4173",
"config/scripts/counterbalanced-benchmark-schedule.mjs": "bc35b40e954fa7ddec0225ed375bd3b113c2e05b8554bc50734d14f89987294d"
},
"baselineCommit": "51e7181850b022bae2d41091fe1229accef4bd50",
"sources": {
"baseline": {
"origin": "51e7181850b022bae2d41091fe1229accef4bd50",
"files": {
"src/main/codex-usage/codex-model-pricing.ts": "05416b11916778dc2c69edfccfb277b0a2fe6ef7e88e0d8c9bdf030f2dc7ea1d",
"src/main/codex-usage/codex-rollout-file-parse.ts": "9ef58c665491ea709166d9580913bb1791a1748435cb2af1a5bb41dfe47df792",
"src/main/codex-usage/codex-rollout-resume-state.ts": "de254c8fbca043c311350adf5466d5e510c4894b9c0d9e1a086a204a8c949f6f",
"src/main/codex-usage/codex-usage-aggregation.ts": "82774021e25bfcf413dbc2faad8d0cb92310a3ddeac87ba640abe4254f19d082",
"src/main/codex-usage/codex-usage-event-attribution.ts": "1b3c37111bb5ff9a28fb9a97d3e40d0c76026b6b5be2d55a0c512b68d5944b02",
"src/main/codex-usage/codex-usage-record-parser.ts": "b73ecd62b6ed20ce9cd7e21ca8e78e8a3f7d9a635311f2d8d56f9d64042a1661",
"src/main/codex-usage/codex-usage-token-delta.ts": "fb3e64b17eef3d8e23fafbcb2ce4a1ebfe44cab1091faa979d8c0269f294d300",
"src/main/usage/jsonl-line-offsets.ts": "ea7f58e248e8c332bfe6d18294261b1941816eb38f079e9a2c0c9b15c06820c5",
"src/main/usage/usage-calendar-range.ts": "fe4c4eb09aaefcae6d2698e113e134663826c3ed9ca3ef960f3047a95686b1dd",
"src/main/usage/usage-event-aggregation.ts": "68caf65824d32c55b706fd78ee90d583439f4b86445862cd75d10f3f4ef122d1",
"src/main/usage/usage-event-attribution.ts": "e7a909437d836c34939e2709ede99794f81cc8746c6c55b1bbdd36bf352e7664",
"src/main/usage/usage-path-comparison.ts": "07be51caa36e43962fac4cb66c1e0bfd4700f58aeb76eae32cc524fff88625d8",
"src/main/usage/usage-record-coercion.ts": "1b7f9fd8a0c01fbe25f82171d9ceefcbc717e87df3e321fc9514c59c9cfbd7da",
"src/main/usage/usage-rollup-merge.ts": "67cc062058439d659f77a8b47ce1b0ad3e2c323a83d2332aaa75e399de98b433",
"src/main/usage/usage-rollup-records.ts": "eff570d174439c5786175081ffe3ac2aedd960ebfc7e317f9f3cfc4091f27dab",
"src/main/usage/usage-session-breakdown-index.ts": "51725b3480ed078506beeec3105bd95402836e10478bd681cdff95fed080f81d"
},
"bundleSha256": "aaf0b5266d4511752229a998db9789c583ca3cd2c0b7bcc4afe03c465ef6e564"
},
"current": {
"origin": "worktree at bundle build",
"files": {
"src/main/codex-usage/codex-model-pricing.ts": "05416b11916778dc2c69edfccfb277b0a2fe6ef7e88e0d8c9bdf030f2dc7ea1d",
"src/main/codex-usage/codex-rollout-file-parse.ts": "6e4937105fc9cf8457b79ac16bcfc71f706f22a87e036166bd461aeea8c07fdc",
"src/main/codex-usage/codex-rollout-resume-state.ts": "aa028d2155b7744c0d961c5649537faa20c39bfe404dea8d9c4c1464d11a0e39",
"src/main/codex-usage/codex-usage-aggregation.ts": "82774021e25bfcf413dbc2faad8d0cb92310a3ddeac87ba640abe4254f19d082",
"src/main/codex-usage/codex-usage-event-attribution.ts": "1b3c37111bb5ff9a28fb9a97d3e40d0c76026b6b5be2d55a0c512b68d5944b02",
"src/main/codex-usage/codex-usage-record-parser.ts": "b73ecd62b6ed20ce9cd7e21ca8e78e8a3f7d9a635311f2d8d56f9d64042a1661",
"src/main/codex-usage/codex-usage-token-delta.ts": "fb3e64b17eef3d8e23fafbcb2ce4a1ebfe44cab1091faa979d8c0269f294d300",
"src/main/usage/jsonl-file-checkpoint.ts": "8eb579f356e1bb23b36319f00b3d4603fff80a963222dcb8a22331319cab3a03",
"src/main/usage/jsonl-file-snapshot.ts": "1bd98ac194f73097cd6c204238fe1db01e1e0d60a604e93c1d7841725971fe0d",
"src/main/usage/jsonl-line-offsets.ts": "f119465fad3771acd72b128853e84eed9165f0f74b75743ad662aa5bab991273",
"src/main/usage/usage-calendar-range.ts": "fe4c4eb09aaefcae6d2698e113e134663826c3ed9ca3ef960f3047a95686b1dd",
"src/main/usage/usage-event-aggregation.ts": "31adda2e8e0a32756416fb49b2c3e02c707703bfe76434aabc95c61a7efd180b",
"src/main/usage/usage-event-attribution.ts": "e7a909437d836c34939e2709ede99794f81cc8746c6c55b1bbdd36bf352e7664",
"src/main/usage/usage-path-comparison.ts": "07be51caa36e43962fac4cb66c1e0bfd4700f58aeb76eae32cc524fff88625d8",
"src/main/usage/usage-record-coercion.ts": "1b7f9fd8a0c01fbe25f82171d9ceefcbc717e87df3e321fc9514c59c9cfbd7da",
"src/main/usage/usage-rollup-merge.ts": "67cc062058439d659f77a8b47ce1b0ad3e2c323a83d2332aaa75e399de98b433",
"src/main/usage/usage-rollup-records.ts": "eff570d174439c5786175081ffe3ac2aedd960ebfc7e317f9f3cfc4091f27dab",
"src/main/usage/usage-session-breakdown-index.ts": "51725b3480ed078506beeec3105bd95402836e10478bd681cdff95fed080f81d"
},
"bundleSha256": "9e56032701d379c5e1b5482c49c5734153fbc2735010d0cbc3bc82e44662fe25"
}
},
"fixture": {
"events": 100000,
"bytes": 20200079,
"sha256": "fa36fd10e36653020c0997890c0c9d9e75f40ee4040adf3a8d79735c4fbe20d5"
},
"schedule": [
["baseline", "current"],
["current", "baseline"]
],
"measurements": [
{
"round": 1,
"order": ["baseline", "current"],
"arm": "baseline",
"stride": 1,
"duringSampleCount": 12,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 2.196832999999998,
"rss": 54837248,
"heapTotal": 7929856,
"heapUsed": 4923952,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 41.750583000000006,
"rss": 85114880,
"heapTotal": 29507584,
"heapUsed": 11458104,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 77.58875,
"rss": 100073472,
"heapTotal": 52396032,
"heapUsed": 17544544,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 115.21895800000001,
"rss": 114868224,
"heapTotal": 92504064,
"heapUsed": 24514776,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 154.876542,
"rss": 137347072,
"heapTotal": 98222080,
"heapUsed": 30184872,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 186.526458,
"rss": 152600576,
"heapTotal": 102416384,
"heapUsed": 34527664,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 223.591625,
"rss": 159465472,
"heapTotal": 109232128,
"heapUsed": 41191320,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 275.32012499999996,
"rss": 171245568,
"heapTotal": 181878784,
"heapUsed": 46842960,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 329.858125,
"rss": 171409408,
"heapTotal": 182140928,
"heapUsed": 46988472,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 372.46854199999996,
"rss": 171507712,
"heapTotal": 182403072,
"heapUsed": 47139504,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 395.50424999999996,
"rss": 171720704,
"heapTotal": 182403072,
"heapUsed": 47282504,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 430.291167,
"rss": 174358528,
"heapTotal": 185024512,
"heapUsed": 49947720,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 460.099042,
"rss": 176521216,
"heapTotal": 186859520,
"heapUsed": 51775528,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "after",
"elapsedMs": 521.9577919999999,
"rss": 198000640,
"heapTotal": 147144704,
"heapUsed": 11731728,
"external": 2455679,
"arrayBuffers": 143819
}
],
"sampledRetainedPeakDeltaBytes": 46851576,
"finalRetainedDeltaBytes": 6807776,
"correctness": {
"claimQueries": 100000,
"committedKeys": null,
"ownedKeys": 100000,
"countedEvents": 100000,
"inputTokens": 10000000,
"outputTokens": 1000000,
"totalTokens": 11000000,
"hasDeferredClaims": false,
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
}
},
{
"round": 1,
"order": ["baseline", "current"],
"arm": "current",
"stride": 1,
"duringSampleCount": 9,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 2.388917000000003,
"rss": 54624256,
"heapTotal": 7929856,
"heapUsed": 4935392,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 43.105542,
"rss": 69894144,
"heapTotal": 11681792,
"heapUsed": 6351040,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 75.529417,
"rss": 75300864,
"heapTotal": 16728064,
"heapUsed": 7330792,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 110.00737500000001,
"rss": 76808192,
"heapTotal": 18169856,
"heapUsed": 8640192,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 147.38125000000002,
"rss": 78299136,
"heapTotal": 18432000,
"heapUsed": 9275016,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 181.302042,
"rss": 83001344,
"heapTotal": 27344896,
"heapUsed": 9940712,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 215.42712500000002,
"rss": 88145920,
"heapTotal": 29442048,
"heapUsed": 11844616,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 250.383958,
"rss": 89227264,
"heapTotal": 29966336,
"heapUsed": 12414208,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 285.865792,
"rss": 92225536,
"heapTotal": 30752768,
"heapUsed": 13073640,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 339.514083,
"rss": 95862784,
"heapTotal": 32079872,
"heapUsed": 14447032,
"external": 2455471,
"arrayBuffers": 143611
},
{
"phase": "after",
"elapsedMs": 349.316458,
"rss": 93257728,
"heapTotal": 29442048,
"heapUsed": 11764560,
"external": 2455471,
"arrayBuffers": 143611
}
],
"sampledRetainedPeakDeltaBytes": 9511640,
"finalRetainedDeltaBytes": 6829168,
"correctness": {
"claimQueries": 100000,
"committedKeys": 100000,
"ownedKeys": 100000,
"countedEvents": 100000,
"inputTokens": 10000000,
"outputTokens": 1000000,
"totalTokens": 11000000,
"hasDeferredClaims": false,
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
}
},
{
"round": 1,
"order": ["baseline", "current"],
"arm": "baseline",
"stride": 10,
"duringSampleCount": 3,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 2.742457999999999,
"rss": 54558720,
"heapTotal": 7667712,
"heapUsed": 4924000,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 40.256542,
"rss": 67108864,
"heapTotal": 15794176,
"heapUsed": 6373000,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 75.06275,
"rss": 72105984,
"heapTotal": 17022976,
"heapUsed": 7649648,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 111.00004200000001,
"rss": 81936384,
"heapTotal": 26198016,
"heapUsed": 8795656,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "after",
"elapsedMs": 153.181625,
"rss": 88260608,
"heapTotal": 24444928,
"heapUsed": 5941160,
"external": 2455679,
"arrayBuffers": 143819
}
],
"sampledRetainedPeakDeltaBytes": 3871656,
"finalRetainedDeltaBytes": 1017160,
"correctness": {
"claimQueries": 100000,
"committedKeys": null,
"ownedKeys": 10000,
"countedEvents": 10000,
"inputTokens": 1000000,
"outputTokens": 100000,
"totalTokens": 1100000,
"hasDeferredClaims": true,
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
}
},
{
"round": 1,
"order": ["baseline", "current"],
"arm": "current",
"stride": 10,
"duringSampleCount": 4,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 2.448875000000001,
"rss": 54788096,
"heapTotal": 7929856,
"heapUsed": 4935416,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 39.214,
"rss": 66109440,
"heapTotal": 10813440,
"heapUsed": 5545328,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 74.328625,
"rss": 66650112,
"heapTotal": 10993664,
"heapUsed": 5766000,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 109.42824999999999,
"rss": 66830336,
"heapTotal": 11255808,
"heapUsed": 5927144,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 144.325792,
"rss": 69926912,
"heapTotal": 11419648,
"heapUsed": 6228632,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "after",
"elapsedMs": 165.405959,
"rss": 72056832,
"heapTotal": 10551296,
"heapUsed": 5981088,
"external": 2455471,
"arrayBuffers": 143611
}
],
"sampledRetainedPeakDeltaBytes": 1293216,
"finalRetainedDeltaBytes": 1045672,
"correctness": {
"claimQueries": 100000,
"committedKeys": 10000,
"ownedKeys": 10000,
"countedEvents": 10000,
"inputTokens": 1000000,
"outputTokens": 100000,
"totalTokens": 1100000,
"hasDeferredClaims": true,
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
}
},
{
"round": 2,
"order": ["current", "baseline"],
"arm": "current",
"stride": 1,
"duringSampleCount": 8,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 2.7266249999999985,
"rss": 54542336,
"heapTotal": 7929856,
"heapUsed": 4935464,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 41.696167,
"rss": 69419008,
"heapTotal": 11157504,
"heapUsed": 6326664,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 76.023,
"rss": 75857920,
"heapTotal": 16465920,
"heapUsed": 7366984,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 111.78200000000001,
"rss": 77578240,
"heapTotal": 17907712,
"heapUsed": 8640264,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 146.21325000000002,
"rss": 78397440,
"heapTotal": 18169856,
"heapUsed": 9204112,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 181.093667,
"rss": 84312064,
"heapTotal": 27344896,
"heapUsed": 9868584,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 216.906709,
"rss": 89128960,
"heapTotal": 29442048,
"heapUsed": 11800592,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 251.848084,
"rss": 89964544,
"heapTotal": 29966336,
"heapUsed": 12519232,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 286.69679199999996,
"rss": 92831744,
"heapTotal": 30752768,
"heapUsed": 13286328,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "after",
"elapsedMs": 315.54558399999996,
"rss": 93945856,
"heapTotal": 29179904,
"heapUsed": 11758736,
"external": 2455471,
"arrayBuffers": 143611
}
],
"sampledRetainedPeakDeltaBytes": 8350864,
"finalRetainedDeltaBytes": 6823272,
"correctness": {
"claimQueries": 100000,
"committedKeys": 100000,
"ownedKeys": 100000,
"countedEvents": 100000,
"inputTokens": 10000000,
"outputTokens": 1000000,
"totalTokens": 11000000,
"hasDeferredClaims": false,
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
}
},
{
"round": 2,
"order": ["current", "baseline"],
"arm": "baseline",
"stride": 1,
"duringSampleCount": 8,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 2.759208000000001,
"rss": 54788096,
"heapTotal": 7929856,
"heapUsed": 4923952,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 41.3715,
"rss": 85540864,
"heapTotal": 28983296,
"heapUsed": 11037688,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 77.633541,
"rss": 103530496,
"heapTotal": 52133888,
"heapUsed": 17544376,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 114.46741599999999,
"rss": 115310592,
"heapTotal": 92504064,
"heapUsed": 24639432,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 152.397541,
"rss": 140820480,
"heapTotal": 98484224,
"heapUsed": 30596424,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 191.927833,
"rss": 154828800,
"heapTotal": 105037824,
"heapUsed": 36979976,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 222.867541,
"rss": 160776192,
"heapTotal": 110313472,
"heapUsed": 42347640,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 266.556791,
"rss": 172752896,
"heapTotal": 183451648,
"heapUsed": 48259976,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 297.627125,
"rss": 183664640,
"heapTotal": 187383808,
"heapUsed": 52060344,
"external": 2455471,
"arrayBuffers": 143611
},
{
"phase": "after",
"elapsedMs": 354.97375,
"rss": 195674112,
"heapTotal": 147406848,
"heapUsed": 11724288,
"external": 2455679,
"arrayBuffers": 143819
}
],
"sampledRetainedPeakDeltaBytes": 47136392,
"finalRetainedDeltaBytes": 6800336,
"correctness": {
"claimQueries": 100000,
"committedKeys": null,
"ownedKeys": 100000,
"countedEvents": 100000,
"inputTokens": 10000000,
"outputTokens": 1000000,
"totalTokens": 11000000,
"hasDeferredClaims": false,
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
}
},
{
"round": 2,
"order": ["current", "baseline"],
"arm": "current",
"stride": 10,
"duringSampleCount": 4,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 2.1205,
"rss": 54591488,
"heapTotal": 7929856,
"heapUsed": 4935416,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 38.77175,
"rss": 65896448,
"heapTotal": 10813440,
"heapUsed": 5596672,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 74.407583,
"rss": 66158592,
"heapTotal": 10993664,
"heapUsed": 5771488,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 109.03608299999999,
"rss": 66912256,
"heapTotal": 11255808,
"heapUsed": 5941256,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "during",
"elapsedMs": 143.820708,
"rss": 69861376,
"heapTotal": 11419648,
"heapUsed": 6247120,
"external": 2586751,
"arrayBuffers": 274891
},
{
"phase": "after",
"elapsedMs": 161.987125,
"rss": 71991296,
"heapTotal": 10551296,
"heapUsed": 5976440,
"external": 2455471,
"arrayBuffers": 143611
}
],
"sampledRetainedPeakDeltaBytes": 1311704,
"finalRetainedDeltaBytes": 1041024,
"correctness": {
"claimQueries": 100000,
"committedKeys": 10000,
"ownedKeys": 10000,
"countedEvents": 10000,
"inputTokens": 1000000,
"outputTokens": 100000,
"totalTokens": 1100000,
"hasDeferredClaims": true,
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
}
},
{
"round": 2,
"order": ["current", "baseline"],
"arm": "baseline",
"stride": 10,
"duringSampleCount": 3,
"finalOnlyMeasurement": false,
"baseline": {
"phase": "before",
"elapsedMs": 2.370624999999997,
"rss": 54919168,
"heapTotal": 7929856,
"heapUsed": 4924000,
"external": 2455431,
"arrayBuffers": 143611
},
"samples": [
{
"phase": "during",
"elapsedMs": 40.106750000000005,
"rss": 67092480,
"heapTotal": 15532032,
"heapUsed": 6373000,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 74.903292,
"rss": 71827456,
"heapTotal": 16760832,
"heapUsed": 7649648,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "during",
"elapsedMs": 110.34537499999999,
"rss": 81395712,
"heapTotal": 26722304,
"heapUsed": 8851816,
"external": 2586543,
"arrayBuffers": 274683
},
{
"phase": "after",
"elapsedMs": 149.984917,
"rss": 86966272,
"heapTotal": 24444928,
"heapUsed": 5941160,
"external": 2455679,
"arrayBuffers": 143819
}
],
"sampledRetainedPeakDeltaBytes": 3927816,
"finalRetainedDeltaBytes": 1017160,
"correctness": {
"claimQueries": 100000,
"committedKeys": null,
"ownedKeys": 10000,
"countedEvents": 10000,
"inputTokens": 1000000,
"outputTokens": 100000,
"totalTokens": 1100000,
"hasDeferredClaims": true,
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
}
}
]
}
@@ -0,0 +1,103 @@
# Verified transcript reads and Codex usage memory
Codex could verify one file generation, reopen a replacement, and publish old
totals under the replacement's metadata. A short parse could also publish a
partial result as a complete file. Unchanged refreshes then reused those totals.
The shared JSONL reader now pins verification, parsing and checkpoint creation
to one descriptor and the initial file size. It proves complete range consumption
and validates generation and byte evidence before publishing. Stable short reads
propagate errors; only evidence that the file changed permits a retry. Each failed
attempt closes its handle. A successful attempt closes before ownership publication
and aggregate work. Metadata-less caches reparse once.
Codex folds owned records into the existing session/day accumulator while reading.
It commits ownership after validation, and lets unchanged forks reclaim keys that
an owner actually loses. Deferred records still advance cumulative token context.
This removes intermediate event arrays while preserving all history and results.
## Reproduce and verify
```sh
ORCA_BACKGROUND_LAUNCH=1 pnpm test src/main/codex-usage src/main/usage
ORCA_BACKGROUND_LAUNCH=1 node config/scripts/codex-usage-parser-memory-benchmark.mjs --baseline-ref 51e7181850b022bae2d41091fe1229accef4bd50 --events 100000 --pairs 2 --sample-ms 35 --output docs/bug-reproductions/usage-scan-append/codex-parser-memory.json
```
The original [memory comparison](./codex-parser-memory.json) bundles the full parser
dependency graph from the named baseline and current worktree. Two counterbalanced
rounds use fresh Node workers and record source/tool hashes and raw forced-GC
snapshots. Every arm preserves identical projections, owned keys and token totals.
| Owned records | Baseline sampled retained heap | Current sampled retained heap |
| ------------------------- | -----------------------------: | ----------------------------: |
| 100,000 | 44.7–45.0 MiB | 8.0–9.1 MiB |
| 10,000, with 90% deferred | 3.69–3.75 MiB | 1.23–1.25 MiB |
Final result retention stays approximately 6.49 → 6.51 MiB for all owned records:
the result still contains the full history. Timer samples miss synchronous peaks,
and forced GC affects timing; these figures measure sampled retained heap rather
than total process memory or speed.
Real-file regressions cover replacement, truncation, rewrite, suffix resume,
partial tails, stable short reads, failed ownership publication, old-cache upgrade
and handle cleanup. Local checks used macOS; CI covers Linux and Windows.
The shared aggregation compatibility suites also passed for Muse and OpenCode.
Head and committed-boundary digests support the append-only contract. They cannot
prove arbitrary middle-only rewrites during growth. Small changed files receive
a full verification reread; unchanged files read no transcript bytes.
The companion Claude PR adopts this reader. The original integrated validation
and benchmarks were recorded from a combined worktree; production source remains
identical when split into PRs.
## Adversarial hardening
Tiny files now use bounded descriptor reads and exact byte comparison for both
passes. This avoids creating a stream without reducing verification. Exact bigint
device/inode values prevent large Windows file IDs from colliding after numeric
rounding; unsafe persisted offsets fall back to a complete parse.
Discovery distinguishes a directory that disappeared from one that cannot be
read. Missing children leave readable siblings available. Permission and I/O
errors abort the refresh, retaining the previous verified totals and showing the
existing error state rather than publishing incomplete totals.
A [counterbalanced ten-round comparison](./shared-reader-small-files.json) of the published reader and hardened
reader over 1,000 tiny files measured median scanner cold time 85.024 → 80.529 ms
and changed time 91.512 → 84.092 ms. Both arms read the same bytes twice; streams
fell from 1,000 to zero. Unchanged scans read zero bytes in both arms and measured
7.808 → 8.241 ms. These page-cache-warm macOS timings are observations, not CI
thresholds or a claim about the original pre-verification reader.
Regression contracts exercise short reads, exact comparison, unsafe offsets,
large file identities, fractional timestamps, and discovery failures. The
original retained-heap artifact above predates this hardening; its source hashes
describe that earlier measurement, rather than the final reader.
## Further risk reduction
The [refreshed memory comparison](./codex-parser-memory-final.json) captures the
final parser and reader source. Its two counterbalanced rounds use eight fresh
Node workers and preserve exact output parity with the original baseline.
| Owned records | Original sampled retained heap | Final sampled retained heap |
| ------------------------- | -----------------------------: | --------------------------: |
| 100,000 | 43.87–44.54 MiB | 8.07–8.30 MiB |
| 10,000, with 90% deferred | 4.53–4.58 MiB | 1.231–1.235 MiB |
The median reductions are approximately 81.5% and 72.9%. Final output retention
stays similar or slightly higher: 6.49 → 6.53–6.55 MiB for owned events, and
0.975 → 1.005–1.014 MiB for deferred events. These are forced-GC samples, not
absolute peaks, full-app memory, or speed measurements.
Real-file lifetime contracts cover both tiny and streamed transcripts. They
prove the descriptor is closed before an ownership callback replaces the path,
the verified old projection keeps its captured metadata, and the next parse
observes the replacement. Failure cases check cleanup when validation fails or
the first close attempt rejects before reaching the native close.
This shortens Windows replacement contention to reading and validation; parsing
can still include asynchronous attribution while the descriptor is pinned.
Ordinary append writes remain compatible. Tiny verification still reads the full
file twice, and bounded append evidence retains its append-only assumption.
@@ -0,0 +1,984 @@
{
"schemaVersion": 1,
"scenario": "1000-small-files",
"scope": "Shared JSONL reader and file metadata hardening, exercised through the combined Claude scanner; the current Claude projection implementation is identical in both arms.",
"capturedAtUtc": "2026-10-09",
"environment": {
"node": "v24.20.0",
"platform": "darwin",
"osRelease": "25.3.0",
"arch": "arm64",
"cpuModel": "Apple M4 Max"
},
"rounds": 10,
"schedule": "Alternating before/after and after/before; five rounds in each order; tests and builds held during this final capture.",
"baselineCommit": "03a2ecaa00138ee44ad0dbb9dbd957cfe1a5e3f8",
"baselineModules": [
"src/main/claude-usage/transcript-record-parser.ts",
"src/main/usage/jsonl-file-checkpoint.ts",
"src/main/usage/jsonl-file-snapshot.ts",
"src/main/usage/jsonl-line-offsets.ts"
],
"baselineHashesVerifiedAgainstGit": true,
"currentSourceHashesVerifiedAgainstFrozenWorktree": true,
"sourceFingerprintLabels": {
"HEAD": "The four pinned published baseline modules supplied as an overlay; this is the existing loader label, not the current Git HEAD.",
"worktree": "Frozen combined-worktree modules, held identical in both arms except the four baseline overlays."
},
"methodology": {
"cold": "1,000 freshly written tiny transcripts; empty scanner cache; filesystem page cache warm.",
"changed": "Append a duplicate usage row with higher token maxima to each file, then scan from the unchanged cache.",
"warm": "Reuse each arm's cold persisted cache without changing files; no transcript content reads.",
"durationMs": "Production scan only, excluding fixture writes and cache serialization.",
"totalDurationMs": "Production scan plus JSON cache serialization, excluding hydration and fixture writes.",
"bytesRead": "All raw bytes reported by the existing benchmark loader, including verification reads.",
"readCount": "Logical filesystem API read records, including streams; this does not claim an OS syscall count.",
"parity": "Exact sessions, daily aggregates, and per-file projections in cold/changed phases across arms, excluding volatile stat values and additive generation/resume metadata.",
"median": "Average of the middle two values for the ten samples; the temporary capture script's upper-middle summaries are retained separately.",
"reproduction": [
"Use loadClaudeUsageBenchmarkScanner and measureClaudeUsageBenchmarkScan from config/scripts/claude-usage-benchmark-scanner.mjs in the combined Claude/shared-reader worktree.",
"Load only the four baselineModules from baselineCommit into the before source-overlay map; load every other module once into a shared worktree-source map used by both arms.",
"Create an isolated agent home containing 1,000 JSONL files, one keyed assistant usage row per file, with input_tokens=100 and output_tokens=10.",
"Measure a cold scan, an unchanged scan from its cache, then append the same keyed row with input_tokens=200 and measure the changed scan. Exclude batched fixture writes from timers.",
"Alternate arm order for ten rounds, assert exact projected behavior parity for cold and changed phases, and assert that every unchanged scan reads zero transcript bytes.",
"Compare byte counts and stream counts separately from duration, and record all raw samples and source fingerprints."
]
},
"captureScriptSha256": "d4bac98e4bd15f0ba4760646b9c63cd69160b4f5df7d09ae7b22b101ea0fc9c5",
"captureScriptStatus": "Local measurement adapter reused the existing loader; no separate benchmark framework or production tool was added.",
"toolingSha256": {
"config/scripts/claude-usage-benchmark-scanner.mjs": "ffb6e5defb921f2ab35726a716f5e731dec439ead53cc2f3a192c7bef4df45d3",
"config/scripts/counterbalanced-benchmark-schedule.mjs": "bc35b40e954fa7ddec0225ed375bd3b113c2e05b8554bc50734d14f89987294d"
},
"historyBytes": 307780,
"changedSnapshotBytes": 615560,
"exactColdChangedProjectionParity": true,
"medians": {
"before": {
"cold": {
"durationMs": 85.02441700000008,
"cacheSerializationMs": 1.705395499999895,
"totalDurationMs": 86.68222949999995,
"bytesRead": 615560.0,
"directReadBytes": 307780.0,
"readCount": 2000.0,
"streamCount": 1000.0,
"cachedStateBytes": 1508259.0
},
"warm": {
"durationMs": 7.808062500000233,
"cacheSerializationMs": 1.4102915000000849,
"totalDurationMs": 9.195042000000285,
"bytesRead": 0.0,
"directReadBytes": 0.0,
"readCount": 0.0,
"streamCount": 0.0,
"cachedStateBytes": 1508259.0
},
"changed": {
"durationMs": 91.5124165000002,
"cacheSerializationMs": 1.5766254999998637,
"totalDurationMs": 93.02370800000017,
"bytesRead": 1231120.0,
"directReadBytes": 615560.0,
"readCount": 2000.0,
"streamCount": 1000.0,
"cachedStateBytes": 1508232.0
}
},
"after": {
"cold": {
"durationMs": 80.5293959999999,
"cacheSerializationMs": 1.6749375000002829,
"totalDurationMs": 82.21989599999984,
"bytesRead": 615560.0,
"directReadBytes": 615560.0,
"readCount": 2000.0,
"streamCount": 0.0,
"cachedStateBytes": 1508252.0
},
"warm": {
"durationMs": 8.240729500000043,
"cacheSerializationMs": 1.4088955000003125,
"totalDurationMs": 9.655500500000016,
"bytesRead": 0.0,
"directReadBytes": 0.0,
"readCount": 0.0,
"streamCount": 0.0,
"cachedStateBytes": 1508252.0
},
"changed": {
"durationMs": 84.09189549999974,
"cacheSerializationMs": 1.587229000000093,
"totalDurationMs": 85.75574999999981,
"bytesRead": 1231120.0,
"directReadBytes": 1231120.0,
"readCount": 2000.0,
"streamCount": 0.0,
"cachedStateBytes": 1508243.0
}
}
},
"durationChangePercent": {
"cold": -5.286741336903466,
"warm": 5.541285049905742,
"changed": -8.108758662274473
},
"samples": [
{
"arm": "before",
"round": 0,
"cold": {
"durationMs": 392.049709,
"cacheSerializationMs": 1.6989579999999478,
"totalDurationMs": 393.74866699999995,
"bytesRead": 615560,
"directReadBytes": 307780,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508277
},
"warm": {
"durationMs": 48.59562500000004,
"cacheSerializationMs": 1.523916999999983,
"totalDurationMs": 50.119542000000024,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508277
},
"changed": {
"durationMs": 233.39462500000002,
"cacheSerializationMs": 1.4007910000000265,
"totalDurationMs": 234.79541600000005,
"bytesRead": 1231120,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508233
}
},
{
"arm": "after",
"round": 0,
"cold": {
"durationMs": 132.50262499999985,
"cacheSerializationMs": 1.6758330000000115,
"totalDurationMs": 134.17845799999986,
"bytesRead": 615560,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508243
},
"warm": {
"durationMs": 13.144000000000005,
"cacheSerializationMs": 1.4084580000001097,
"totalDurationMs": 14.552458000000115,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508243
},
"changed": {
"durationMs": 111.28033299999993,
"cacheSerializationMs": 1.5340830000000096,
"totalDurationMs": 112.81441599999994,
"bytesRead": 1231120,
"directReadBytes": 1231120,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508271
}
},
{
"arm": "after",
"round": 1,
"cold": {
"durationMs": 174.1105,
"cacheSerializationMs": 1.588083000000097,
"totalDurationMs": 175.6985830000001,
"bytesRead": 615560,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508241
},
"warm": {
"durationMs": 12.503875000000107,
"cacheSerializationMs": 1.357582999999977,
"totalDurationMs": 13.861458000000084,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508241
},
"changed": {
"durationMs": 127.85537499999987,
"cacheSerializationMs": 1.5461250000000746,
"totalDurationMs": 129.40149999999994,
"bytesRead": 1231120,
"directReadBytes": 1231120,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508289
}
},
{
"arm": "before",
"round": 1,
"cold": {
"durationMs": 126.94487499999991,
"cacheSerializationMs": 1.7684590000001208,
"totalDurationMs": 128.71333400000003,
"bytesRead": 615560,
"directReadBytes": 307780,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508223
},
"warm": {
"durationMs": 8.401833000000124,
"cacheSerializationMs": 1.3886669999999413,
"totalDurationMs": 9.790500000000065,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508223
},
"changed": {
"durationMs": 114.26195800000005,
"cacheSerializationMs": 1.8519999999998618,
"totalDurationMs": 116.11395799999991,
"bytesRead": 1231120,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508231
}
},
{
"arm": "before",
"round": 2,
"cold": {
"durationMs": 91.58558300000004,
"cacheSerializationMs": 1.4894590000003518,
"totalDurationMs": 93.0750420000004,
"bytesRead": 615560,
"directReadBytes": 307780,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508261
},
"warm": {
"durationMs": 8.029250000000047,
"cacheSerializationMs": 1.4045000000000982,
"totalDurationMs": 9.433750000000146,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508261
},
"changed": {
"durationMs": 95.63120799999979,
"cacheSerializationMs": 1.6717079999998532,
"totalDurationMs": 97.30291599999964,
"bytesRead": 1231120,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508229
}
},
{
"arm": "after",
"round": 2,
"cold": {
"durationMs": 109.59825000000001,
"cacheSerializationMs": 2.067250000000058,
"totalDurationMs": 111.66550000000007,
"bytesRead": 615560,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508283
},
"warm": {
"durationMs": 13.671583000000282,
"cacheSerializationMs": 1.3706250000000182,
"totalDurationMs": 15.0422080000003,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508283
},
"changed": {
"durationMs": 108.32254200000034,
"cacheSerializationMs": 1.7173749999997199,
"totalDurationMs": 110.03991700000006,
"bytesRead": 1231120,
"directReadBytes": 1231120,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508251
}
},
{
"arm": "after",
"round": 3,
"cold": {
"durationMs": 80.29837499999985,
"cacheSerializationMs": 1.6784579999998641,
"totalDurationMs": 81.97683299999971,
"bytesRead": 615560,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508213
},
"warm": {
"durationMs": 8.142833000000337,
"cacheSerializationMs": 1.4027909999999792,
"totalDurationMs": 9.545624000000316,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508213
},
"changed": {
"durationMs": 88.40525000000025,
"cacheSerializationMs": 1.6283330000001115,
"totalDurationMs": 90.03358300000036,
"bytesRead": 1231120,
"directReadBytes": 1231120,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508237
}
},
{
"arm": "before",
"round": 3,
"cold": {
"durationMs": 75.91833399999996,
"cacheSerializationMs": 1.7118329999998423,
"totalDurationMs": 77.6301669999998,
"bytesRead": 615560,
"directReadBytes": 307780,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508261
},
"warm": {
"durationMs": 7.40179100000023,
"cacheSerializationMs": 1.40166599999975,
"totalDurationMs": 8.80345699999998,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508261
},
"changed": {
"durationMs": 86.82458300000008,
"cacheSerializationMs": 1.5110839999997552,
"totalDurationMs": 88.33566699999983,
"bytesRead": 1231120,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508229
}
},
{
"arm": "before",
"round": 4,
"cold": {
"durationMs": 86.88720899999998,
"cacheSerializationMs": 1.7541670000000522,
"totalDurationMs": 88.64137600000004,
"bytesRead": 615560,
"directReadBytes": 307780,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508281
},
"warm": {
"durationMs": 7.466167000000041,
"cacheSerializationMs": 1.4322910000000775,
"totalDurationMs": 8.898458000000119,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508281
},
"changed": {
"durationMs": 91.41150000000016,
"cacheSerializationMs": 1.5426669999997102,
"totalDurationMs": 92.95416699999987,
"bytesRead": 1231120,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508227
}
},
{
"arm": "after",
"round": 4,
"cold": {
"durationMs": 80.76041699999996,
"cacheSerializationMs": 1.702541999999994,
"totalDurationMs": 82.46295899999996,
"bytesRead": 615560,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508227
},
"warm": {
"durationMs": 8.741708000000017,
"cacheSerializationMs": 1.4854580000001079,
"totalDurationMs": 10.227166000000125,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508227
},
"changed": {
"durationMs": 77.78320800000029,
"cacheSerializationMs": 1.4414170000000013,
"totalDurationMs": 79.22462500000029,
"bytesRead": 1231120,
"directReadBytes": 1231120,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508195
}
},
{
"arm": "after",
"round": 5,
"cold": {
"durationMs": 75.24266699999998,
"cacheSerializationMs": 1.6534579999997732,
"totalDurationMs": 76.89612499999976,
"bytesRead": 615560,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508281
},
"warm": {
"durationMs": 8.298541999999998,
"cacheSerializationMs": 1.406333999999788,
"totalDurationMs": 9.704875999999786,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508281
},
"changed": {
"durationMs": 76.96762499999977,
"cacheSerializationMs": 1.446374999999989,
"totalDurationMs": 78.41399999999976,
"bytesRead": 1231120,
"directReadBytes": 1231120,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508247
}
},
{
"arm": "before",
"round": 5,
"cold": {
"durationMs": 83.16162500000019,
"cacheSerializationMs": 1.561457999999675,
"totalDurationMs": 84.72308299999986,
"bytesRead": 615560,
"directReadBytes": 307780,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508211
},
"warm": {
"durationMs": 7.586875000000418,
"cacheSerializationMs": 1.3694590000000062,
"totalDurationMs": 8.956334000000425,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508211
},
"changed": {
"durationMs": 81.44929200000024,
"cacheSerializationMs": 1.6840000000001965,
"totalDurationMs": 83.13329200000044,
"bytesRead": 1231120,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508301
}
},
{
"arm": "before",
"round": 6,
"cold": {
"durationMs": 83.09058300000015,
"cacheSerializationMs": 1.5731669999995574,
"totalDurationMs": 84.66374999999971,
"bytesRead": 615560,
"directReadBytes": 307780,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508219
},
"warm": {
"durationMs": 7.298624999999447,
"cacheSerializationMs": 1.4347920000000158,
"totalDurationMs": 8.733416999999463,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508219
},
"changed": {
"durationMs": 84.96458299999995,
"cacheSerializationMs": 1.6105840000000171,
"totalDurationMs": 86.57516699999996,
"bytesRead": 1231120,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508219
}
},
{
"arm": "after",
"round": 6,
"cold": {
"durationMs": 70.85174999999981,
"cacheSerializationMs": 1.4307080000007772,
"totalDurationMs": 72.28245800000059,
"bytesRead": 615560,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508265
},
"warm": {
"durationMs": 7.037790999999743,
"cacheSerializationMs": 1.4527090000001408,
"totalDurationMs": 8.490499999999884,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508265
},
"changed": {
"durationMs": 71.76158299999952,
"cacheSerializationMs": 1.6893340000005992,
"totalDurationMs": 73.45091700000012,
"bytesRead": 1231120,
"directReadBytes": 1231120,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508197
}
},
{
"arm": "after",
"round": 7,
"cold": {
"durationMs": 71.874417,
"cacheSerializationMs": 1.7616250000000946,
"totalDurationMs": 73.63604200000009,
"bytesRead": 615560,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508241
},
"warm": {
"durationMs": 7.877457999999933,
"cacheSerializationMs": 1.4134170000006634,
"totalDurationMs": 9.290875000000597,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508241
},
"changed": {
"durationMs": 71.59512500000073,
"cacheSerializationMs": 1.6312079999997877,
"totalDurationMs": 73.22633300000052,
"bytesRead": 1231120,
"directReadBytes": 1231120,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508171
}
},
{
"arm": "before",
"round": 7,
"cold": {
"durationMs": 76.68987500000003,
"cacheSerializationMs": 1.752542000000176,
"totalDurationMs": 78.4424170000002,
"bytesRead": 615560,
"directReadBytes": 307780,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508263
},
"warm": {
"durationMs": 6.472415999999612,
"cacheSerializationMs": 1.423917000000074,
"totalDurationMs": 7.896332999999686,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508263
},
"changed": {
"durationMs": 93.93520900000021,
"cacheSerializationMs": 1.528958999999304,
"totalDurationMs": 95.46416799999952,
"bytesRead": 1231120,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508253
}
},
{
"arm": "before",
"round": 8,
"cold": {
"durationMs": 99.58229100000062,
"cacheSerializationMs": 2.150957999999264,
"totalDurationMs": 101.73324899999989,
"bytesRead": 615560,
"directReadBytes": 307780,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508241
},
"warm": {
"durationMs": 8.437750000000051,
"cacheSerializationMs": 1.4160830000000715,
"totalDurationMs": 9.853833000000122,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508241
},
"changed": {
"durationMs": 91.61333300000024,
"cacheSerializationMs": 1.4799160000002303,
"totalDurationMs": 93.09324900000047,
"bytesRead": 1231120,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508269
}
},
{
"arm": "after",
"round": 8,
"cold": {
"durationMs": 118.9740839999995,
"cacheSerializationMs": 1.6740420000005543,
"totalDurationMs": 120.64812600000005,
"bytesRead": 615560,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508267
},
"warm": {
"durationMs": 7.485000000000582,
"cacheSerializationMs": 1.4093330000005153,
"totalDurationMs": 8.894333000001097,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508267
},
"changed": {
"durationMs": 83.58533299999999,
"cacheSerializationMs": 1.4950419999995574,
"totalDurationMs": 85.08037499999955,
"bytesRead": 1231120,
"directReadBytes": 1231120,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508251
}
},
{
"arm": "after",
"round": 9,
"cold": {
"durationMs": 76.86124999999993,
"cacheSerializationMs": 1.6329170000008162,
"totalDurationMs": 78.49416700000074,
"bytesRead": 615560,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508261
},
"warm": {
"durationMs": 8.182917000000089,
"cacheSerializationMs": 1.4232080000001588,
"totalDurationMs": 9.606125000000247,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508261
},
"changed": {
"durationMs": 84.59845799999948,
"cacheSerializationMs": 1.8326670000005834,
"totalDurationMs": 86.43112500000007,
"bytesRead": 1231120,
"directReadBytes": 1231120,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508239
}
},
{
"arm": "before",
"round": 9,
"cold": {
"durationMs": 79.30645899999945,
"cacheSerializationMs": 1.5558329999994385,
"totalDurationMs": 80.86229199999889,
"bytesRead": 615560,
"directReadBytes": 307780,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508257
},
"warm": {
"durationMs": 8.131749999999556,
"cacheSerializationMs": 1.381416999999601,
"totalDurationMs": 9.513166999999157,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508257
},
"changed": {
"durationMs": 90.45725000000039,
"cacheSerializationMs": 1.6439999999993233,
"totalDurationMs": 92.10124999999971,
"bytesRead": 1231120,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508269
}
}
],
"capturedUpperMiddleSummaries": {
"before": {
"cold": {
"durationMs": 86.88720899999998,
"cacheSerializationMs": 1.7118329999998423,
"totalDurationMs": 88.64137600000004,
"bytesRead": 615560,
"directReadBytes": 307780,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508261
},
"warm": {
"durationMs": 8.029250000000047,
"cacheSerializationMs": 1.4160830000000715,
"totalDurationMs": 9.433750000000146,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508261
},
"changed": {
"durationMs": 91.61333300000024,
"cacheSerializationMs": 1.6105840000000171,
"totalDurationMs": 93.09324900000047,
"bytesRead": 1231120,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 1000,
"cachedStateBytes": 1508233
}
},
"after": {
"cold": {
"durationMs": 80.76041699999996,
"cacheSerializationMs": 1.6758330000000115,
"totalDurationMs": 82.46295899999996,
"bytesRead": 615560,
"directReadBytes": 615560,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508261
},
"warm": {
"durationMs": 8.298541999999998,
"cacheSerializationMs": 1.4093330000005153,
"totalDurationMs": 9.704875999999786,
"bytesRead": 0,
"directReadBytes": 0,
"readCount": 0,
"streamCount": 0,
"cachedStateBytes": 1508261
},
"changed": {
"durationMs": 84.59845799999948,
"cacheSerializationMs": 1.6283330000001115,
"totalDurationMs": 86.43112500000007,
"bytesRead": 1231120,
"directReadBytes": 1231120,
"readCount": 2000,
"streamCount": 0,
"cachedStateBytes": 1508247
}
}
},
"sourceFingerprints": {
"before": {
"src/main/claude-usage/scanner.ts": {
"source": "worktree",
"sha256": "ef42add20f63152cf2ebe106e0794119b9c388c763736c8450b559ae7c2418f5"
},
"src/main/claude-usage/transcript-file-discovery.ts": {
"source": "worktree",
"sha256": "54c84d0b99b45c0fa13e589a1bcec2b479cb01e8b17bcbddbb65a4bc85a03401"
},
"src/main/claude-usage/transcript-projection-integrity.ts": {
"source": "worktree",
"sha256": "f4efccd6f1be0603a90174f66ac961d07ea5bf1c1fbebe015b0537deee1c58f2"
},
"src/main/claude-usage/transcript-record-parser.ts": {
"source": "HEAD",
"sha256": "d4fab89434c1b6c60480a561310542246be34ef86bc13da0340c1dfdbb269854"
},
"src/main/claude-usage/transcript-resume-projection.ts": {
"source": "worktree",
"sha256": "b2ed0a589b60fdccb02c72fa234368853d32be080a84ee3bcc93f5834478a153"
},
"src/main/claude-usage/transcript-token-checkpoint.ts": {
"source": "worktree",
"sha256": "e96cc4aa9b899cad90aee462f053f483f5bb010dea99de1ee11a9f8fd8a2d0f2"
},
"src/main/claude-usage/transcript-usage-projection.ts": {
"source": "worktree",
"sha256": "56e2fe017f806670595dae766b6291e09b05b321b83e4b83e1ae5166bab2041c"
},
"src/main/claude-usage/usage-aggregation.ts": {
"source": "worktree",
"sha256": "be4dad6f41eaf720252137b16b9a96a2cfdd2e73b05eee097993653ce6c5e509"
},
"src/main/claude-usage/worktree-attribution.ts": {
"source": "worktree",
"sha256": "7450e5ca7d8dcb66d1f5003a3e445c29f05c1416c6cd63aeb3f5f74804bc6213"
},
"src/main/usage/jsonl-file-checkpoint.ts": {
"source": "HEAD",
"sha256": "8eb579f356e1bb23b36319f00b3d4603fff80a963222dcb8a22331319cab3a03"
},
"src/main/usage/jsonl-file-snapshot.ts": {
"source": "HEAD",
"sha256": "1bd98ac194f73097cd6c204238fe1db01e1e0d60a604e93c1d7841725971fe0d"
},
"src/main/usage/jsonl-line-offsets.ts": {
"source": "HEAD",
"sha256": "f119465fad3771acd72b128853e84eed9165f0f74b75743ad662aa5bab991273"
}
},
"after": {
"src/main/claude-usage/scanner.ts": {
"source": "worktree",
"sha256": "ef42add20f63152cf2ebe106e0794119b9c388c763736c8450b559ae7c2418f5"
},
"src/main/claude-usage/transcript-file-discovery.ts": {
"source": "worktree",
"sha256": "54c84d0b99b45c0fa13e589a1bcec2b479cb01e8b17bcbddbb65a4bc85a03401"
},
"src/main/claude-usage/transcript-projection-integrity.ts": {
"source": "worktree",
"sha256": "f4efccd6f1be0603a90174f66ac961d07ea5bf1c1fbebe015b0537deee1c58f2"
},
"src/main/claude-usage/transcript-record-parser.ts": {
"source": "worktree",
"sha256": "6598c02e38ea1776ef5ef926ca424dbc5880f27e7ff03b651cf8f60b791996f6"
},
"src/main/claude-usage/transcript-resume-projection.ts": {
"source": "worktree",
"sha256": "b2ed0a589b60fdccb02c72fa234368853d32be080a84ee3bcc93f5834478a153"
},
"src/main/claude-usage/transcript-token-checkpoint.ts": {
"source": "worktree",
"sha256": "e96cc4aa9b899cad90aee462f053f483f5bb010dea99de1ee11a9f8fd8a2d0f2"
},
"src/main/claude-usage/transcript-usage-projection.ts": {
"source": "worktree",
"sha256": "56e2fe017f806670595dae766b6291e09b05b321b83e4b83e1ae5166bab2041c"
},
"src/main/claude-usage/usage-aggregation.ts": {
"source": "worktree",
"sha256": "be4dad6f41eaf720252137b16b9a96a2cfdd2e73b05eee097993653ce6c5e509"
},
"src/main/claude-usage/worktree-attribution.ts": {
"source": "worktree",
"sha256": "7450e5ca7d8dcb66d1f5003a3e445c29f05c1416c6cd63aeb3f5f74804bc6213"
},
"src/main/usage/jsonl-file-checkpoint.ts": {
"source": "worktree",
"sha256": "b7f307564989dabaf1dfda095a26c7bd9f0683e0efcabe79a010c7b58fa1a2d4"
},
"src/main/usage/jsonl-file-snapshot.ts": {
"source": "worktree",
"sha256": "b1b8869db92c08c762160acba7570aabff25879649cb6ffa3826912ee35deffe"
},
"src/main/usage/jsonl-line-offsets.ts": {
"source": "worktree",
"sha256": "f3574adfad4549c6330e2ac52760c91be5a8cbb9bf8e1beb440e3226087074db"
}
}
},
"limits": [
"One macOS host and page-cache-warm fixtures; durations are evidence for this scenario, not a platform-independent performance contract.",
"This comparison is against the published incremental reader implementation, not the pre-PR full-scan implementation.",
"Both arms still read each small cold/changed snapshot twice; unchanged scans read zero transcript bytes.",
"Warm duration rose by about 0.43 ms over 1,000 files in this capture; exact bigint metadata adds CPU work without adding stat calls.",
"Large-file append-only checkpoints retain their documented limitation for arbitrary middle rewrites during growth."
]
}
@@ -0,0 +1,145 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { mkdtempSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
import type * as NodeFsPromises from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { parseCodexUsageFile } from './codex-rollout-file-parse'
const { openedHandles, closedHandles, failNextClose } = vi.hoisted(() => ({
openedHandles: new Array<NodeFsPromises.FileHandle>(),
closedHandles: new Set<NodeFsPromises.FileHandle>(),
failNextClose: { current: false }
}))
vi.mock('node:fs/promises', async () => {
const actual = await vi.importActual<typeof NodeFsPromises>('node:fs/promises')
return {
...actual,
open: async (...args: Parameters<typeof actual.open>) => {
const handle = await actual.open(...args)
openedHandles.push(handle)
const closeHandle = handle.close.bind(handle)
vi.spyOn(handle, 'close').mockImplementation(async () => {
if (failNextClose.current) {
failNextClose.current = false
throw new Error('descriptor close failed')
}
await closeHandle()
closedHandles.add(handle)
})
return handle
}
}
})
function rollout(sessionId: string, recordCount: number): string {
const records = [JSON.stringify({ type: 'session_meta', payload: { id: sessionId } })]
for (let index = 0; index < recordCount; index++) {
records.push(
JSON.stringify({
timestamp: new Date(Date.UTC(2026, 4, 26, 12, index)).toISOString(),
type: 'event_msg',
payload: {
type: 'token_count',
info: {
model: 'gpt-5-codex',
last_token_usage: { input_tokens: 1, total_tokens: 1 },
total_token_usage: { input_tokens: index + 1, total_tokens: index + 1 }
}
}
})
)
}
return `${records.join('\n')}\n`
}
let directory: string
beforeEach(() => {
directory = mkdtempSync(join(tmpdir(), 'orca-codex-rollout-lifetime-'))
openedHandles.length = 0
closedHandles.clear()
failNextClose.current = false
})
afterEach(() => {
rmSync(directory, { recursive: true, force: true })
vi.clearAllMocks()
})
describe('Codex rollout descriptor lifetime', () => {
it.each([3, 100])(
'closes the verified %i-record rollout before publishing ownership',
async (recordCount) => {
const rolloutPath = join(directory, 'rollout.jsonl')
const replacementPath = join(directory, 'replacement.tmp')
writeFileSync(rolloutPath, rollout('verified-session', recordCount))
const originalStat = statSync(rolloutPath, { bigint: true })
writeFileSync(replacementPath, rollout('replacement-session', 2))
let committedKeys = 0
const verified = await parseCodexUsageFile(rolloutPath, () => null, {
commitEventKey: () => {
expect(closedHandles.size).toBe(openedHandles.length)
expect(openedHandles.every((handle) => handle.fd === -1)).toBe(true)
if (committedKeys++ === 0) {
renameSync(replacementPath, rolloutPath)
}
}
})
expect(committedKeys).toBe(recordCount)
expect(verified.sessions.map(({ sessionId, eventCount }) => [sessionId, eventCount])).toEqual(
[['verified-session', recordCount]]
)
expect(verified.size).toBe(Number(originalStat.size))
expect(verified.physicalFileId).toBe(
originalStat.ino === 0n ? null : `${originalStat.dev}:${originalStat.ino}`
)
expect(verified.parseResumeState !== null).toBe(recordCount === 100)
const refreshed = await parseCodexUsageFile(
rolloutPath,
() => null,
verified.parseResumeState
? { resume: { state: verified.parseResumeState, previous: verified } }
: {}
)
expect(
refreshed.sessions.map(({ sessionId, eventCount }) => [sessionId, eventCount])
).toEqual([['replacement-session', 2]])
expect(closedHandles.size).toBe(openedHandles.length)
}
)
it('keeps the descriptor closed when publishing ownership fails', async () => {
const rolloutPath = join(directory, 'rollout.jsonl')
writeFileSync(rolloutPath, rollout('failed-claims', 3))
await expect(
parseCodexUsageFile(rolloutPath, () => null, {
commitEventKey: () => {
expect(closedHandles.size).toBe(openedHandles.length)
throw new Error('ownership publication failed')
}
})
).rejects.toThrow('ownership publication failed')
expect(openedHandles).toHaveLength(1)
expect(closedHandles.size).toBe(1)
expect(openedHandles[0]?.fd).toBe(-1)
})
it('publishes no ownership if closing the verified descriptor fails', async () => {
const rolloutPath = join(directory, 'rollout.jsonl')
writeFileSync(rolloutPath, rollout('close-failure', 3))
const commitEventKey = vi.fn()
failNextClose.current = true
await expect(parseCodexUsageFile(rolloutPath, () => null, { commitEventKey })).rejects.toThrow(
'descriptor close failed'
)
expect(commitEventKey).not.toHaveBeenCalled()
expect(openedHandles).toHaveLength(1)
expect(closedHandles.size).toBe(1)
expect(openedHandles[0]?.fd).toBe(-1)
})
})
+109 -100
View File
@@ -1,6 +1,11 @@
import { basename } from 'node:path'
import { stat } from 'node:fs/promises'
import { readJsonlFileSnapshot, type JsonlFileSnapshot } from '../usage/jsonl-file-snapshot'
import { readJsonlLinesFromOffset } from '../usage/jsonl-line-offsets'
import {
openJsonlFileReader,
validateJsonlFileReader,
jsonlPhysicalFileId
} from '../usage/jsonl-file-checkpoint'
import { attributeCodexUsageEvent } from './codex-usage-event-attribution'
import type { UsageWorktreeResolver } from '../usage/usage-worktree-resolver'
import { parseCodexUsageRecord, type CodexUsageParseContext } from './codex-usage-record-parser'
@@ -10,7 +15,6 @@ import {
resolveCodexRolloutResume
} from './codex-rollout-resume-state'
import type {
CodexUsageAttributedEvent,
CodexUsageDailyAggregate,
CodexUsageParseResumeState,
CodexUsagePersistedFile,
@@ -24,17 +28,24 @@ const { finalizeSessions, mergeSessions, mergeDailyAggregates, sortDailyAggregat
export type CodexRolloutParseOptions = {
/** Suffix-only parse for a diverged legacy copied-session bridge. */
legacySourceSkipBytes?: number
claimEventKey?: (eventKey: string) => boolean
canClaimEventKey?: (eventKey: string) => boolean
commitEventKey?: (eventKey: string) => void
/** Resume point verified by the caller, with the cached projection to extend. */
resume?: { state: CodexUsageParseResumeState; previous: CodexUsagePersistedFile }
}
export async function getProcessedFileInfo(filePath: string): Promise<CodexUsageProcessedFile> {
const fileStat = await stat(filePath)
const fileStat = await readJsonlFileSnapshot(filePath)
return processedFileInfo(filePath, fileStat)
}
function processedFileInfo(filePath: string, fileStat: JsonlFileSnapshot): CodexUsageProcessedFile {
return {
path: filePath,
mtimeMs: fileStat.mtimeMs,
size: fileStat.size
size: fileStat.size,
ctimeMs: fileStat.ctimeMs,
physicalFileId: jsonlPhysicalFileId(fileStat)
}
}
@@ -86,106 +97,104 @@ export async function parseCodexUsageFile(
resolveWorktree: UsageWorktreeResolver,
options: CodexRolloutParseOptions = {}
): Promise<CodexUsagePersistedFile> {
// Why: the caller verified this resume point while walking the directory, and
// every file discovered or parsed since then has run in between. Re-verify
// here, against the file about to be read, or a rollout replaced in that gap
// gets the cached session id, cwd, model and running totals stitched onto an
// unrelated file's records — and `processedFile` below re-stats to the new
// size, so the reuse path then freezes the corrupted projection.
if (
options.resume &&
(await resolveCodexRolloutResume(filePath, options.resume.previous)) === null
) {
return parseCodexUsageFile(filePath, resolveWorktree, { ...options, resume: undefined })
}
let parseOptions = options
for (;;) {
const reader = await openJsonlFileReader(filePath)
try {
// Verification, parsing and checkpointing must all read the same opened file.
if (
parseOptions.resume &&
(await resolveCodexRolloutResume(filePath, parseOptions.resume.previous, reader)) === null
) {
parseOptions = { ...parseOptions, resume: undefined }
continue
}
const processedFile = await getProcessedFileInfo(filePath)
const legacySourceSkipBytes = options.legacySourceSkipBytes ?? 0
const startOffset = options.resume?.state.parsedBytes ?? legacySourceSkipBytes
const context = createParseContext(filePath, options)
const legacySourceSkipBytes = parseOptions.legacySourceSkipBytes ?? 0
const startOffset = parseOptions.resume?.state.parsedBytes ?? legacySourceSkipBytes
const context = createParseContext(filePath, parseOptions)
const accumulator = codexUsageAggregation.createAccumulator()
const ownedEventKeys = new Set<string>()
let hasDeferredClaims = false
let parsedBytes = startOffset
let resumeContext = context
let partialTailProducedEvent = false
const events: CodexUsageAttributedEvent[] = []
const ownedEventKeys = new Set<string>()
let hasDeferredClaims = false
let parsedBytes = startOffset
// Points at the context as of `parsedBytes`, which excludes a partial tail.
let resumeContext = context
let partialTailProducedEvent = false
for await (const { line, endOffset, terminated } of readJsonlLinesFromOffset(
filePath,
startOffset
)) {
if (!terminated) {
// Only the final fragment can be unterminated, and the next scan re-reads
// it, so its context edits must not leak into the persisted resume point.
resumeContext = { ...context }
}
const parsed = parseCodexUsageRecord(line, context)
if (terminated) {
parsedBytes = endOffset
} else if (parsed) {
partialTailProducedEvent = true
}
if (!parsed) {
continue
}
// Why: fork/resume rollouts start with a copied prefix of the parent file.
// Events another file already owns are dropped here, but the record still
// advanced context.previousTotals above, so later deltas stay correct.
if (options.claimEventKey && !options.claimEventKey(parsed.eventKey)) {
hasDeferredClaims = true
continue
}
ownedEventKeys.add(parsed.eventKey)
const attributed = await attributeCodexUsageEvent(parsed, resolveWorktree)
if (attributed) {
events.push(attributed)
}
}
// A counted-but-unterminated tail would be counted again on resume, and a
// legacy suffix offset is recomputed per scan, so neither may be resumed. A
// prefix under the resumable floor is turned away by the builder itself.
const resumeStateSuppressed = partialTailProducedEvent || legacySourceSkipBytes > 0
const parseResumeState = resumeStateSuppressed
? null
: await buildCodexRolloutResumeState(
for await (const { line, endOffset, terminated } of readJsonlLinesFromOffset(
filePath,
parsedBytes,
resumeContext,
// Already verified against the file at the top of this scan.
options.resume?.state.headDigest ?? null
)
startOffset,
reader
)) {
if (!terminated) {
// The next scan rereads this tail, including its context changes.
resumeContext = { ...context }
}
const parsed = parseCodexUsageRecord(line, context)
if (terminated) {
parsedBytes = endOffset
} else if (parsed) {
partialTailProducedEvent = true
}
if (!parsed) {
continue
}
// Deferred records still advance cumulative totals without retaining their copies.
if (parseOptions.canClaimEventKey && !parseOptions.canClaimEventKey(parsed.eventKey)) {
hasDeferredClaims = true
continue
}
ownedEventKeys.add(parsed.eventKey)
const attributed = await attributeCodexUsageEvent(parsed, resolveWorktree)
if (attributed) {
accumulator.add(attributed)
}
}
// Why: a resume point only exists past the resumable floor and `parsedBytes`
// only grows, so the builder's other null — a prefix too short to be worth
// resuming — is unreachable here and this null means a short read: the file
// shrank past the prefix this parse merged history for, after the
// re-verification above and during the read. `processedFile` already re-stat'd
// to the smaller size, so persisting that pair would let the next scan reuse a
// pre-truncation total forever. An unterminated tail proves the file still
// runs past the resume offset, so it cannot be this case.
if (options.resume && !resumeStateSuppressed && parseResumeState === null) {
return parseCodexUsageFile(filePath, resolveWorktree, { ...options, resume: undefined })
}
// A counted unterminated row would be counted again from the committed offset.
const resumeStateSuppressed = partialTailProducedEvent || legacySourceSkipBytes > 0
const parseResumeState = resumeStateSuppressed
? null
: await buildCodexRolloutResumeState(
filePath,
parsedBytes,
resumeContext,
parseOptions.resume?.state.headDigest ?? null,
reader
)
const appended = codexUsageAggregation.aggregate(events)
const previous = options.resume?.previous
if (!previous) {
return {
...processedFile,
...appended,
ownedEventKeys: [...ownedEventKeys],
hasDeferredClaims,
parseResumeState
if (
!(await validateJsonlFileReader(reader, parseOptions.resume?.state)) ||
(parseOptions.resume && !resumeStateSuppressed && parseResumeState === null)
) {
parseOptions = { ...parseOptions, resume: undefined }
continue
}
await reader.handle.close()
// Discarded reads cannot claim keys that would hide records in another file.
for (const eventKey of ownedEventKeys) {
parseOptions.commitEventKey?.(eventKey)
}
const processedFile = processedFileInfo(filePath, reader.stats)
const appended = accumulator.finalize()
const previous = parseOptions.resume?.previous
const committedKeys = previous ? new Set(previous.ownedEventKeys) : ownedEventKeys
if (previous) {
for (const key of ownedEventKeys) {
committedKeys.add(key)
}
}
return {
...processedFile,
...(previous ? mergeRolloutProjections(previous, appended) : appended),
ownedEventKeys: [...committedKeys],
hasDeferredClaims: previous?.hasDeferredClaims || hasDeferredClaims,
parseResumeState
}
} finally {
await reader.handle.close()
}
}
return {
...processedFile,
...mergeRolloutProjections(previous, appended),
ownedEventKeys: [...new Set([...previous.ownedEventKeys, ...ownedEventKeys])],
hasDeferredClaims: previous.hasDeferredClaims || hasDeferredClaims,
parseResumeState
}
}
@@ -1,183 +1,46 @@
/**
* Decides whether a grown rollout can be parsed from where the last scan
* stopped. A wrong answer here silently corrupts usage totals, so every check
* fails closed: anything unproven falls back to a full reparse.
*/
import { createReadStream } from 'node:fs'
import { stat } from 'node:fs/promises'
import { createHash } from 'node:crypto'
import type { CodexUsageParseContext } from './codex-usage-record-parser'
import type { CodexUsageParseResumeState, CodexUsagePersistedFile } from './types'
import {
buildJsonlFileCheckpoint,
resolveJsonlFileCheckpoint,
type JsonlFileReader
} from '../usage/jsonl-file-checkpoint'
/** Bytes hashed immediately before the resume offset. Large enough to span a
* whole token_count record, small enough that verifying it is free next to
* re-reading a multi-megabyte rollout. */
const BOUNDARY_WINDOW_BYTES = 4096
/** Bytes hashed at the very start of the parsed prefix. The boundary window can
* only speak for the bytes next to the resume offset, so without this a
* same-length rewrite that leaves the tail intact resumes over changed
* history. Every realistic rotation or rewrite of a rollout replaces the
* leading session_meta line, which lands in this window. */
const HEAD_WINDOW_BYTES = 4096
/**
* Shortest prefix worth resuming over. A resumed scan verifies the prefix
* twice — once when the scanner plans the resume, once against the file it is
* about to read — and then records the moved boundary, so it pays five windows
* whatever the file size. A cold reparse pays the prefix itself plus the two
* windows it records. Below this length reading the whole file is cheaper, and
* for a prefix short enough that the two windows overlap it is far cheaper,
* because each verification then rehashes the entire prefix.
*
* Measured on the byte oracle: a 12,191 B prefix costs 21,234 B resumed against
* 21,137 B cold; at 13,699 B it is 21,234 B against 22,645 B.
*/
export const MIN_RESUMABLE_PREFIX_BYTES = 3 * BOUNDARY_WINDOW_BYTES
/** Below the floor the two windows could also overlap, so every offset that
* reaches the digest reads is guaranteed to give them a disjoint layout. */
export function isResumablePrefixLength(parsedBytes: number): boolean {
return parsedBytes > MIN_RESUMABLE_PREFIX_BYTES
}
async function readWindowDigest(
filePath: string,
start: number,
endExclusive: number
): Promise<string | null> {
const expectedBytes = endExclusive - start
const hash = createHash('sha256')
let readBytes = 0
const stream = createReadStream(filePath, { start, end: endExclusive - 1 })
for await (const chunk of stream) {
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
hash.update(buffer)
readBytes += buffer.length
}
// A short read means the file no longer reaches the offset we recorded.
return readBytes === expectedBytes ? `${expectedBytes}:${hash.digest('hex')}` : null
}
type PrefixDigests = { headDigest: string; boundaryDigest: string }
/**
* Both window digests for a prefix of `parsedBytes`, or null if the file no
* longer reaches that offset. The window layout is a pure function of
* `parsedBytes`, so a later verification hashes exactly the same ranges. Only
* called for a prefix past `MIN_RESUMABLE_PREFIX_BYTES`, so the two windows are
* always disjoint and always their full size.
*
* `carriedHeadDigest` lets a caller that already verified the head window this
* scan skip re-reading those bytes. Deferring to it is self-correcting: if the
* head did change after that read, the next scan compares against the carried
* value and falls back to a full reparse.
*/
async function readPrefixDigests(
filePath: string,
parsedBytes: number,
carriedHeadDigest: string | null = null
): Promise<PrefixDigests | null> {
const headDigest = carriedHeadDigest?.startsWith(`${HEAD_WINDOW_BYTES}:`)
? carriedHeadDigest
: await readWindowDigest(filePath, 0, HEAD_WINDOW_BYTES)
if (headDigest === null) {
return null
}
const boundaryDigest = await readWindowDigest(
filePath,
parsedBytes - BOUNDARY_WINDOW_BYTES,
parsedBytes
)
return boundaryDigest === null ? null : { headDigest, boundaryDigest }
}
async function readPhysicalFileId(filePath: string): Promise<string | null> {
try {
const fileStat = await stat(filePath)
return fileStat.ino === 0 ? null : `${fileStat.dev}:${fileStat.ino}`
} catch {
return null
}
}
function isUsableResumeState(
resume: CodexUsageParseResumeState | null | undefined
): resume is CodexUsageParseResumeState {
return (
resume != null &&
Number.isInteger(resume.parsedBytes) &&
resume.parsedBytes >= 0 &&
typeof resume.boundaryDigest === 'string' &&
// State persisted before the head window existed cannot be verified.
typeof resume.headDigest === 'string' &&
// Caches written before the floor existed can hold a shorter prefix.
isResumablePrefixLength(resume.parsedBytes) &&
typeof resume.sessionId === 'string'
)
}
export { MIN_RESUMABLE_PREFIX_BYTES, isResumablePrefixLength } from '../usage/jsonl-file-checkpoint'
export async function buildCodexRolloutResumeState(
filePath: string,
parsedBytes: number,
context: CodexUsageParseContext,
verifiedHeadDigest: string | null = null
verifiedHeadDigest: string | null = null,
reader?: JsonlFileReader
): Promise<CodexUsageParseResumeState | null> {
if (!isResumablePrefixLength(parsedBytes)) {
return null
}
const digests = await readPrefixDigests(filePath, parsedBytes, verifiedHeadDigest)
if (digests === null) {
return null
}
return {
const checkpoint = await buildJsonlFileCheckpoint(
filePath,
parsedBytes,
boundaryDigest: digests.boundaryDigest,
headDigest: digests.headDigest,
physicalFileId: await readPhysicalFileId(filePath),
sessionId: context.sessionId,
sessionCwd: context.sessionCwd,
currentCwd: context.currentCwd,
currentModel: context.currentModel,
previousTotals: context.previousTotals
}
verifiedHeadDigest,
reader
)
return checkpoint === null
? null
: {
...checkpoint,
sessionId: context.sessionId,
sessionCwd: context.sessionCwd,
currentCwd: context.currentCwd,
currentModel: context.currentModel,
previousTotals: context.previousTotals
}
}
/**
* Returns the resume point only when the recorded prefix still looks like the
* file's prefix. Deliberately does not consult any timestamp: filesystems vary
* in clock granularity, so a rewrite can land under the mtime, ctime or
* birthtime the cache already holds. `physicalFileId` is a cheap extra catch
* rather than a rotation check — ext4 and overlayfs hand a recreated path the
* inode the old file freed, so it only fires on filesystems that allocate a
* fresh one. Truncation needs no separate check: a file that no longer reaches
* the offset cannot produce the recorded digests.
*
* Bounded by design, so for a prefix larger than the two windows together this
* cannot prove every byte between them is intact.
*/
export async function resolveCodexRolloutResume(
filePath: string,
previous: CodexUsagePersistedFile | undefined
previous: CodexUsagePersistedFile | undefined,
reader?: JsonlFileReader
): Promise<CodexUsageParseResumeState | null> {
const resume = previous?.parseResumeState
if (!isUsableResumeState(resume)) {
if (!resume || typeof resume.sessionId !== 'string') {
return null
}
const physicalFileId = await readPhysicalFileId(filePath)
if (
resume.physicalFileId !== null &&
physicalFileId !== null &&
resume.physicalFileId !== physicalFileId
) {
return null
}
const digests = await readPrefixDigests(filePath, resume.parsedBytes)
if (digests === null) {
return null
}
return digests.boundaryDigest === resume.boundaryDigest &&
digests.headDigest === resume.headDigest
? resume
: null
return (await resolveJsonlFileCheckpoint(filePath, resume, reader)) === null ? null : resume
}
@@ -5,6 +5,7 @@ import { getOrcaManagedCodexHomePath, getSystemCodexHomePath } from '../codex/co
import { getCodexAccountHomeSessionDirectories } from '../codex/codex-account-home-discovery'
import { getLegacyCopiedCodexSessionBridgeScanPreference } from '../codex/codex-session-bridge'
import { normalizeFsPath } from '../usage/usage-path-comparison'
import { isDefinitiveAbsence } from '../../shared/definitive-filesystem-absence'
const YIELD_EVERY_DISCOVERY_ENTRIES = 100
@@ -25,7 +26,12 @@ async function walkJsonlFiles(
dirPath: string,
progress: { entriesVisited: number } = { entriesVisited: 0 }
): Promise<string[]> {
const entries = await readdir(dirPath, { withFileTypes: true })
const entries = await readdir(dirPath, { withFileTypes: true }).catch((error: unknown) => {
if (isDefinitiveAbsence(error)) {
return []
}
throw error
})
const files: string[] = []
for (const entry of entries) {
@@ -79,11 +85,7 @@ function hasLegacyCopiedSessionBridgeMarkers(): boolean {
export async function listCodexSessionFiles(): Promise<string[]> {
const files: string[] = []
for (const dirPath of getCodexSessionDirectories()) {
try {
appendDiscoveredFiles(files, await walkJsonlFiles(dirPath))
} catch {
// Missing or unreadable history in one home should not hide the other.
}
appendDiscoveredFiles(files, await walkJsonlFiles(dirPath))
}
return dedupeCodexSessionFileAliases(files, hasLegacyCopiedSessionBridgeMarkers())
}
@@ -0,0 +1,333 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
appendFileSync,
mkdirSync,
mkdtempSync,
renameSync,
rmSync,
statSync,
writeFileSync
} from 'node:fs'
import type * as NodeFsPromises from 'node:fs/promises'
import type * as NodeOs from 'node:os'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
const { homedirMock, getPathMock, parseReads, handles, mutation } = vi.hoisted(() => {
const parseReads: { path: string; start: number; bytes: number }[] = []
const mutation: {
beforeParse: ((path: string) => void) | null
duringParse: ((path: string) => void) | null
endBeforeEOF: number | null
zeroDescriptorRead: boolean
} = {
beforeParse: null,
duringParse: null,
endBeforeEOF: null,
zeroDescriptorRead: false
}
return {
homedirMock: vi.fn<() => string>(),
getPathMock: vi.fn<(name: string) => string>(),
parseReads,
handles: { opened: 0, closed: 0 },
mutation
}
})
vi.mock('electron', () => ({ app: { getPath: getPathMock } }))
vi.mock('node:os', async () => ({
...(await vi.importActual<typeof NodeOs>('node:os')),
homedir: homedirMock
}))
vi.mock('node:fs/promises', async () => {
const actual = await vi.importActual<typeof NodeFsPromises>('node:fs/promises')
return {
...actual,
open: async (...args: Parameters<typeof actual.open>) => {
const handle = await actual.open(...args)
handles.opened++
const createStream = handle.createReadStream.bind(handle)
const readDescriptor = handle.read.bind(handle)
const close = handle.close.bind(handle)
vi.spyOn(handle, 'read').mockImplementation(async (...readArgs) => {
const result = await readDescriptor(...readArgs)
if (mutation.zeroDescriptorRead) {
mutation.zeroDescriptorRead = false
return { ...result, bytesRead: 0 }
}
return result
})
let closed = false
vi.spyOn(handle, 'close').mockImplementation(async () => {
await close()
if (!closed) {
closed = true
handles.closed++
}
})
vi.spyOn(handle, 'createReadStream').mockImplementation((options) => {
const start = options?.start ?? 0
const end = options?.end
const parseRead = end !== undefined && end - start + 1 !== 4096
const filePath = String(args[0])
if (parseRead) {
mutation.beforeParse?.(filePath)
}
const forcedEnd = parseRead ? mutation.endBeforeEOF : null
if (forcedEnd !== null) {
mutation.endBeforeEOF = null
}
const stream = createStream(forcedEnd === null ? options : { ...options, end: forcedEnd })
if (parseRead) {
const read = { path: filePath, start, bytes: 0 }
parseReads.push(read)
stream.on('data', (chunk: string | Buffer) => {
read.bytes += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length
})
stream.once('data', () => mutation.duringParse?.(filePath))
}
return stream
})
return handle
}
}
})
import { parseCodexUsageFile } from './codex-rollout-file-parse'
import { scanCodexUsageFiles } from './scanner'
let root: string
let rolloutPath: string
beforeEach(() => {
root = mkdtempSync(join(tmpdir(), 'orca-codex-checkpoint-races-'))
homedirMock.mockReturnValue(root)
getPathMock.mockReturnValue(join(root, 'user-data'))
vi.stubEnv('ORCA_USER_DATA_PATH', join(root, 'user-data'))
vi.stubEnv('XDG_CONFIG_HOME', join(root, '.config'))
vi.stubEnv('CODEX_HOME', join(root, '.codex'))
const sessions = join(root, '.codex', 'sessions')
mkdirSync(sessions, { recursive: true })
rolloutPath = join(sessions, 'rollout.jsonl')
parseReads.length = 0
handles.opened = 0
handles.closed = 0
mutation.beforeParse = null
mutation.duringParse = null
mutation.endBeforeEOF = null
mutation.zeroDescriptorRead = false
})
afterEach(() => {
rmSync(root, { recursive: true, force: true })
vi.unstubAllEnvs()
expect(handles.closed).toBe(handles.opened)
})
function record(index: number, baseline: number): string {
return `${JSON.stringify({
timestamp: `2026-05-26T12:${String(index % 60).padStart(2, '0')}:00.000Z`,
type: 'event_msg',
payload: {
type: 'token_count',
info: {
model: 'gpt-5-codex',
last_token_usage: { input_tokens: 1, total_tokens: 1 },
total_token_usage: {
input_tokens: baseline + index + 1,
total_tokens: baseline + index + 1
}
}
}
})}\n`
}
function records(from: number, to: number, baseline = 1000): string {
return Array.from({ length: to - from }, (_, index) => record(from + index, baseline)).join('')
}
function transcript(sessionId: string, count: number, baseline = 1000): string {
const metadata = `${JSON.stringify({ type: 'session_meta', payload: { id: sessionId, cwd: root } })}\n`
const padding = `${JSON.stringify({ type: 'tool-result', content: 'x'.repeat(16_384) })}\n`
return metadata + padding + records(0, count, baseline)
}
function offsets(): number[] {
return parseReads.filter((read) => read.path === rolloutPath).map((read) => read.start)
}
describe('Codex checkpoint file races', () => {
it.each(['rewrite', 'rotate'])(
'discards a %s after verification before reading the suffix',
async (kind) => {
writeFileSync(rolloutPath, transcript('session-old', 40))
const first = await scanCodexUsageFiles([], [])
const offset = first.processedFiles[0]?.parseResumeState?.parsedBytes ?? 0
expect(offset).toBeGreaterThan(12_288)
appendFileSync(rolloutPath, records(40, 42))
const replacement = transcript('session-new', 60, 2000)
mutation.beforeParse = (filePath) => {
mutation.beforeParse = null
if (kind === 'rotate') {
const replacementPath = join(root, 'replacement.jsonl')
writeFileSync(replacementPath, replacement)
renameSync(replacementPath, filePath)
} else {
writeFileSync(filePath, replacement)
}
}
parseReads.length = 0
const second = await scanCodexUsageFiles([], first.processedFiles)
expect(mutation.beforeParse).toBeNull()
expect(offsets()).toEqual([offset, 0])
expect(parseReads.every((read) => read.bytes > 0)).toBe(true)
const cold = await scanCodexUsageFiles([], [])
expect(second.sessions).toEqual(cold.sessions)
expect(second.dailyAggregates).toEqual(cold.dailyAggregates)
expect(second.sessions[0]?.sessionId).toBe('session-new')
const third = await scanCodexUsageFiles([], second.processedFiles)
expect(third.processedFiles[0]).toBe(second.processedFiles[0])
expect(third.sessions).toEqual(cold.sessions)
}
)
it('discards cold-read events before claiming their keys when the file changes during parsing', async () => {
writeFileSync(rolloutPath, transcript('session-old', 40))
mutation.duringParse = (filePath) => {
mutation.duringParse = null
writeFileSync(filePath, transcript('session-new', 60, 2000))
}
const commitEventKey = vi.fn()
const parsed = await parseCodexUsageFile(rolloutPath, () => null, { commitEventKey })
expect(offsets()).toEqual([0, 0])
expect(parsed.sessions[0]?.sessionId).toBe('session-new')
expect(parsed.sessions[0]?.eventCount).toBe(60)
expect(commitEventKey).toHaveBeenCalledTimes(60)
const cold = await parseCodexUsageFile(rolloutPath, () => null)
expect(parsed.ownedEventKeys).toEqual(cold.ownedEventKeys)
expect(parsed.sessions).toEqual(cold.sessions)
})
it('leaves an append after the captured EOF for the next scan', async () => {
writeFileSync(rolloutPath, transcript('session-stable', 40))
const capturedSize = statSync(rolloutPath).size
mutation.duringParse = (filePath) => {
mutation.duringParse = null
appendFileSync(filePath, records(40, 41))
}
const first = await scanCodexUsageFiles([], [])
expect(first.sessions[0]?.eventCount).toBe(40)
expect(first.processedFiles[0]?.size).toBe(capturedSize)
expect(first.processedFiles[0]?.parseResumeState?.parsedBytes).toBe(capturedSize)
const second = await scanCodexUsageFiles([], first.processedFiles)
expect(second.sessions[0]?.eventCount).toBe(41)
const cold = await scanCodexUsageFiles([], [])
expect(second.sessions).toEqual(cold.sessions)
expect(second.dailyAggregates).toEqual(cold.dailyAggregates)
})
it('releases old owner claims and lets an unchanged deferred fork reclaim them', async () => {
const prefix = transcript('session-old', 40)
const forkPath = join(dirname(rolloutPath), 'zzzz-fork.jsonl')
writeFileSync(rolloutPath, prefix)
writeFileSync(forkPath, prefix + records(40, 41))
const first = await scanCodexUsageFiles([], [])
const fork = first.processedFiles.find((file) => file.path === forkPath)
expect(fork?.hasDeferredClaims).toBe(true)
expect(fork?.ownedEventKeys).toHaveLength(1)
appendFileSync(rolloutPath, records(40, 42))
mutation.beforeParse = (filePath) => {
if (filePath === rolloutPath) {
mutation.beforeParse = null
writeFileSync(filePath, transcript('session-new', 60, 2000))
}
}
parseReads.length = 0
const second = await scanCodexUsageFiles([], first.processedFiles)
expect(parseReads.filter((read) => read.path === forkPath).map((read) => read.start)).toEqual([
0
])
expect(
second.processedFiles.find((file) => file.path === forkPath)?.ownedEventKeys
).toHaveLength(41)
const cold = await scanCodexUsageFiles([], [])
expect(second.sessions).toEqual(cold.sessions)
expect(second.dailyAggregates).toEqual(cold.dailyAggregates)
const third = await scanCodexUsageFiles([], second.processedFiles)
expect(third.sessions).toEqual(cold.sessions)
})
it('keeps an unchanged deferred fork cached when a short owner merely appends', async () => {
const prefix = records(0, 3)
const forkPath = join(dirname(rolloutPath), 'zzzz-fork.jsonl')
writeFileSync(rolloutPath, prefix)
writeFileSync(forkPath, prefix + records(3, 4))
const first = await scanCodexUsageFiles([], [])
const fork = first.processedFiles.find((file) => file.path === forkPath)
expect(fork?.hasDeferredClaims).toBe(true)
expect(first.processedFiles[0]?.parseResumeState).toBeNull()
appendFileSync(rolloutPath, records(4, 5))
parseReads.length = 0
const second = await scanCodexUsageFiles([], first.processedFiles)
expect(second.processedFiles.find((file) => file.path === forkPath)).toBe(fork)
expect(parseReads.some((read) => read.path === forkPath)).toBe(false)
expect(second.dailyAggregates.reduce((sum, row) => sum + row.totalTokens, 0)).toBe(5)
})
it('rejects an unchanged file stream that stops early before claiming partial history', async () => {
writeFileSync(rolloutPath, transcript('session-stable', 40))
mutation.endBeforeEOF = Buffer.byteLength(transcript('session-stable', 10)) - 1
const commitEventKey = vi.fn()
await expect(parseCodexUsageFile(rolloutPath, () => null, { commitEventKey })).rejects.toThrow(
'JSONL snapshot stream ended before the unchanged file boundary.'
)
expect(mutation.endBeforeEOF).toBeNull()
expect(offsets()).toEqual([0])
expect(commitEventKey).not.toHaveBeenCalled()
const cold = await scanCodexUsageFiles([], [])
expect(cold.sessions[0]?.eventCount).toBe(40)
})
it('leaves the previous scan intact when an unchanged suffix stream stops early', async () => {
const prefix = transcript('session-stable', 40)
writeFileSync(rolloutPath, prefix)
const previous = await scanCodexUsageFiles([], [])
appendFileSync(rolloutPath, records(40, 42))
mutation.endBeforeEOF = Buffer.byteLength(prefix + records(40, 41)) - 1
parseReads.length = 0
await expect(scanCodexUsageFiles([], previous.processedFiles)).rejects.toThrow(
'JSONL snapshot stream ended before the unchanged file boundary.'
)
expect(mutation.endBeforeEOF).toBeNull()
expect(offsets()).toEqual([Buffer.byteLength(prefix)])
expect(previous.sessions[0]?.eventCount).toBe(40)
expect(previous.processedFiles[0]?.size).toBe(Buffer.byteLength(prefix))
const recovered = await scanCodexUsageFiles([], previous.processedFiles)
expect(recovered.sessions[0]?.eventCount).toBe(42)
})
it('rejects a stable zero-byte read of the skipped prefix in a short legacy source', async () => {
writeFileSync(rolloutPath, records(0, 3))
mutation.zeroDescriptorRead = true
const commitEventKey = vi.fn()
const legacySourceSkipBytes = Buffer.byteLength(records(0, 1))
await expect(
parseCodexUsageFile(rolloutPath, () => null, { legacySourceSkipBytes, commitEventKey })
).rejects.toThrow('JSONL snapshot read ended before the unchanged file boundary.')
expect(mutation.zeroDescriptorRead).toBe(false)
expect(offsets()).toEqual([])
expect(commitEventKey).not.toHaveBeenCalled()
const recovered = await parseCodexUsageFile(rolloutPath, () => null, { legacySourceSkipBytes })
expect(recovered.sessions[0]?.eventCount).toBe(2)
})
})
@@ -0,0 +1,192 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
mkdirSync,
mkdtempSync,
readFileSync,
renameSync,
rmSync,
statSync,
utimesSync,
writeFileSync
} from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import type * as JsonlLineOffsets from '../usage/jsonl-line-offsets'
const fixture = vi.hoisted(() => {
const parseOffsets: number[] = []
return { root: '', parseOffsets }
})
vi.mock('../codex/codex-home-paths', () => ({
getOrcaManagedCodexHomePath: () => join(fixture.root, 'runtime'),
getSystemCodexHomePath: () => join(fixture.root, 'system')
}))
vi.mock('../codex/codex-account-home-discovery', () => ({
getCodexAccountHomeSessionDirectories: () => []
}))
vi.mock('../codex/codex-session-bridge', () => ({
getLegacyCopiedCodexSessionBridgeScanPreference: () => null
}))
vi.mock('../usage/jsonl-line-offsets', async () => {
const actual = await vi.importActual<typeof JsonlLineOffsets>('../usage/jsonl-line-offsets')
return {
...actual,
readJsonlLinesFromOffset: async function* (
...args: Parameters<typeof actual.readJsonlLinesFromOffset>
) {
fixture.parseOffsets.push(args[1])
yield* actual.readJsonlLinesFromOffset(...args)
}
}
})
import { scanCodexUsageFiles } from './scanner'
const preservedMtime = new Date('2026-05-26T12:00:00.000Z')
let rolloutPath: string
beforeEach(() => {
fixture.root = mkdtempSync(join(tmpdir(), 'orca-codex-file-generation-'))
const sessions = join(fixture.root, 'runtime', 'sessions')
mkdirSync(sessions, { recursive: true })
rolloutPath = join(sessions, 'rollout.jsonl')
fixture.parseOffsets.length = 0
})
afterEach(() => {
rmSync(fixture.root, { recursive: true, force: true })
})
function transcript(sessionId: string, count: number, changedIndex = -1): string {
const metadata = JSON.stringify({
type: 'session_meta',
payload: { id: sessionId, cwd: fixture.root }
})
const records = Array.from({ length: count }, (_, index) => {
const tokens = index === changedIndex ? 20 : 10
return JSON.stringify({
timestamp: new Date(preservedMtime.getTime() + index * 60_000).toISOString(),
type: 'event_msg',
payload: {
type: 'token_count',
info: {
model: 'gpt-5-codex',
last_token_usage: { input_tokens: tokens, total_tokens: tokens }
}
}
})
})
return `${[metadata, ...records].join('\n')}\n`
}
function writePreservingMtime(filePath: string, content: string): void {
writeFileSync(filePath, content)
utimesSync(filePath, preservedMtime, preservedMtime)
}
function totalTokens(rows: { totalTokens: number }[]): number {
return rows.reduce((sum, row) => sum + row.totalTokens, 0)
}
describe('Codex cached rollout generation', () => {
it.each([1, 80])(
'reparses a same-size preserved-mtime atomic replacement containing %i events',
async (count) => {
const original = transcript('session-old', count)
const replacement = transcript('session-new', count, Math.floor(count / 2))
expect(Buffer.byteLength(replacement)).toBe(Buffer.byteLength(original))
writePreservingMtime(rolloutPath, original)
const before = statSync(rolloutPath, { bigint: true })
const first = await scanCodexUsageFiles([], [])
expect(totalTokens(first.dailyAggregates)).toBe(count * 10)
expect(Boolean(first.processedFiles[0]?.parseResumeState)).toBe(count > 1)
const replacementPath = join(fixture.root, 'replacement.jsonl')
writePreservingMtime(replacementPath, replacement)
renameSync(replacementPath, rolloutPath)
const after = statSync(rolloutPath, { bigint: true })
expect(after.size).toBe(before.size)
expect(after.mtimeNs).toBe(before.mtimeNs)
if (before.ino !== 0n && after.ino !== 0n) {
expect(`${after.dev}:${after.ino}`).not.toBe(`${before.dev}:${before.ino}`)
}
fixture.parseOffsets.length = 0
const second = await scanCodexUsageFiles([], first.processedFiles)
expect(totalTokens(second.dailyAggregates)).toBe(count * 10 + 10)
expect(second.sessions[0]?.sessionId).toBe('session-new')
expect(fixture.parseOffsets).toEqual([0])
const cold = await scanCodexUsageFiles([], [])
expect(second.sessions).toEqual(cold.sessions)
expect(second.dailyAggregates).toEqual(cold.dailyAggregates)
fixture.parseOffsets.length = 0
const third = await scanCodexUsageFiles([], second.processedFiles)
expect(third.processedFiles[0]).toBe(second.processedFiles[0])
expect(fixture.parseOffsets).toEqual([])
}
)
it('cold-reparses a same-inode middle rewrite when size and mtime are preserved', async () => {
const original = transcript('session-stable', 80)
const replacement = transcript('session-stable', 80, 40)
expect(Buffer.byteLength(replacement)).toBe(Buffer.byteLength(original))
expect(replacement.slice(0, 4096)).toBe(original.slice(0, 4096))
expect(replacement.slice(-4096)).toBe(original.slice(-4096))
writePreservingMtime(rolloutPath, original)
const before = statSync(rolloutPath)
const first = await scanCodexUsageFiles([], [])
expect(totalTokens(first.dailyAggregates)).toBe(800)
expect(first.processedFiles[0]?.parseResumeState?.parsedBytes).toBe(before.size)
await new Promise((resolve) => setTimeout(resolve, 20))
writePreservingMtime(rolloutPath, replacement)
const after = statSync(rolloutPath)
expect(after.ino).toBe(before.ino)
expect(after.dev).toBe(before.dev)
expect(after.size).toBe(before.size)
expect(after.mtimeMs).toBe(before.mtimeMs)
expect(after.ctimeMs).not.toBe(before.ctimeMs)
expect(readFileSync(rolloutPath, 'utf8')).toBe(replacement)
fixture.parseOffsets.length = 0
const second = await scanCodexUsageFiles([], first.processedFiles)
expect(totalTokens(second.dailyAggregates)).toBe(810)
expect(fixture.parseOffsets).toEqual([0])
expect(second.processedFiles[0]?.ctimeMs).toBe(after.ctimeMs)
const cold = await scanCodexUsageFiles([], [])
expect(second.sessions).toEqual(cold.sessions)
expect(second.dailyAggregates).toEqual(cold.dailyAggregates)
})
it('cold-reparses legacy metadata once even when the old checkpoint remains valid', async () => {
writePreservingMtime(rolloutPath, transcript('session-legacy', 80))
const first = await scanCodexUsageFiles([], [])
expect(first.processedFiles[0]?.parseResumeState?.parsedBytes).toBeGreaterThan(12_288)
const legacy = first.processedFiles.map((file) => {
const oldFile = { ...file }
delete oldFile.physicalFileId
delete oldFile.ctimeMs
return oldFile
})
fixture.parseOffsets.length = 0
const second = await scanCodexUsageFiles([], legacy)
expect(fixture.parseOffsets).toEqual([0])
expect(second.sessions).toEqual(first.sessions)
expect(second.dailyAggregates).toEqual(first.dailyAggregates)
const stats = statSync(rolloutPath)
expect(second.processedFiles[0]?.physicalFileId).toBe(
stats.ino === 0 ? null : `${stats.dev}:${stats.ino}`
)
expect(second.processedFiles[0]?.ctimeMs).toBe(stats.ctimeMs)
fixture.parseOffsets.length = 0
const third = await scanCodexUsageFiles([], second.processedFiles)
expect(third.processedFiles[0]).toBe(second.processedFiles[0])
expect(fixture.parseOffsets).toEqual([])
})
})
@@ -31,6 +31,22 @@ vi.mock('node:fs/promises', async () => {
? Object.assign(stats, { dev: Number(identity.dev), ino: Number(identity.ino) })
: stats
},
open: async (...args: Parameters<typeof actual.open>) => {
const handle = await actual.open(...args)
const identity = fixture.identities.get(String(args[0]))
const stat = handle.stat.bind(handle)
vi.spyOn(handle, 'stat').mockImplementation(async (options) => {
if (options?.bigint) {
const stats = await stat({ bigint: true })
return identity ? Object.assign(stats, identity) : stats
}
const stats = await stat()
return identity
? Object.assign(stats, { dev: Number(identity.dev), ino: Number(identity.ino) })
: stats
})
return handle
},
realpath: async (filePath: PathLike) => {
if (fixture.realpathErrors.has(String(filePath))) {
throw new Error('Fixture realpath unavailable')
@@ -2,19 +2,22 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { tmpdir } from 'node:os'
import type * as NodeOs from 'node:os'
import type * as NodeFs from 'node:fs'
import type * as NodeFsPromises from 'node:fs/promises'
import type * as JsonlLineOffsets from '../usage/jsonl-line-offsets'
import { join } from 'node:path'
const { getPathMock, homedirMock, streamReads, onStreamOpen } = vi.hoisted(() => {
const streamReads: { path: string; bytes: number; start: number; bounded: boolean }[] = []
const { getPathMock, homedirMock, streamReads, onStreamOpen, parsingPaths } = vi.hoisted(() => {
const streamReads: { path: string; bytes: number; start: number; digest: boolean }[] = []
// Seam for mutating the tree mid-scan, between two files' parse reads.
const onStreamOpen: { current: ((path: string, bounded: boolean) => void) | null } = {
const onStreamOpen: { current: ((path: string, digest: boolean) => void) | null } = {
current: null
}
return {
getPathMock: vi.fn<(name: string) => string>(),
homedirMock: vi.fn<() => string>(),
streamReads,
onStreamOpen
onStreamOpen,
parsingPaths: new Set<string>()
}
})
@@ -44,16 +47,84 @@ vi.mock('node:fs', async () => {
const filePath = String(path)
const range = typeof options === 'object' && options !== null ? options : {}
const start = range.start ?? 0
const size = actual.statSync(filePath).size
const stop = range.end === undefined ? size : Math.min(size, range.end + 1)
streamReads.push({
path: filePath,
bytes: Math.max(0, stop - start),
start,
bounded: range.end !== undefined
const read = { path: filePath, bytes: 0, start, digest: !parsingPaths.has(filePath) }
streamReads.push(read)
onStreamOpen.current?.(filePath, read.digest)
const stream = actual.createReadStream(path, options)
stream.on('data', (chunk: string | Buffer) => {
read.bytes += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length
})
onStreamOpen.current?.(filePath, range.end !== undefined)
return actual.createReadStream(path, options)
return stream
}
}
})
vi.mock('node:fs/promises', async () => {
const actual = await vi.importActual<typeof NodeFsPromises>('node:fs/promises')
return {
...actual,
open: async (...args: Parameters<typeof actual.open>) => {
const handle = await actual.open(...args)
const createStream = handle.createReadStream.bind(handle)
const readWindow = handle.read.bind(handle)
let streamActive = false
vi.spyOn(handle, 'read').mockImplementation(async (...readArgs) => {
if (streamActive) {
return readWindow(...readArgs)
}
const filePath = String(args[0])
const digest = !parsingPaths.has(filePath)
onStreamOpen.current?.(filePath, digest)
const result = await readWindow(...readArgs)
const position = readArgs.at(-1)
streamReads.push({
path: filePath,
bytes: result.bytesRead,
start: typeof position === 'number' ? position : 0,
digest
})
return result
})
vi.spyOn(handle, 'createReadStream').mockImplementation((options) => {
const filePath = String(args[0])
const read = {
path: filePath,
bytes: 0,
start: options?.start ?? 0,
digest: !parsingPaths.has(filePath)
}
streamReads.push(read)
onStreamOpen.current?.(filePath, read.digest)
streamActive = true
const stream = createStream(options)
const finish = (): void => {
streamActive = false
}
stream.once('end', finish)
stream.once('close', finish)
stream.on('data', (chunk: string | Buffer) => {
read.bytes += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length
})
return stream
})
return handle
}
}
})
vi.mock('../usage/jsonl-line-offsets', async () => {
const actual = await vi.importActual<typeof JsonlLineOffsets>('../usage/jsonl-line-offsets')
return {
...actual,
readJsonlLinesFromOffset: async function* (
...args: Parameters<typeof actual.readJsonlLinesFromOffset>
) {
parsingPaths.add(args[0])
try {
yield* actual.readJsonlLinesFromOffset(...args)
} finally {
parsingPaths.delete(args[0])
}
}
}
})
@@ -62,7 +133,7 @@ import { mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync, appendFileSync
import { scanCodexUsageFiles } from './scanner'
import type { CodexUsagePersistedFile } from './types'
/** Mirrors BOUNDARY_WINDOW_BYTES in codex-rollout-resume-state.ts. */
/** Mirrors the checkpoint window in jsonl-file-checkpoint.ts. */
const BOUNDARY_WINDOW_BYTES = 4096
/** Enough records (~377 B each) to put a prefix past MIN_RESUMABLE_PREFIX_BYTES.
@@ -148,12 +219,10 @@ function bytesReadFor(filePath: string): number {
.reduce((total, entry) => total + entry.bytes, 0)
}
/** Offsets at which this file's parse reads opened, in order. Bounded reads are
* digest windows; an unbounded one at 0 is a full reparse and at the recorded
* offset is a resume, so this says exactly which path a scan took. */
/** Digest and parse streams both have end bounds; the iterator marks parse reads. */
function parseReadOffsets(filePath: string): number[] {
return streamReads
.filter((entry) => entry.path === filePath && !entry.bounded)
.filter((entry) => entry.path === filePath && !entry.digest)
.map((entry) => entry.start)
}
@@ -195,6 +264,7 @@ beforeEach(() => {
mkdirSync(sessionsDir, { recursive: true })
streamReads.length = 0
onStreamOpen.current = null
parsingPaths.clear()
})
afterEach(() => {
@@ -231,14 +301,13 @@ describe('scanCodexUsageFiles incremental append', () => {
expect(totalTokens(second.dailyAggregates)).toBe(202)
expect(second.sessions[0]?.eventCount).toBe(202)
// The defect: the scanner restarts at byte 0 and re-reads the whole file.
// The fix reads the appended bytes plus five bounded windows: a head and a
// boundary window to plan the resume, both again at the point of use so a
// rollout replaced in between cannot be stitched onto, then the moved
// boundary to record the new resume point.
// Eleven windows prove the planned prefix, opened snapshot and final checkpoint.
expect(reparsedFromStart(rolloutPath)).toBe(false)
expect(bytesReadFor(rolloutPath)).toBeLessThan(sizeBeforeAppend)
expect(bytesReadFor(rolloutPath)).toBeLessThanOrEqual(appendedBytes + 5 * BOUNDARY_WINDOW_BYTES)
expect(bytesReadFor(rolloutPath)).toBeGreaterThan(appendedBytes)
expect(bytesReadFor(rolloutPath)).toBeLessThanOrEqual(
appendedBytes + 11 * BOUNDARY_WINDOW_BYTES
)
})
it('carries cumulative token totals across the resume boundary', async () => {
@@ -379,10 +448,8 @@ describe('scanCodexUsageFiles incremental append', () => {
appendFileSync(rolloutPath, usageRecordRange(RESUMABLE_RECORDS, RESUMABLE_RECORDS + 2), 'utf-8')
const truncated = `${sessionMeta('session-shrinker')}${usageRecordRange(0, 5)}`
onStreamOpen.current = (path, bounded) => {
// Bounded reads are the digest windows; the unbounded one is the parse
// read, which opens after every check this scan is going to make.
if (path === rolloutPath && !bounded) {
onStreamOpen.current = (path, digest) => {
if (path === rolloutPath && !digest) {
onStreamOpen.current = null
writeFileSync(path, truncated, 'utf-8')
}
@@ -429,10 +496,10 @@ describe('scanCodexUsageFiles incremental append', () => {
const replacement = `${sessionMeta('session-other')}${usageRecordRange(200, 260)}`
// Past the recorded offset, so every digest window still reads its full size.
expect(Buffer.byteLength(replacement)).toBeGreaterThan(resumeOffset)
onStreamOpen.current = (path, bounded) => {
onStreamOpen.current = (path, digest) => {
// Lands while the earlier-sorted rollout is being parsed, which is after
// the scanner's discovery loop verified every file's prefix.
if (path === driverPath && !bounded) {
if (path === driverPath && !digest) {
onStreamOpen.current = null
writeFileSync(targetPath, replacement, 'utf-8')
}
+54 -9
View File
@@ -61,12 +61,19 @@ export async function scanCodexUsageFiles(
const previous = previousByPath.get(filePath)
// When an owner disappears, only deferred-claim files need reparse.
const mustReclaimDeferred = lostOwnerPath && previous?.hasDeferredClaims !== false
const hasGenerationMetadata =
previous &&
typeof previous.ctimeMs === 'number' &&
(typeof previous.physicalFileId === 'string' || previous.physicalFileId === null)
const canReuse =
!mustReclaimDeferred &&
legacySourceSkipBytes === 0 &&
previous &&
hasGenerationMetadata &&
previous.mtimeMs === fileInfo.mtimeMs &&
previous.size === fileInfo.size &&
previous.ctimeMs === fileInfo.ctimeMs &&
previous.physicalFileId === fileInfo.physicalFileId &&
Array.isArray(previous.ownedEventKeys) &&
typeof previous.hasDeferredClaims === 'boolean'
if (canReuse) {
@@ -75,7 +82,13 @@ export async function scanCodexUsageFiles(
// Why: rollouts are append-only and grow all day, so re-reading each one
// from byte 0 dominated scans (#20940). A reclaim or a legacy suffix
// offset still needs the whole file, so neither may resume.
if (!mustReclaimDeferred && legacySourceSkipBytes === 0 && previous) {
if (
!mustReclaimDeferred &&
legacySourceSkipBytes === 0 &&
previous &&
hasGenerationMetadata &&
fileInfo.size > previous.size
) {
const state = await resolveCodexRolloutResume(filePath, previous)
if (state) {
resumeByPath.set(filePath, { state, previous })
@@ -97,7 +110,7 @@ export async function scanCodexUsageFiles(
// stay deterministic.
const eventOwnerByKey = new Map<string, string>()
for (const filePath of files) {
const retained = reusedByPath.get(filePath) ?? resumeByPath.get(filePath)?.previous
const retained = previousByPath.get(filePath)
for (const eventKey of retained?.ownedEventKeys ?? []) {
// First retained claim wins so conflicting projections stay deterministic.
if (!eventOwnerByKey.has(eventKey)) {
@@ -107,20 +120,37 @@ export async function scanCodexUsageFiles(
}
const parsedByPath = new Map<string, CodexUsagePersistedFile>()
for (const [index, filePath] of pathsToParse.entries()) {
let lostOwnerKeys = lostOwnerPath
const parseAndPublish = async (
filePath: string,
resume?: CodexRolloutResumePlan
): Promise<void> => {
const previous =
parsedByPath.get(filePath) ?? reusedByPath.get(filePath) ?? previousByPath.get(filePath)
const processed = await parseCodexUsageFile(filePath, resolveWorktree, {
legacySourceSkipBytes: legacySourceSkipBytesByPath.get(filePath) ?? 0,
resume: resumeByPath.get(filePath),
claimEventKey: (eventKey) => {
resume,
canClaimEventKey: (eventKey) => {
const owner = eventOwnerByKey.get(eventKey)
if (owner !== undefined && owner !== filePath) {
return false
}
return owner === undefined || owner === filePath
},
commitEventKey: (eventKey) => {
eventOwnerByKey.set(eventKey, filePath)
return true
}
})
const currentKeys = new Set(processed.ownedEventKeys)
for (const key of previous?.ownedEventKeys ?? []) {
if (eventOwnerByKey.get(key) === filePath && !currentKeys.has(key)) {
eventOwnerByKey.delete(key)
lostOwnerKeys = true
}
}
reusedByPath.delete(filePath)
parsedByPath.set(filePath, processed)
}
for (const [index, filePath] of pathsToParse.entries()) {
await parseAndPublish(filePath, resumeByPath.get(filePath))
// Why: Codex session history can grow large, and scans run on the Electron
// main process. Yield regularly so opening Settings does not stall while
@@ -131,6 +161,21 @@ export async function scanCodexUsageFiles(
}
}
// A replaced owner can release records that an unchanged fork previously deferred.
while (lostOwnerKeys) {
lostOwnerKeys = false
for (const [index, filePath] of files.entries()) {
const current = parsedByPath.get(filePath) ?? reusedByPath.get(filePath)
if (current?.hasDeferredClaims) {
await parseAndPublish(filePath)
onFilesScanned?.(1)
}
if ((index + 1) % YIELD_EVERY_FILES === 0) {
await yieldToEventLoop()
}
}
}
const processedFiles: CodexUsagePersistedFile[] = []
const sessionsById = new Map<string, CodexUsageSession>()
const dailyByKey = new Map<string, CodexUsageDailyAggregate>()
@@ -0,0 +1,91 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
import type * as FsPromises from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
const state = vi.hoisted(() => ({ managed: '', system: '', blocked: '', code: '' }))
vi.mock('../codex/codex-home-paths', () => ({
getOrcaManagedCodexHomePath: () => state.managed,
getSystemCodexHomePath: () => state.system
}))
vi.mock('../codex/codex-account-home-discovery', () => ({
getCodexAccountHomeSessionDirectories: () => []
}))
vi.mock('../codex/codex-session-bridge', () => ({
getLegacyCopiedCodexSessionBridgeScanPreference: () => null
}))
vi.mock('node:fs/promises', async () => {
const actual = await vi.importActual<typeof FsPromises>('node:fs/promises')
return {
...actual,
readdir: async (...args: Parameters<typeof actual.readdir>) => {
if (String(args[0]) === state.blocked) {
throw Object.assign(new Error(`Cannot discover ${state.blocked}`), { code: state.code })
}
return actual.readdir(...args)
}
}
})
import { listCodexSessionFiles } from './codex-session-file-discovery'
let root: string
let sessions: string
beforeEach(async () => {
root = await mkdtemp(join(tmpdir(), 'orca-codex-discovery-error-'))
state.managed = join(root, 'managed')
state.system = join(root, 'system')
state.blocked = ''
state.code = ''
sessions = join(state.managed, 'sessions')
await mkdir(sessions, { recursive: true })
await mkdir(join(state.system, 'sessions'), { recursive: true })
})
afterEach(async () => {
await rm(root, { recursive: true, force: true })
})
describe('Codex history discovery failures', () => {
it.each(['EACCES', 'EPERM', 'EIO', 'UNKNOWN'])(
'rejects an inaccessible root (%s)',
async (code) => {
state.blocked = sessions
state.code = code
await expect(listCodexSessionFiles()).rejects.toMatchObject({ code })
}
)
it('rejects an unreadable child instead of publishing an incomplete history list', async () => {
await writeFile(join(sessions, 'kept.jsonl'), '')
state.blocked = join(sessions, 'inaccessible')
await mkdir(state.blocked)
state.code = 'EACCES'
await expect(listCodexSessionFiles()).rejects.toMatchObject({ code: 'EACCES' })
})
it.each(['ENOENT', 'ENOTDIR'])(
'keeps sibling files when a child disappears (%s)',
async (code) => {
const sibling = join(sessions, 'kept.jsonl')
await writeFile(sibling, '')
state.blocked = join(sessions, 'removed')
await mkdir(state.blocked)
state.code = code
await expect(listCodexSessionFiles()).resolves.toEqual([sibling])
}
)
it('accepts absent homes while preserving the available history', async () => {
const sibling = join(sessions, 'kept.jsonl')
await writeFile(sibling, '')
await rm(state.system, { recursive: true })
await expect(listCodexSessionFiles()).resolves.toEqual([sibling])
})
})
+3
View File
@@ -12,6 +12,9 @@ export type CodexUsageProcessedFile = {
path: string
mtimeMs: number
size: number
/** Absent in older caches; identifies the file that produced this projection. */
physicalFileId?: string | null
ctimeMs?: number
}
/** Everything needed to resume parsing a grown rollout where the last scan
+309
View File
@@ -0,0 +1,309 @@
import { createReadStream } from 'node:fs'
import { open, type FileHandle } from 'node:fs/promises'
import { createHash } from 'node:crypto'
import {
readJsonlFileSnapshot,
readJsonlHandleSnapshot,
readJsonlSnapshotRange,
releaseJsonlReadStream,
sameJsonlFileSnapshot,
verifyJsonlSnapshotContents,
type JsonlFileSnapshot
} from './jsonl-file-snapshot'
const CHECKPOINT_WINDOW_BYTES = 4096
// Below this measured crossover, verifying the prefix costs more than rereading it.
export const MIN_RESUMABLE_PREFIX_BYTES = 3 * CHECKPOINT_WINDOW_BYTES
export type JsonlFileCheckpoint = {
parsedBytes: number
boundaryDigest: string
headDigest: string
physicalFileId: string | null
}
export type JsonlFileReader = {
path: string
handle: FileHandle
stats: JsonlFileSnapshot
snapshotCheckpoint: JsonlFileCheckpoint | null
snapshotChunks: Buffer[] | null
snapshotBytes: number
streamStartOffset: number
streamBytes: number
onChunk: (chunk: Buffer) => void
prepareRead: (startOffset: number) => Promise<void>
}
export function isResumablePrefixLength(parsedBytes: number): boolean {
return parsedBytes > MIN_RESUMABLE_PREFIX_BYTES
}
async function readWindowDigest(
filePath: string,
start: number,
endExclusive: number,
reader?: JsonlFileReader
): Promise<string | null> {
const expectedBytes = endExclusive - start
const hash = createHash('sha256')
let readBytes = 0
if (expectedBytes === 0) {
return `0:${hash.digest('hex')}`
}
if (reader) {
const buffer = await readJsonlSnapshotRange(
filePath,
reader.handle,
reader.stats,
start,
endExclusive
)
if (!buffer) {
return null
}
hash.update(buffer)
return `${buffer.length}:${hash.digest('hex')}`
}
const range = { start, end: endExclusive - 1 }
const stream = createReadStream(filePath, range)
try {
for await (const chunk of stream) {
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
hash.update(buffer)
readBytes += buffer.length
}
} finally {
await releaseJsonlReadStream(stream)
}
return readBytes === expectedBytes ? `${expectedBytes}:${hash.digest('hex')}` : null
}
async function readPrefixDigests(
filePath: string,
parsedBytes: number,
carriedHeadDigest: string | null = null,
reader?: JsonlFileReader
): Promise<{ headDigest: string; boundaryDigest: string } | null> {
const headBytes = Math.min(CHECKPOINT_WINDOW_BYTES, parsedBytes)
const headDigest = carriedHeadDigest?.startsWith(`${headBytes}:`)
? carriedHeadDigest
: await readWindowDigest(filePath, 0, headBytes, reader)
if (headDigest === null) {
return null
}
const boundaryDigest = await readWindowDigest(
filePath,
Math.max(0, parsedBytes - CHECKPOINT_WINDOW_BYTES),
parsedBytes,
reader
)
return boundaryDigest === null ? null : { headDigest, boundaryDigest }
}
export function jsonlPhysicalFileId(stats: {
dev: number | bigint
ino: number | bigint
}): string | null {
return stats.ino === 0 || stats.ino === 0n ? null : `${stats.dev}:${stats.ino}`
}
export async function openJsonlFileReader(filePath: string): Promise<JsonlFileReader> {
while (true) {
const handle = await open(filePath, 'r')
let retained = false
try {
const stats = await readJsonlHandleSnapshot(handle)
const reader: JsonlFileReader = {
path: filePath,
handle,
stats,
snapshotCheckpoint: null,
snapshotChunks: isResumablePrefixLength(stats.size) ? null : [],
snapshotBytes: 0,
streamStartOffset: 0,
streamBytes: 0,
onChunk: (chunk) => {
reader.streamBytes += chunk.length
if (reader.snapshotChunks) {
reader.snapshotChunks.push(chunk)
reader.snapshotBytes += chunk.length
}
},
prepareRead: async (startOffset) => {
reader.streamStartOffset = Math.min(startOffset, stats.size)
if (!reader.snapshotChunks || reader.snapshotBytes !== 0 || startOffset <= 0) {
return
}
const skipped = await readJsonlSnapshotRange(
filePath,
handle,
stats,
0,
Math.min(startOffset, stats.size)
)
if (!skipped) {
return
}
reader.snapshotChunks.push(skipped)
reader.snapshotBytes += skipped.length
}
}
if (!isResumablePrefixLength(stats.size)) {
retained = true
return reader
}
const digests = await readPrefixDigests(filePath, stats.size, null, reader)
if (!digests) {
continue
}
reader.snapshotCheckpoint = {
parsedBytes: stats.size,
...digests,
physicalFileId: jsonlPhysicalFileId(stats)
}
retained = true
return reader
} finally {
if (!retained) {
await handle.close()
}
}
}
}
async function readPhysicalFileId(filePath: string): Promise<string | null> {
try {
const fileStat = await readJsonlFileSnapshot(filePath)
return jsonlPhysicalFileId(fileStat)
} catch {
return null
}
}
export async function buildJsonlFileCheckpoint(
filePath: string,
parsedBytes: number,
verifiedHeadDigest: string | null = null,
reader?: JsonlFileReader
): Promise<JsonlFileCheckpoint | null> {
if (!isResumablePrefixLength(parsedBytes)) {
return null
}
if (reader?.snapshotCheckpoint?.parsedBytes === parsedBytes) {
return reader.snapshotCheckpoint
}
const digests = await readPrefixDigests(
filePath,
parsedBytes,
verifiedHeadDigest ?? reader?.snapshotCheckpoint?.headDigest,
reader
)
return digests === null
? null
: {
parsedBytes,
...digests,
physicalFileId: reader
? jsonlPhysicalFileId(reader.stats)
: await readPhysicalFileId(filePath)
}
}
// Append-only evidence catches rotation, truncation and head/tail rewrites; it cannot prove the middle.
export async function resolveJsonlFileCheckpoint(
filePath: string,
checkpoint: JsonlFileCheckpoint | null | undefined,
reader?: JsonlFileReader
): Promise<JsonlFileCheckpoint | null> {
if (
!checkpoint ||
!Number.isSafeInteger(checkpoint.parsedBytes) ||
!isResumablePrefixLength(checkpoint.parsedBytes) ||
(reader && checkpoint.parsedBytes > reader.stats.size) ||
typeof checkpoint.boundaryDigest !== 'string' ||
typeof checkpoint.headDigest !== 'string'
) {
return null
}
const currentFileId = reader
? jsonlPhysicalFileId(reader.stats)
: await readPhysicalFileId(filePath)
if (
checkpoint.physicalFileId !== null &&
currentFileId !== null &&
checkpoint.physicalFileId !== currentFileId
) {
return null
}
const digests = await readPrefixDigests(
filePath,
checkpoint.parsedBytes,
reader?.snapshotCheckpoint?.headDigest,
reader
)
return digests?.boundaryDigest === checkpoint.boundaryDigest &&
digests?.headDigest === checkpoint.headDigest
? checkpoint
: null
}
export async function validateJsonlFileReader(
reader: JsonlFileReader,
originalCheckpoint?: JsonlFileCheckpoint | null
): Promise<boolean> {
const [current, pathStats] = await Promise.all([
readJsonlHandleSnapshot(reader.handle),
readJsonlFileSnapshot(reader.path)
])
if (reader.streamBytes !== reader.stats.size - reader.streamStartOffset) {
if (
sameJsonlFileSnapshot(reader.stats, current) &&
sameJsonlFileSnapshot(reader.stats, pathStats)
) {
throw new Error('JSONL snapshot stream ended before the unchanged file boundary.')
}
return false
}
if (
current.dev !== reader.stats.dev ||
current.ino !== reader.stats.ino ||
pathStats.dev !== current.dev ||
pathStats.ino !== current.ino ||
current.size < reader.stats.size ||
(current.size === reader.stats.size &&
(current.mtimeMs !== reader.stats.mtimeMs || current.ctimeMs !== reader.stats.ctimeMs))
) {
return false
}
const snapshot = reader.snapshotCheckpoint
if (!snapshot) {
if (!reader.snapshotChunks || reader.snapshotBytes !== reader.stats.size) {
return false
}
const chunks = reader.snapshotChunks
reader.snapshotChunks = null
return verifyJsonlSnapshotContents(reader.path, reader.handle, reader.stats, chunks)
}
const snapshotDigests = await readPrefixDigests(reader.path, snapshot.parsedBytes, null, reader)
if (
snapshotDigests?.headDigest !== snapshot.headDigest ||
snapshotDigests?.boundaryDigest !== snapshot.boundaryDigest
) {
return false
}
if (!originalCheckpoint || originalCheckpoint.parsedBytes === snapshot.parsedBytes) {
return true
}
const originalBoundary = await readWindowDigest(
reader.path,
Math.max(0, originalCheckpoint.parsedBytes - CHECKPOINT_WINDOW_BYTES),
originalCheckpoint.parsedBytes,
reader
)
return (
originalCheckpoint.headDigest === snapshotDigests.headDigest &&
originalCheckpoint.boundaryDigest === originalBoundary
)
}
+253
View File
@@ -0,0 +1,253 @@
import { appendFile, mkdtemp, rename, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
MIN_RESUMABLE_PREFIX_BYTES,
jsonlPhysicalFileId,
openJsonlFileReader,
resolveJsonlFileCheckpoint,
validateJsonlFileReader,
type JsonlFileReader
} from './jsonl-file-checkpoint'
import { readJsonlLinesFromOffset, type JsonlLineAtOffset } from './jsonl-line-offsets'
let directory: string
let filePath: string
let reader: JsonlFileReader | undefined
beforeEach(async () => {
directory = await mkdtemp(join(tmpdir(), 'orca-jsonl-reader-'))
filePath = join(directory, 'transcript.jsonl')
})
afterEach(async () => {
await reader?.handle.close()
reader = undefined
await rm(directory, { recursive: true, force: true })
})
async function openFixture(content: string): Promise<JsonlFileReader> {
await writeFile(filePath, content)
reader = await openJsonlFileReader(filePath)
return reader
}
async function collect(active: JsonlFileReader, start = 0): Promise<JsonlLineAtOffset[]> {
const lines: JsonlLineAtOffset[] = []
for await (const line of readJsonlLinesFromOffset(filePath, start, active)) {
lines.push(line)
}
return lines
}
function positionalReadArgs(args: readonly unknown[]): [Buffer, number, number, number] {
const [buffer, offset, length, position] = args
if (
!Buffer.isBuffer(buffer) ||
typeof offset !== 'number' ||
typeof length !== 'number' ||
typeof position !== 'number'
) {
throw new Error('Expected positional descriptor read arguments.')
}
return [buffer, offset, length, position]
}
describe('pinned small JSONL reader', () => {
it('encodes large physical identifiers exactly and preserves numeric caller compatibility', () => {
expect(jsonlPhysicalFileId({ dev: 1n, ino: 9_007_199_254_740_992n })).toBe('1:9007199254740992')
expect(jsonlPhysicalFileId({ dev: 1n, ino: 9_007_199_254_740_993n })).toBe('1:9007199254740993')
expect(jsonlPhysicalFileId({ dev: 3, ino: 4 })).toBe('3:4')
expect(jsonlPhysicalFileId({ dev: 3, ino: 0 })).toBeNull()
expect(jsonlPhysicalFileId({ dev: 3n, ino: 0n })).toBeNull()
})
it.each([Number.NaN, Infinity, -1, 1.5, Number.MAX_SAFE_INTEGER + 1])(
'rejects an invalid persisted offset %s before attempting a range read',
async (parsedBytes) => {
await expect(
resolveJsonlFileCheckpoint(filePath, {
parsedBytes,
headDigest: '4096:invalid',
boundaryDigest: '4096:invalid',
physicalFileId: null
})
).resolves.toBeNull()
}
)
it('verifies an empty snapshot without creating a stream or issuing a read', async () => {
const active = await openFixture('')
const read = vi.spyOn(active.handle, 'read')
const stream = vi.spyOn(active.handle, 'createReadStream')
expect(await collect(active)).toEqual([])
expect(await validateJsonlFileReader(active)).toBe(true)
expect(stream).not.toHaveBeenCalled()
expect(read).not.toHaveBeenCalled()
})
it('reads and verifies all bytes twice without creating a stream', async () => {
const contents = '会🙂\r\n\r\nnext\n尾🙂'
const active = await openFixture(contents)
const read = vi.spyOn(active.handle, 'read')
const stream = vi.spyOn(active.handle, 'createReadStream')
expect(await collect(active)).toEqual([
{ line: '会🙂', endOffset: Buffer.byteLength('会🙂\r\n'), terminated: true },
{ line: '', endOffset: Buffer.byteLength('会🙂\r\n\r\n'), terminated: true },
{ line: 'next', endOffset: Buffer.byteLength('会🙂\r\n\r\nnext\n'), terminated: true },
{ line: '尾🙂', endOffset: Buffer.byteLength(contents), terminated: false }
])
expect(await validateJsonlFileReader(active)).toBe(true)
expect(stream).not.toHaveBeenCalled()
expect(read).toHaveBeenCalledTimes(2)
expect(
(await Promise.all(read.mock.results.map((result) => result.value))).map(
(result) => result.bytesRead
)
).toEqual([Buffer.byteLength(contents), Buffer.byteLength(contents)])
})
it('accounts for a skipped legacy prefix in the full snapshot proof', async () => {
const contents = 'legacy会\r\nnew🙂\r\n'
const active = await openFixture(contents)
const offset = Buffer.byteLength('legacy会\r\n')
const read = vi.spyOn(active.handle, 'read')
expect(await collect(active, offset)).toEqual([
{ line: 'new🙂', endOffset: Buffer.byteLength(contents), terminated: true }
])
expect(await validateJsonlFileReader(active)).toBe(true)
expect(active.streamBytes).toBe(Buffer.byteLength(contents) - offset)
expect(active.snapshotBytes).toBe(Buffer.byteLength(contents))
expect(read).toHaveBeenCalledTimes(3)
})
it('accumulates short positional reads without exposing gaps or duplicate bytes', async () => {
const contents = '会🙂\r\nnext\n終🙂'
const active = await openFixture(contents)
const read = active.handle.read.bind(active.handle)
vi.spyOn(active.handle, 'read').mockImplementation(async (...args) => {
const [buffer, offset, length, position] = positionalReadArgs(args)
return read(buffer, offset, Math.min(length, 3), position)
})
expect(await collect(active)).toEqual([
{ line: '会🙂', endOffset: Buffer.byteLength('会🙂\r\n'), terminated: true },
{ line: 'next', endOffset: Buffer.byteLength('会🙂\r\nnext\n'), terminated: true },
{ line: '終🙂', endOffset: Buffer.byteLength(contents), terminated: false }
])
expect(await validateJsonlFileReader(active)).toBe(true)
})
it('rejects a stable premature EOF instead of publishing partial history', async () => {
const active = await openFixture('one\ntwo\n')
const read = active.handle.read.bind(active.handle)
vi.spyOn(active.handle, 'read').mockImplementationOnce(async (...args) => {
const result = await read(...args)
return { ...result, bytesRead: 0 }
})
await expect(collect(active)).rejects.toThrow(/snapshot read.*unchanged file boundary/)
expect(active.streamBytes).toBe(0)
expect(active.snapshotBytes).toBe(0)
})
it('rejects an unchanged EOF after a positive short read without yielding a partial line', async () => {
const active = await openFixture('one\ntwo\n')
const read = active.handle.read.bind(active.handle)
vi.spyOn(active.handle, 'read')
.mockImplementationOnce(async (...args) => {
const [buffer, offset, length, position] = positionalReadArgs(args)
return read(buffer, offset, Math.min(length, 2), position)
})
.mockImplementationOnce(async (...args) => {
const result = await read(...args)
return { ...result, bytesRead: 0 }
})
await expect(collect(active)).rejects.toThrow(/snapshot read.*unchanged file boundary/)
expect(active.streamBytes).toBe(0)
expect(active.snapshotBytes).toBe(0)
})
it('rejects a zero-byte verification read even after parsing every line', async () => {
const active = await openFixture('one\ntwo\n')
await collect(active)
const read = active.handle.read.bind(active.handle)
vi.spyOn(active.handle, 'read').mockImplementationOnce(async (...args) => {
const result = await read(...args)
return { ...result, bytesRead: 0 }
})
await expect(validateJsonlFileReader(active)).rejects.toThrow(
/snapshot read.*unchanged file boundary/
)
})
it('reads a skipped legacy prefix again when its proof changes during parsing', async () => {
const active = await openFixture('old\nnext\n')
await collect(active, 4)
await writeFile(filePath, 'NEW\nnext\nappend\n')
expect(await validateJsonlFileReader(active)).toBe(false)
})
it('leaves an append after the captured boundary for the next scan', async () => {
const original = 'one会🙂\r\n'
const active = await openFixture(original)
await appendFile(filePath, 'two🙂\n')
expect(await collect(active)).toEqual([
{ line: 'one会🙂', endOffset: Buffer.byteLength(original), terminated: true }
])
expect(await validateJsonlFileReader(active)).toBe(true)
expect(active.streamBytes).toBe(Buffer.byteLength(original))
})
it('rejects a full small-file middle rewrite even when the source also grows', async () => {
const original = 'head\nold\ntail\n'
const active = await openFixture(original)
await collect(active)
await writeFile(filePath, 'head\nNEW\ntail\nappend\n')
expect(await validateJsonlFileReader(active)).toBe(false)
})
it('keeps the pinned handle but rejects path rotation', async () => {
const original = 'one\n'
const active = await openFixture(original)
const replacement = join(directory, 'replacement.jsonl')
await writeFile(replacement, 'two\n')
await rename(filePath, join(directory, 'rotated.jsonl'))
await rename(replacement, filePath)
expect(await collect(active)).toEqual([{ line: 'one', endOffset: 4, terminated: true }])
expect(await validateJsonlFileReader(active)).toBe(false)
})
it('invalidates a truncation encountered by the descriptor loop', async () => {
const active = await openFixture('one\ntwo\n')
await writeFile(filePath, 'one\n')
expect(await collect(active)).toEqual([])
expect(await validateJsonlFileReader(active)).toBe(false)
})
it('uses the bounded buffer at the crossover and streams larger snapshots', async () => {
const active = await openFixture(`${'x'.repeat(MIN_RESUMABLE_PREFIX_BYTES - 1)}\n`)
const stream = vi.spyOn(active.handle, 'createReadStream')
await collect(active)
expect(await validateJsonlFileReader(active)).toBe(true)
expect(stream).not.toHaveBeenCalled()
await active.handle.close()
reader = undefined
const large = await openFixture(`${'x'.repeat(MIN_RESUMABLE_PREFIX_BYTES)}\n`)
const largeStream = vi.spyOn(large.handle, 'createReadStream')
await collect(large)
expect(await validateJsonlFileReader(large)).toBe(true)
expect(largeStream).toHaveBeenCalledOnce()
})
})
@@ -0,0 +1,92 @@
import { mkdtemp, open, rm, stat, utimes, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import {
normalizeJsonlFileSnapshot,
readJsonlFileSnapshot,
readJsonlHandleSnapshot,
sameJsonlFileSnapshot
} from './jsonl-file-snapshot'
let directory: string
let filePath: string
beforeEach(async () => {
directory = await mkdtemp(join(tmpdir(), 'orca-jsonl-snapshot-'))
filePath = join(directory, 'transcript.jsonl')
await writeFile(filePath, 'one\n')
})
afterEach(async () => {
await rm(directory, { recursive: true, force: true })
})
describe('JSONL file snapshot normalization', () => {
it.each(['dev', 'ino'] as const)(
'distinguishes exact %s identifiers that collide as JavaScript numbers',
async (field) => {
const stats = await stat(filePath, { bigint: true })
const lower = { ...stats, [field]: 9_007_199_254_740_992n }
const upper = { ...stats, [field]: 9_007_199_254_740_993n }
expect(Number(lower[field])).toBe(Number(upper[field]))
expect(normalizeJsonlFileSnapshot(lower)[field]).not.toBe(
normalizeJsonlFileSnapshot(upper)[field]
)
expect(
sameJsonlFileSnapshot(normalizeJsonlFileSnapshot(lower), normalizeJsonlFileSnapshot(upper))
).toBe(false)
}
)
it.each([0.123456789, 1.999999999, 1_784_029_260.125, 2_147_483_647.875])(
'preserves numeric stat timestamps at fractional epoch seconds %s',
async (seconds) => {
await utimes(filePath, seconds, seconds)
const numeric = await stat(filePath)
const snapshot = await readJsonlFileSnapshot(filePath)
const handle = await open(filePath, 'r')
try {
expect(snapshot.mtimeMs).toBe(numeric.mtimeMs)
expect(snapshot.ctimeMs).toBe(numeric.ctimeMs)
expect(snapshot.size).toBe(numeric.size)
expect(await readJsonlHandleSnapshot(handle)).toEqual(snapshot)
} finally {
await handle.close()
}
}
)
it('retains fractional nanoseconds instead of truncating bigint milliseconds', async () => {
const stats = await stat(filePath, { bigint: true })
const snapshot = normalizeJsonlFileSnapshot({
...stats,
mtimeNs: 1_784_029_260_123_456_789n,
ctimeNs: 1_784_029_260_987_654_321n
})
expect(snapshot.mtimeMs).toBe(1_784_029_260_000 + 123.456789)
expect(snapshot.ctimeMs).toBe(1_784_029_260_000 + 987.654321)
expect(snapshot.mtimeMs).not.toBe(1_784_029_260_123)
expect(snapshot.ctimeMs).not.toBe(1_784_029_260_987)
})
it('accepts zero and the largest safe byte offset', async () => {
const stats = await stat(filePath, { bigint: true })
expect(normalizeJsonlFileSnapshot({ ...stats, size: 0n }).size).toBe(0)
expect(
normalizeJsonlFileSnapshot({ ...stats, size: BigInt(Number.MAX_SAFE_INTEGER) }).size
).toBe(Number.MAX_SAFE_INTEGER)
})
it.each([-1n, BigInt(Number.MAX_SAFE_INTEGER) + 1n, 1n << 1024n])(
'rejects a file size that is not a safe byte offset: %s',
async (size) => {
const stats = await stat(filePath, { bigint: true })
expect(() => normalizeJsonlFileSnapshot({ ...stats, size })).toThrow(/safe byte offset/)
}
)
})
+113
View File
@@ -0,0 +1,113 @@
import type { BigIntStats, ReadStream } from 'node:fs'
import { stat, type FileHandle } from 'node:fs/promises'
const SNAPSHOT_FIELDS = ['dev', 'ino', 'size', 'mtimeMs', 'ctimeMs'] as const
export type JsonlFileSnapshot = {
dev: bigint
ino: bigint
size: number
mtimeMs: number
ctimeMs: number
}
export async function releaseJsonlReadStream(stream: ReadStream): Promise<void> {
if (!stream.closed) {
await new Promise<void>((resolve) => {
stream.once('close', resolve)
stream.destroy()
})
}
}
function nanosecondsToMilliseconds(value: bigint): number {
return Number(value / 1_000_000_000n) * 1000 + Number(value % 1_000_000_000n) / 1_000_000
}
export function normalizeJsonlFileSnapshot(
stats: Pick<BigIntStats, 'dev' | 'ino' | 'size' | 'mtimeNs' | 'ctimeNs'>
): JsonlFileSnapshot {
const size = Number(stats.size)
if (!Number.isSafeInteger(size) || size < 0) {
throw new RangeError('JSONL snapshot size cannot be represented as a safe byte offset.')
}
return {
dev: stats.dev,
ino: stats.ino,
size,
mtimeMs: nanosecondsToMilliseconds(stats.mtimeNs),
ctimeMs: nanosecondsToMilliseconds(stats.ctimeNs)
}
}
export async function readJsonlFileSnapshot(filePath: string): Promise<JsonlFileSnapshot> {
return normalizeJsonlFileSnapshot(await stat(filePath, { bigint: true }))
}
export async function readJsonlHandleSnapshot(handle: FileHandle): Promise<JsonlFileSnapshot> {
return normalizeJsonlFileSnapshot(await handle.stat({ bigint: true }))
}
export function sameJsonlFileSnapshot(left: JsonlFileSnapshot, right: JsonlFileSnapshot): boolean {
return SNAPSHOT_FIELDS.every((field) => left[field] === right[field])
}
export async function rejectUnchangedJsonlFileRead(
filePath: string,
handle: FileHandle,
initial: JsonlFileSnapshot
): Promise<void> {
const [current, pathStats] = await Promise.all([
readJsonlHandleSnapshot(handle),
readJsonlFileSnapshot(filePath)
])
if (sameJsonlFileSnapshot(initial, current) && sameJsonlFileSnapshot(initial, pathStats)) {
throw new Error('JSONL snapshot read ended before the unchanged file boundary.')
}
}
export async function readJsonlSnapshotRange(
filePath: string,
handle: FileHandle,
initial: JsonlFileSnapshot,
start: number,
endExclusive: number
): Promise<Buffer | null> {
const buffer = Buffer.allocUnsafe(endExclusive - start)
let consumed = 0
while (consumed < buffer.length) {
const { bytesRead } = await handle.read(
buffer,
consumed,
buffer.length - consumed,
start + consumed
)
if (bytesRead === 0) {
await rejectUnchangedJsonlFileRead(filePath, handle, initial)
return null
}
consumed += bytesRead
}
return buffer
}
export async function verifyJsonlSnapshotContents(
filePath: string,
handle: FileHandle,
initial: JsonlFileSnapshot,
pieces: readonly Buffer[]
): Promise<boolean> {
const current = await readJsonlSnapshotRange(filePath, handle, initial, 0, initial.size)
if (!current) {
return false
}
let consumed = 0
for (const piece of pieces) {
const end = consumed + piece.length
if (end > current.length || piece.compare(current, consumed, end) !== 0) {
return false
}
consumed = end
}
return consumed === current.length
}
+50 -4
View File
@@ -5,6 +5,8 @@
* CRLF transcript would otherwise drift one byte per line.
*/
import { createReadStream } from 'node:fs'
import { isResumablePrefixLength, type JsonlFileReader } from './jsonl-file-checkpoint'
import { readJsonlSnapshotRange, releaseJsonlReadStream } from './jsonl-file-snapshot'
const LINE_FEED = 0x0a
const CARRIAGE_RETURN = 0x0d
@@ -23,17 +25,61 @@ function decodeLine(pieces: Buffer[]): string {
return raw.toString('utf-8', 0, end)
}
async function* readJsonlChunks(
filePath: string,
startOffset: number,
reader?: Pick<JsonlFileReader, 'handle' | 'stats'>
): AsyncGenerator<Buffer> {
if (reader && !isResumablePrefixLength(reader.stats.size)) {
// Tiny snapshots avoid stream setup while keeping the same pinned-range proof.
const chunk = await readJsonlSnapshotRange(
filePath,
reader.handle,
reader.stats,
startOffset,
reader.stats.size
)
if (chunk) {
yield chunk
}
return
}
const stream = reader
? reader.handle.createReadStream({
start: startOffset,
end: reader.stats.size - 1,
autoClose: false
})
: createReadStream(filePath, { start: startOffset })
try {
for await (const chunk of stream) {
yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
}
} finally {
if (!reader) {
await releaseJsonlReadStream(stream)
}
}
}
export async function* readJsonlLinesFromOffset(
filePath: string,
startOffset: number
startOffset: number,
reader?: Pick<JsonlFileReader, 'handle' | 'stats'> & {
onChunk?: JsonlFileReader['onChunk']
prepareRead?: JsonlFileReader['prepareRead']
}
): AsyncGenerator<JsonlLineAtOffset> {
const stream = createReadStream(filePath, { start: startOffset })
await reader?.prepareRead?.(startOffset)
if (reader && reader.stats.size <= startOffset) {
return
}
const pending: Buffer[] = []
let pendingBytes = 0
let endOffset = startOffset
for await (const rawChunk of stream) {
const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk)
for await (const chunk of readJsonlChunks(filePath, startOffset, reader)) {
reader?.onChunk?.(chunk)
let searchFrom = 0
for (;;) {
const lineFeedIndex = chunk.indexOf(LINE_FEED, searchFrom)
@@ -55,6 +55,145 @@ it('folds events without rescanning accumulated location breakdowns', () => {
expect(result.dailyAggregates).toHaveLength(1000)
})
it('folds streamed events immediately with the batch counters, metrics and metadata', () => {
const [seed] = events(1)
const first: Event = {
...seed,
sessionId: 'mixed',
timestamp: '2026-09-09T00:00:00Z',
day: '2026-09-09',
projectKey: 'z',
projectLabel: 'Z',
repoId: 'repo-z',
worktreeId: 'tree-z',
model: 'large',
inputTokens: 3,
cachedInputTokens: 1,
outputTokens: 5,
reasoningOutputTokens: 2,
totalTokens: 8,
cost: 2
}
const input: Event[] = [
first,
{
...seed,
sessionId: 'unknown',
timestamp: '2026-09-10T00:00:00Z',
day: '2026-09-10',
model: null
},
{
...first,
timestamp: '2026-09-07T00:00:00Z',
day: '2026-09-07',
projectKey: 'a',
projectLabel: 'A',
repoId: 'repo-a',
worktreeId: 'tree-a',
model: 'small',
inputTokens: 7,
cachedInputTokens: 3,
outputTokens: 11,
reasoningOutputTokens: 5,
totalTokens: 18,
cost: 5
},
{
...first,
timestamp: '2026-09-09T12:00:00Z',
projectLabel: 'Changed label',
repoId: 'changed-repo',
worktreeId: 'changed-tree',
inputTokens: 13,
cachedInputTokens: 7,
outputTokens: 17,
reasoningOutputTokens: 11,
totalTokens: 30,
cost: 7
}
]
const expected = aggregation.aggregate(input)
const accumulator = aggregation.createAccumulator()
for (const event of input) {
accumulator.add(event)
event.totalTokens = 999
event.cost = 999
event.model = 'changed'
event.projectLabel = 'Changed after adding'
}
const result = accumulator.finalize()
expect(result).toEqual(expected)
expect(result.sessions.map((session) => session.sessionId)).toEqual(['unknown', 'mixed'])
expect(result.sessions[0].primaryModel).toBe('Unknown model')
const mixed = result.sessions[1]
expect(mixed).toMatchObject({
firstTimestamp: '2026-09-07T00:00:00Z',
lastTimestamp: '2026-09-09T12:00:00Z',
primaryModel: 'Mixed models',
hasMixedModels: true,
primaryProjectLabel: 'Multiple locations',
hasMixedLocations: true,
primaryRepoId: 'repo-z',
primaryWorktreeId: 'tree-z',
eventCount: 3,
totalInputTokens: 23,
totalCachedInputTokens: 11,
totalOutputTokens: 33,
totalReasoningOutputTokens: 18,
totalTokens: 56,
cost: 14
})
expect(mixed.locationBreakdown).toMatchObject([
{ locationKey: 'z', projectLabel: 'Z', totalTokens: 38, cost: 9 },
{ locationKey: 'a', projectLabel: 'A', totalTokens: 18, cost: 5 }
])
expect(mixed.modelBreakdown).toMatchObject([
{ modelKey: 'large', totalTokens: 38, cost: 9 },
{ modelKey: 'small', totalTokens: 18, cost: 5 }
])
expect(mixed.locationModelBreakdown).toMatchObject([
{ locationKey: 'z', modelKey: 'large', totalTokens: 38, cost: 9 },
{ locationKey: 'a', modelKey: 'small', totalTokens: 18, cost: 5 }
])
expect(result.dailyAggregates).toMatchObject([
{ day: '2026-09-07', projectKey: 'a', totalTokens: 18, cost: 5 },
{ day: '2026-09-09', projectKey: 'z', projectLabel: 'Z', totalTokens: 38, cost: 9 },
{ day: '2026-09-10', projectKey: 'path-0', totalTokens: 3, cost: 0.5 }
])
})
it('keeps first encounter order when streamed rollup sort keys tie', () => {
const accumulator = aggregation.createAccumulator()
for (const [index, event] of events(4).entries()) {
accumulator.add({ ...event, sessionId: `session-${index % 2}`, projectLabel: 'Same label' })
}
const result = accumulator.finalize()
expect(result.sessions.map((session) => session.sessionId)).toEqual(['session-0', 'session-1'])
expect(result.sessions[0].locationBreakdown.map((entry) => entry.locationKey)).toEqual([
'path-0',
'path-2'
])
expect(result.sessions[0].modelBreakdown.map((entry) => entry.modelKey)).toEqual([
'model-0',
'model-2'
])
expect(result.dailyAggregates.map((entry) => entry.projectKey)).toEqual([
'path-0',
'path-1',
'path-2',
'path-3'
])
})
it('finalizes an empty stream once and rejects subsequent additions', () => {
const accumulator = aggregation.createAccumulator()
const result = accumulator.finalize()
expect(result).toEqual({ sessions: [], dailyAggregates: [] })
expect(accumulator.finalize()).toBe(result)
expect(() => accumulator.add(events(1)[0])).toThrow('Cannot add usage events after finalization')
})
it('merges rollups using first-match indexes without mutating source breakdowns', () => {
const source = aggregation.aggregate(events(1000)).sessions[0]
const existing = structuredClone(source)
+35 -8
View File
@@ -25,6 +25,11 @@ export type UsageEventAggregationOptions<TEvent, TMetric> = {
cloneSessionForMerge(session: UsageSession<TMetric>): UsageSession<TMetric>
}
export type UsageEventAggregationResult<TMetric> = {
sessions: UsageSession<TMetric>[]
dailyAggregates: UsageDailyAggregate<TMetric>[]
}
export function createUsageEventAggregation<
TEvent extends UsageAttributedEventFields,
TMetric extends object
@@ -214,15 +219,17 @@ export function createUsageEventAggregation<
)
}
function aggregate(events: TEvent[]): {
sessions: UsageSession<TMetric>[]
dailyAggregates: UsageDailyAggregate<TMetric>[]
} {
/** Finalization closes the stream and releases its lookup indexes. */
function createAccumulator() {
const sessionsById = new Map<string, UsageSession<TMetric>>()
const dailyByKey = new Map<string, UsageDailyAggregate<TMetric>>()
const breakdownsBySession = new Map<string, UsageSessionBreakdownIndex<TMetric>>()
let result: UsageEventAggregationResult<TMetric> | null = null
for (const event of events) {
function add(event: TEvent): void {
if (result) {
throw new Error('Cannot add usage events after finalization')
}
const eventMetric = metric.fromEvent(event)
const session = sessionsById.get(event.sessionId) ?? createEmptySession(event)
if (!sessionsById.has(event.sessionId)) {
@@ -269,14 +276,34 @@ export function createUsageEventAggregation<
metric.fold(daily, eventMetric)
}
return {
sessions: finalizeSessions(sessionsById),
dailyAggregates: sortDailyAggregates(dailyByKey)
function finalize(): UsageEventAggregationResult<TMetric> {
if (result) {
return result
}
result = {
sessions: finalizeSessions(sessionsById),
dailyAggregates: sortDailyAggregates(dailyByKey)
}
sessionsById.clear()
dailyByKey.clear()
breakdownsBySession.clear()
return result
}
return { add, finalize }
}
function aggregate(events: TEvent[]) {
const accumulator = createAccumulator()
for (const event of events) {
accumulator.add(event)
}
return accumulator.finalize()
}
return {
aggregate,
createAccumulator,
finalizeSessions,
sortDailyAggregates,
mergeSessions: (