diff --git a/.github/workflows/performance-contracts.yml b/.github/workflows/performance-contracts.yml index d45d8b8f45a..958696bc908 100644 --- a/.github/workflows/performance-contracts.yml +++ b/.github/workflows/performance-contracts.yml @@ -16,6 +16,10 @@ on: # without these a rename lands green and only breaks the next nightly. - 'src/main/sqlite/sync-database.test.ts' - 'src/main/runtime/orchestration/db/row-column-lists.test.ts' + - 'src/main/codex-usage/codex-rollout-file-lifetime.test.ts' + - 'src/main/codex-usage/scanner-incremental-append.test.ts' + - 'src/main/usage/jsonl-file-reader.test.ts' + - 'src/main/usage/jsonl-file-snapshot.test.ts' - 'src/relay/fs-path-metadata-symlink-concurrency.test.ts' - 'src/renderer/src/components/editor/rich-markdown-list-tokenizers.test.ts' - 'src/renderer/src/components/editor/rich-markdown-lowlight-cache.test.ts' diff --git a/config/performance-audit.md b/config/performance-audit.md index f105bfbe787..eeadfd8778a 100644 --- a/config/performance-audit.md +++ b/config/performance-audit.md @@ -16,8 +16,9 @@ manual review. A warning identifies repeated setup, not proof of visible lag. `pnpm test:perf:contracts` runs the explicit selection in `vitest.performance.config.ts`: SQLite statement reuse and schema parity, relay -filesystem concurrency, tokenizer rejection, highlighting cache, queued -cancellation, terminal backing-memory retention and detector fixtures. Missing +filesystem concurrency, incremental Codex scans and reader lifetimes, pinned JSONL +readers and snapshots, tokenizer rejection, highlighting cache, queued cancellation, +terminal backing-memory retention and detector fixtures. Missing listed files fail configuration loading. Tests run serially, without retries, and inherit the full suite's setup and forced-GC support. This makes existing regression coverage easy to run and attribute; it does not create new workload diff --git a/config/scripts/codex-usage-parser-memory-benchmark.mjs b/config/scripts/codex-usage-parser-memory-benchmark.mjs new file mode 100644 index 00000000000..ce49d505d94 --- /dev/null +++ b/config/scripts/codex-usage-parser-memory-benchmark.mjs @@ -0,0 +1,300 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { appendFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { availableParallelism, cpus, release, tmpdir, totalmem } from 'node:os' +import { dirname, join, relative, resolve, sep } from 'node:path' +import { performance } from 'node:perf_hooks' +import { pathToFileURL } from 'node:url' + +const root = resolve(import.meta.dirname, '../..') +const digest = (value) => createHash('sha256').update(value).digest('hex') +const args = process.argv.slice(2) + +function argument(name, fallback) { + const index = args.indexOf(name) + return index === -1 ? fallback : args[index + 1] +} + +function positiveInteger(value, label) { + const parsed = Number(value) + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${label} must be a positive integer`) + } + return parsed +} + +function memorySnapshot(phase, started) { + globalThis.gc() + globalThis.gc() + return { phase, elapsedMs: performance.now() - started, ...process.memoryUsage() } +} + +async function runWorker() { + assert.equal(typeof globalThis.gc, 'function', 'Run workers with --expose-gc') + const parser = await import(pathToFileURL(argument('--bundle')).href) + const count = positiveInteger(argument('--events'), 'events') + const stride = positiveInteger(argument('--stride'), 'stride') + const intervalMs = positiveInteger(argument('--sample-ms'), 'sample-ms') + const arm = argument('--arm') + let claimQueries = 0 + let committedKeys = 0 + const canClaim = () => claimQueries++ % stride === 0 + const options = + arm === 'baseline' + ? { claimEventKey: canClaim } + : { canClaimEventKey: canClaim, commitEventKey: () => committedKeys++ } + const started = performance.now() + const baseline = memorySnapshot('before', started) + const samples = [] + const timer = setInterval(() => samples.push(memorySnapshot('during', started)), intervalMs) + let result + try { + result = await parser.parseCodexUsageFile(argument('--fixture'), () => null, options) + } finally { + clearInterval(timer) + } + globalThis.__orcaMemoryResult = result + const final = memorySnapshot('after', started) + const duringSampleCount = samples.length + samples.push(final) + const expected = Math.ceil(count / stride) + const correctness = { + claimQueries, + committedKeys: arm === 'baseline' ? null : committedKeys, + ownedKeys: result.ownedEventKeys.length, + countedEvents: result.sessions.reduce((sum, session) => sum + session.eventCount, 0), + inputTokens: result.sessions.reduce((sum, session) => sum + session.totalInputTokens, 0), + outputTokens: result.sessions.reduce((sum, session) => sum + session.totalOutputTokens, 0), + totalTokens: result.sessions.reduce((sum, session) => sum + session.totalTokens, 0), + hasDeferredClaims: result.hasDeferredClaims, + projectionSha256: digest( + JSON.stringify({ + sessions: result.sessions, + dailyAggregates: result.dailyAggregates, + ownedEventKeys: result.ownedEventKeys, + hasDeferredClaims: result.hasDeferredClaims + }) + ) + } + assert.equal(correctness.claimQueries, count) + assert.equal(correctness.ownedKeys, expected) + assert.equal(correctness.countedEvents, expected) + assert.equal(correctness.inputTokens, expected * 100) + assert.equal(correctness.outputTokens, expected * 10) + assert.equal(correctness.totalTokens, expected * 110) + assert.equal(correctness.hasDeferredClaims, stride > 1 && count > 1) + if (arm === 'current') { + assert.equal(committedKeys, expected) + } + process.stdout.write( + `${JSON.stringify({ + arm, + stride, + duringSampleCount, + finalOnlyMeasurement: duringSampleCount === 0, + baseline, + samples, + sampledRetainedPeakDeltaBytes: + Math.max(...samples.map((sample) => sample.heapUsed)) - baseline.heapUsed, + finalRetainedDeltaBytes: final.heapUsed - baseline.heapUsed, + correctness + })}\n` + ) + delete globalThis.__orcaMemoryResult +} + +function writeFixture(filePath, eventCount) { + const cwd = join('benchmark', 'workspace') + writeFileSync( + filePath, + `${JSON.stringify({ type: 'session_meta', payload: { id: 'session', cwd } })}\n` + ) + let batch = '' + for (let index = 0; index < eventCount; index++) { + batch += `${JSON.stringify({ + timestamp: new Date(1791547200000 + index * 1000).toISOString(), + type: 'event_msg', + payload: { + type: 'token_count', + info: { + model: 'gpt-5-codex', + last_token_usage: { input_tokens: 100, output_tokens: 10, total_tokens: 110 } + } + } + })}\n` + if ((index + 1) % 5000 === 0) { + appendFileSync(filePath, batch) + batch = '' + } + } + if (batch) { + appendFileSync(filePath, batch) + } +} + +async function runComparison() { + const { build, version: esbuildVersion } = await import('esbuild') + const { runProcessSync, describeProcessFailure } = await import('./script-child-process.mjs') + const { buildCounterbalancedSchedule } = await import('./counterbalanced-benchmark-schedule.mjs') + const eventCount = positiveInteger(argument('--events', '100000'), 'events') + const pairCount = positiveInteger(argument('--pairs', '2'), 'pairs') + const intervalMs = positiveInteger(argument('--sample-ms', '35'), 'sample-ms') + const schedule = buildCounterbalancedSchedule(pairCount, 'baseline', 'current') + const execute = (program, commandArgs) => { + const result = runProcessSync({ + program, + args: commandArgs, + cwd: root, + env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' }, + timeoutMs: 120000, + maxOutputBytes: 16 * 1024 * 1024 + }) + if (result.code !== 0 || result.timedOut || result.outputTruncated) { + throw new Error(describeProcessFailure(result)) + } + return result.stdout + } + const baselineCommit = execute('git', [ + 'rev-parse', + '--verify', + `${argument('--baseline-ref', 'HEAD')}^{commit}` + ]).trim() + const temporary = mkdtempSync(join(tmpdir(), 'orca-codex-parser-memory-')) + try { + const sources = {} + const bundles = {} + for (const arm of ['baseline', 'current']) { + const sourceHashes = new Map() + const bundlePath = join(temporary, `${arm}.mjs`) + await build({ + entryPoints: [join(root, 'src/main/codex-usage/codex-rollout-file-parse.ts')], + bundle: true, + platform: 'node', + target: 'node20', + format: 'esm', + outfile: bundlePath, + logLevel: 'silent', + plugins: [ + { + name: 'record-parser-source-graph', + setup(builder) { + builder.onLoad({ filter: /\.[cm]?[jt]sx?$/ }, ({ path }) => { + const sourcePath = relative(root, path).split(sep).join('/') + if (!sourcePath.startsWith('src/')) { + return undefined + } + const contents = + arm === 'baseline' + ? execute('git', ['show', `${baselineCommit}:${sourcePath}`]) + : readFileSync(path, 'utf8') + sourceHashes.set(sourcePath, digest(contents)) + return { + contents, + resolveDir: dirname(path), + loader: sourcePath.endsWith('.tsx') ? 'tsx' : 'ts' + } + }) + } + } + ] + }) + bundles[arm] = bundlePath + sources[arm] = { + origin: arm === 'baseline' ? baselineCommit : 'worktree at bundle build', + files: Object.fromEntries( + [...sourceHashes].sort(([left], [right]) => left.localeCompare(right)) + ), + bundleSha256: digest(readFileSync(bundlePath)) + } + } + const fixturePath = join(temporary, 'rollout.jsonl') + writeFixture(fixturePath, eventCount) + const fixtureBytes = readFileSync(fixturePath) + const measurements = [] + for (let round = 0; round < schedule.length; round++) { + for (const stride of [1, 10]) { + const paired = [] + for (const arm of schedule[round]) { + const measurement = JSON.parse( + execute(process.execPath, [ + '--expose-gc', + import.meta.filename, + '--worker', + '--bundle', + bundles[arm], + '--fixture', + fixturePath, + '--events', + String(eventCount), + '--stride', + String(stride), + '--sample-ms', + String(intervalMs), + '--arm', + arm + ]) + ) + paired.push(measurement) + measurements.push({ round: round + 1, order: schedule[round], ...measurement }) + } + assert.equal(paired[0].correctness.projectionSha256, paired[1].correctness.projectionSha256) + } + } + const artifact = { + schemaVersion: 1, + recordedAt: new Date().toISOString(), + command: [ + 'node', + relative(root, import.meta.filename) + .split(sep) + .join('/'), + ...args + ].join(' '), + environment: { + node: process.version, + v8: process.versions.v8, + platform: process.platform, + architecture: process.arch, + osRelease: release(), + cpuModel: cpus()[0]?.model ?? null, + availableParallelism: availableParallelism(), + totalMemoryBytes: totalmem(), + esbuild: esbuildVersion, + timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone + }, + instrumentation: { + metric: 'sampled retained heapUsed after two forced full GCs; not absolute peak memory', + sampleIntervalMs: intervalMs, + isolation: 'one fresh Node process for each arm, scenario and round', + finalResult: 'held by a global reference through the final sample', + observerMemory: 'retained snapshot records contribute small duration-dependent overhead', + baseline: 'all bundled project source modules read from the pinned baseline commit', + current: 'all bundled project source modules read and hashed once before any worker runs', + timing: 'elapsed values include forced GC and are not a speed benchmark' + }, + instrumentationSha256: Object.fromEntries( + [ + import.meta.filename, + join(import.meta.dirname, 'script-child-process.mjs'), + join(import.meta.dirname, 'counterbalanced-benchmark-schedule.mjs') + ].map((path) => [relative(root, path).split(sep).join('/'), digest(readFileSync(path))]) + ), + baselineCommit, + sources, + fixture: { events: eventCount, bytes: fixtureBytes.length, sha256: digest(fixtureBytes) }, + schedule, + measurements + } + const output = `${JSON.stringify(artifact, null, 2)}\n` + const outputPath = argument('--output') + if (outputPath) { + writeFileSync(resolve(root, outputPath), output) + } else { + process.stdout.write(output) + } + } finally { + rmSync(temporary, { recursive: true, force: true }) + } +} + +await (args.includes('--worker') ? runWorker() : runComparison()) diff --git a/config/vitest.performance.config.ts b/config/vitest.performance.config.ts index 0a219a6a257..44f7b49d750 100644 --- a/config/vitest.performance.config.ts +++ b/config/vitest.performance.config.ts @@ -7,6 +7,10 @@ import { nodeRuntimePool } from './scripts/vitest-node-runtime-pool' const contracts = [ 'src/main/sqlite/sync-database.test.ts', 'src/main/runtime/orchestration/db/row-column-lists.test.ts', + 'src/main/codex-usage/codex-rollout-file-lifetime.test.ts', + 'src/main/codex-usage/scanner-incremental-append.test.ts', + 'src/main/usage/jsonl-file-reader.test.ts', + 'src/main/usage/jsonl-file-snapshot.test.ts', 'src/relay/fs-path-metadata-symlink-concurrency.test.ts', 'src/renderer/src/components/editor/rich-markdown-list-tokenizers.test.ts', 'src/renderer/src/components/editor/rich-markdown-lowlight-cache.test.ts', diff --git a/docs/bug-reproductions/usage-scan-append/codex-memory-exploration.json b/docs/bug-reproductions/usage-scan-append/codex-memory-exploration.json new file mode 100644 index 00000000000..221b45fd339 --- /dev/null +++ b/docs/bug-reproductions/usage-scan-append/codex-memory-exploration.json @@ -0,0 +1,63 @@ +{ + "status": "historical exploratory parser-retention comparison, not the final pinned-source benchmark", + "baselineCommit": "51e7181850b022bae2d41091fe1229accef4bd50", + "sourceProvenance": { + "baseline": "Only codex-rollout-file-parse.ts was loaded with git show from this commit; other bundled project dependencies came from the worktree.", + "current": "Actual worktree parser and project dependencies at the time of the exploratory run.", + "sourceHashes": null, + "limitation": "Source hashes and individual heap snapshots were not saved during this run. Later hashes cannot establish the earlier source. Use the reproducible full-graph comparison for final evidence." + }, + "environment": { + "node": "v24.20.0", + "v8": "13.6.233.17-node.53", + "platform": "darwin", + "architecture": "arm64", + "timeZone": "America/Los_Angeles" + }, + "instrumentation": { + "fixture": "100,000 event_msg/token_count rows with distinct one-second timestamps, model gpt-5-codex, last usage input 100/output 10/total 110, and one session_meta row; written in batches of 5,000.", + "clockOrigin": 1791547200000, + "metric": "heapUsed after two forced full GCs before parse, every 35 ms during parse, and after parse", + "isolation": "Each measurement had its own async function scope; result held on globalThis through the final sample, removed at function end, with setImmediate before the next baseline sample.", + "ownership": "baseline claimEventKey and current canClaimEventKey accept queryIndex++ % stride === 0; current commitEventKey is a no-op", + "interpretation": "Sampled retained memory, not absolute process peak. Timings are not a speed benchmark." + }, + "measurements": [ + { + "arm": "baseline", + "stride": 1, + "samples": 9, + "claimedKeys": 100000, + "countedEvents": 100000, + "sampledRetainedPeakDeltaMiB": 45.45518493652344, + "finalRetainedDeltaMiB": 6.44708251953125 + }, + { + "arm": "current", + "stride": 1, + "samples": 10, + "claimedKeys": 100000, + "countedEvents": 100000, + "sampledRetainedPeakDeltaMiB": 7.5893402099609375, + "finalRetainedDeltaMiB": 6.328620910644531 + }, + { + "arm": "baseline", + "stride": 10, + "samples": 4, + "claimedKeys": 10000, + "countedEvents": 10000, + "sampledRetainedPeakDeltaMiB": 3.7055206298828125, + "finalRetainedDeltaMiB": 0.6778564453125 + }, + { + "arm": "current", + "stride": 10, + "samples": 5, + "claimedKeys": 10000, + "countedEvents": 10000, + "sampledRetainedPeakDeltaMiB": 0.91009521484375, + "finalRetainedDeltaMiB": 0.66552734375 + } + ] +} diff --git a/docs/bug-reproductions/usage-scan-append/codex-parser-memory-final.json b/docs/bug-reproductions/usage-scan-append/codex-parser-memory-final.json new file mode 100644 index 00000000000..bce7e703d09 --- /dev/null +++ b/docs/bug-reproductions/usage-scan-append/codex-parser-memory-final.json @@ -0,0 +1,978 @@ +{ + "schemaVersion": 1, + "recordedAt": "2026-10-10T00:27:00.815Z", + "command": "node config/scripts/codex-usage-parser-memory-benchmark.mjs --baseline-ref 51e7181850b022bae2d41091fe1229accef4bd50 --events 100000 --pairs 2 --sample-ms 35 --output docs/bug-reproductions/usage-scan-append/codex-parser-memory-final.json", + "environment": { + "node": "v24.20.0", + "v8": "13.6.233.17-node.53", + "platform": "darwin", + "architecture": "arm64", + "osRelease": "25.3.0", + "cpuModel": "Apple M4 Max", + "availableParallelism": 16, + "totalMemoryBytes": 137438953472, + "esbuild": "0.28.2", + "timeZone": "America/Los_Angeles" + }, + "instrumentation": { + "metric": "sampled retained heapUsed after two forced full GCs; not absolute peak memory", + "sampleIntervalMs": 35, + "isolation": "one fresh Node process for each arm, scenario and round", + "finalResult": "held by a global reference through the final sample", + "observerMemory": "retained snapshot records contribute small duration-dependent overhead", + "baseline": "all bundled project source modules read from the pinned baseline commit", + "current": "all bundled project source modules read and hashed once before any worker runs", + "timing": "elapsed values include forced GC and are not a speed benchmark" + }, + "instrumentationSha256": { + "config/scripts/codex-usage-parser-memory-benchmark.mjs": "434e9818f8d8b132603912b104560a000e9e2635a2564cf65ae7e87cff483a1a", + "config/scripts/script-child-process.mjs": "a7441428bf7b5c9a897b6a3c057c50f46acca24c4f218a4b849de5aec53c4173", + "config/scripts/counterbalanced-benchmark-schedule.mjs": "bc35b40e954fa7ddec0225ed375bd3b113c2e05b8554bc50734d14f89987294d" + }, + "baselineCommit": "51e7181850b022bae2d41091fe1229accef4bd50", + "sources": { + "baseline": { + "origin": "51e7181850b022bae2d41091fe1229accef4bd50", + "files": { + "src/main/codex-usage/codex-model-pricing.ts": "05416b11916778dc2c69edfccfb277b0a2fe6ef7e88e0d8c9bdf030f2dc7ea1d", + "src/main/codex-usage/codex-rollout-file-parse.ts": "9ef58c665491ea709166d9580913bb1791a1748435cb2af1a5bb41dfe47df792", + "src/main/codex-usage/codex-rollout-resume-state.ts": "de254c8fbca043c311350adf5466d5e510c4894b9c0d9e1a086a204a8c949f6f", + "src/main/codex-usage/codex-usage-aggregation.ts": "82774021e25bfcf413dbc2faad8d0cb92310a3ddeac87ba640abe4254f19d082", + "src/main/codex-usage/codex-usage-event-attribution.ts": "1b3c37111bb5ff9a28fb9a97d3e40d0c76026b6b5be2d55a0c512b68d5944b02", + "src/main/codex-usage/codex-usage-record-parser.ts": "b73ecd62b6ed20ce9cd7e21ca8e78e8a3f7d9a635311f2d8d56f9d64042a1661", + "src/main/codex-usage/codex-usage-token-delta.ts": "fb3e64b17eef3d8e23fafbcb2ce4a1ebfe44cab1091faa979d8c0269f294d300", + "src/main/usage/jsonl-line-offsets.ts": "ea7f58e248e8c332bfe6d18294261b1941816eb38f079e9a2c0c9b15c06820c5", + "src/main/usage/usage-calendar-range.ts": "fe4c4eb09aaefcae6d2698e113e134663826c3ed9ca3ef960f3047a95686b1dd", + "src/main/usage/usage-event-aggregation.ts": "68caf65824d32c55b706fd78ee90d583439f4b86445862cd75d10f3f4ef122d1", + "src/main/usage/usage-event-attribution.ts": "e7a909437d836c34939e2709ede99794f81cc8746c6c55b1bbdd36bf352e7664", + "src/main/usage/usage-path-comparison.ts": "07be51caa36e43962fac4cb66c1e0bfd4700f58aeb76eae32cc524fff88625d8", + "src/main/usage/usage-record-coercion.ts": "1b7f9fd8a0c01fbe25f82171d9ceefcbc717e87df3e321fc9514c59c9cfbd7da", + "src/main/usage/usage-rollup-merge.ts": "67cc062058439d659f77a8b47ce1b0ad3e2c323a83d2332aaa75e399de98b433", + "src/main/usage/usage-rollup-records.ts": "eff570d174439c5786175081ffe3ac2aedd960ebfc7e317f9f3cfc4091f27dab", + "src/main/usage/usage-session-breakdown-index.ts": "51725b3480ed078506beeec3105bd95402836e10478bd681cdff95fed080f81d" + }, + "bundleSha256": "aaf0b5266d4511752229a998db9789c583ca3cd2c0b7bcc4afe03c465ef6e564" + }, + "current": { + "origin": "worktree at bundle build", + "files": { + "src/main/codex-usage/codex-model-pricing.ts": "05416b11916778dc2c69edfccfb277b0a2fe6ef7e88e0d8c9bdf030f2dc7ea1d", + "src/main/codex-usage/codex-rollout-file-parse.ts": "f39a10c1dce47a71d46c0948d05df30aa6dcc044c81b777b6a013be6f23e9c7a", + "src/main/codex-usage/codex-rollout-resume-state.ts": "aa028d2155b7744c0d961c5649537faa20c39bfe404dea8d9c4c1464d11a0e39", + "src/main/codex-usage/codex-usage-aggregation.ts": "82774021e25bfcf413dbc2faad8d0cb92310a3ddeac87ba640abe4254f19d082", + "src/main/codex-usage/codex-usage-event-attribution.ts": "1b3c37111bb5ff9a28fb9a97d3e40d0c76026b6b5be2d55a0c512b68d5944b02", + "src/main/codex-usage/codex-usage-record-parser.ts": "b73ecd62b6ed20ce9cd7e21ca8e78e8a3f7d9a635311f2d8d56f9d64042a1661", + "src/main/codex-usage/codex-usage-token-delta.ts": "fb3e64b17eef3d8e23fafbcb2ce4a1ebfe44cab1091faa979d8c0269f294d300", + "src/main/usage/jsonl-file-checkpoint.ts": "b7f307564989dabaf1dfda095a26c7bd9f0683e0efcabe79a010c7b58fa1a2d4", + "src/main/usage/jsonl-file-snapshot.ts": "b1b8869db92c08c762160acba7570aabff25879649cb6ffa3826912ee35deffe", + "src/main/usage/jsonl-line-offsets.ts": "f3574adfad4549c6330e2ac52760c91be5a8cbb9bf8e1beb440e3226087074db", + "src/main/usage/usage-calendar-range.ts": "fe4c4eb09aaefcae6d2698e113e134663826c3ed9ca3ef960f3047a95686b1dd", + "src/main/usage/usage-event-aggregation.ts": "31adda2e8e0a32756416fb49b2c3e02c707703bfe76434aabc95c61a7efd180b", + "src/main/usage/usage-event-attribution.ts": "e7a909437d836c34939e2709ede99794f81cc8746c6c55b1bbdd36bf352e7664", + "src/main/usage/usage-path-comparison.ts": "07be51caa36e43962fac4cb66c1e0bfd4700f58aeb76eae32cc524fff88625d8", + "src/main/usage/usage-record-coercion.ts": "1b7f9fd8a0c01fbe25f82171d9ceefcbc717e87df3e321fc9514c59c9cfbd7da", + "src/main/usage/usage-rollup-merge.ts": "67cc062058439d659f77a8b47ce1b0ad3e2c323a83d2332aaa75e399de98b433", + "src/main/usage/usage-rollup-records.ts": "eff570d174439c5786175081ffe3ac2aedd960ebfc7e317f9f3cfc4091f27dab", + "src/main/usage/usage-session-breakdown-index.ts": "51725b3480ed078506beeec3105bd95402836e10478bd681cdff95fed080f81d" + }, + "bundleSha256": "9f8b81ddae88e3d4fdd48175a53f68011d9606bfe169d6f5660238b3db74abaf" + } + }, + "fixture": { + "events": 100000, + "bytes": 20200079, + "sha256": "fa36fd10e36653020c0997890c0c9d9e75f40ee4040adf3a8d79735c4fbe20d5" + }, + "schedule": [ + ["baseline", "current"], + ["current", "baseline"] + ], + "measurements": [ + { + "round": 1, + "order": ["baseline", "current"], + "arm": "baseline", + "stride": 1, + "duringSampleCount": 9, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 2.961416, + "rss": 54771712, + "heapTotal": 7667712, + "heapUsed": 4923944, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 43.085458, + "rss": 79740928, + "heapTotal": 28196864, + "heapUsed": 10333960, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 80.171208, + "rss": 100352000, + "heapTotal": 50823168, + "heapUsed": 16436440, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 116.55525, + "rss": 109264896, + "heapTotal": 56590336, + "heapUsed": 22257384, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 153.371208, + "rss": 129040384, + "heapTotal": 95125504, + "heapUsed": 27179344, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 191.0125, + "rss": 146718720, + "heapTotal": 100581376, + "heapUsed": 32420792, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 223.885208, + "rss": 154877952, + "heapTotal": 105824256, + "heapUsed": 37958976, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 261.673083, + "rss": 166674432, + "heapTotal": 178733056, + "heapUsed": 43593472, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 302.397916, + "rss": 184074240, + "heapTotal": 183451648, + "heapUsed": 48113072, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 337.490916, + "rss": 187842560, + "heapTotal": 186597376, + "heapUsed": 51628032, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "after", + "elapsedMs": 404.128916, + "rss": 197738496, + "heapTotal": 147144704, + "heapUsed": 11730912, + "external": 2455679, + "arrayBuffers": 143819 + } + ], + "sampledRetainedPeakDeltaBytes": 46704088, + "finalRetainedDeltaBytes": 6806968, + "correctness": { + "claimQueries": 100000, + "committedKeys": null, + "ownedKeys": 100000, + "countedEvents": 100000, + "inputTokens": 10000000, + "outputTokens": 1000000, + "totalTokens": 11000000, + "hasDeferredClaims": false, + "projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a" + } + }, + { + "round": 1, + "order": ["baseline", "current"], + "arm": "current", + "stride": 1, + "duringSampleCount": 9, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 2.704749999999997, + "rss": 54919168, + "heapTotal": 7929856, + "heapUsed": 4938600, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 40.513166999999996, + "rss": 68812800, + "heapTotal": 11419648, + "heapUsed": 6306616, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 75.017375, + "rss": 75153408, + "heapTotal": 16465920, + "heapUsed": 7232808, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 109.88325, + "rss": 75939840, + "heapTotal": 16990208, + "heapUsed": 7780688, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 145.465458, + "rss": 77348864, + "heapTotal": 18432000, + "heapUsed": 9102536, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 180.921875, + "rss": 78675968, + "heapTotal": 27344896, + "heapUsed": 9704632, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 215.932708, + "rss": 85442560, + "heapTotal": 27869184, + "heapUsed": 10366048, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 250.763458, + "rss": 88440832, + "heapTotal": 29966336, + "heapUsed": 12341424, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 286.477917, + "rss": 89866240, + "heapTotal": 30752768, + "heapUsed": 12972312, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 321.256625, + "rss": 91324416, + "heapTotal": 31277056, + "heapUsed": 13643896, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "after", + "elapsedMs": 336.48325, + "rss": 92192768, + "heapTotal": 29704192, + "heapUsed": 11803368, + "external": 2455471, + "arrayBuffers": 143611 + } + ], + "sampledRetainedPeakDeltaBytes": 8705296, + "finalRetainedDeltaBytes": 6864768, + "correctness": { + "claimQueries": 100000, + "committedKeys": 100000, + "ownedKeys": 100000, + "countedEvents": 100000, + "inputTokens": 10000000, + "outputTokens": 1000000, + "totalTokens": 11000000, + "hasDeferredClaims": false, + "projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a" + } + }, + { + "round": 1, + "order": ["baseline", "current"], + "arm": "baseline", + "stride": 10, + "duringSampleCount": 4, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 2.7109999999999985, + "rss": 54607872, + "heapTotal": 7929856, + "heapUsed": 4924000, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 41.222042, + "rss": 66764800, + "heapTotal": 11337728, + "heapUsed": 6303312, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 75.894208, + "rss": 71761920, + "heapTotal": 16498688, + "heapUsed": 7411304, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 111.375667, + "rss": 75366400, + "heapTotal": 25935872, + "heapUsed": 8445088, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 146.555292, + "rss": 83542016, + "heapTotal": 27410432, + "heapUsed": 9730008, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "after", + "elapsedMs": 172.027042, + "rss": 86786048, + "heapTotal": 24182784, + "heapUsed": 5946064, + "external": 2455679, + "arrayBuffers": 143819 + } + ], + "sampledRetainedPeakDeltaBytes": 4806008, + "finalRetainedDeltaBytes": 1022064, + "correctness": { + "claimQueries": 100000, + "committedKeys": null, + "ownedKeys": 10000, + "countedEvents": 10000, + "inputTokens": 1000000, + "outputTokens": 100000, + "totalTokens": 1100000, + "hasDeferredClaims": true, + "projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac" + } + }, + { + "round": 1, + "order": ["baseline", "current"], + "arm": "current", + "stride": 10, + "duringSampleCount": 4, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 3.0069579999999974, + "rss": 54706176, + "heapTotal": 7667712, + "heapUsed": 4938616, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 40.785916, + "rss": 65945600, + "heapTotal": 10551296, + "heapUsed": 5556160, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 75.849791, + "rss": 66551808, + "heapTotal": 10731520, + "heapUsed": 5764056, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 110.944416, + "rss": 66797568, + "heapTotal": 10993664, + "heapUsed": 5903120, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 145.28670799999998, + "rss": 68075520, + "heapTotal": 11419648, + "heapUsed": 6229832, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "after", + "elapsedMs": 172.400125, + "rss": 70189056, + "heapTotal": 11075584, + "heapUsed": 6002200, + "external": 2455471, + "arrayBuffers": 143611 + } + ], + "sampledRetainedPeakDeltaBytes": 1291216, + "finalRetainedDeltaBytes": 1063584, + "correctness": { + "claimQueries": 100000, + "committedKeys": 10000, + "ownedKeys": 10000, + "countedEvents": 10000, + "inputTokens": 1000000, + "outputTokens": 100000, + "totalTokens": 1100000, + "hasDeferredClaims": true, + "projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac" + } + }, + { + "round": 2, + "order": ["current", "baseline"], + "arm": "current", + "stride": 1, + "duringSampleCount": 10, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 2.3475420000000007, + "rss": 54722560, + "heapTotal": 7929856, + "heapUsed": 4938624, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 40.420334, + "rss": 68485120, + "heapTotal": 11419648, + "heapUsed": 6288496, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 76.145292, + "rss": 74825728, + "heapTotal": 16465920, + "heapUsed": 7269912, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 110.49249999999999, + "rss": 75481088, + "heapTotal": 16990208, + "heapUsed": 7924696, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 145.027584, + "rss": 76890112, + "heapTotal": 18432000, + "heapUsed": 9178464, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 180.48204199999998, + "rss": 78184448, + "heapTotal": 27344896, + "heapUsed": 9725784, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 217.70899999999997, + "rss": 86163456, + "heapTotal": 27869184, + "heapUsed": 10424152, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 250.85633399999998, + "rss": 89620480, + "heapTotal": 29966336, + "heapUsed": 12375976, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 286.69275, + "rss": 91013120, + "heapTotal": 30490624, + "heapUsed": 12988984, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 354.68100000000004, + "rss": 91783168, + "heapTotal": 31014912, + "heapUsed": 13393048, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 366.923584, + "rss": 91799552, + "heapTotal": 31014912, + "heapUsed": 13405528, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "after", + "elapsedMs": 400.572084, + "rss": 93257728, + "heapTotal": 29442048, + "heapUsed": 11784552, + "external": 2455471, + "arrayBuffers": 143611 + } + ], + "sampledRetainedPeakDeltaBytes": 8466904, + "finalRetainedDeltaBytes": 6845928, + "correctness": { + "claimQueries": 100000, + "committedKeys": 100000, + "ownedKeys": 100000, + "countedEvents": 100000, + "inputTokens": 10000000, + "outputTokens": 1000000, + "totalTokens": 11000000, + "hasDeferredClaims": false, + "projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a" + } + }, + { + "round": 2, + "order": ["current", "baseline"], + "arm": "baseline", + "stride": 1, + "duringSampleCount": 17, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 3.085000000000008, + "rss": 54870016, + "heapTotal": 7929856, + "heapUsed": 4923944, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 66.69608300000002, + "rss": 70172672, + "heapTotal": 16498688, + "heapUsed": 7281848, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 85.53883300000001, + "rss": 73515008, + "heapTotal": 17809408, + "heapUsed": 8570768, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 152.638541, + "rss": 85196800, + "heapTotal": 30031872, + "heapUsed": 12156408, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 159.173375, + "rss": 85639168, + "heapTotal": 30228480, + "heapUsed": 12357872, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 225.200083, + "rss": 104693760, + "heapTotal": 52133888, + "heapUsed": 17698272, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 297.829375, + "rss": 104857600, + "heapTotal": 52396032, + "heapUsed": 17837984, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 310.679916, + "rss": 104988672, + "heapTotal": 52396032, + "heapUsed": 17986776, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 343.32525, + "rss": 106856448, + "heapTotal": 54493184, + "heapUsed": 19932448, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 408.458833, + "rss": 111263744, + "heapTotal": 57901056, + "heapUsed": 23407968, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 420.895625, + "rss": 111493120, + "heapTotal": 58163200, + "heapUsed": 23541184, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 474.20687499999997, + "rss": 120029184, + "heapTotal": 94601216, + "heapUsed": 26919320, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 494.13695800000005, + "rss": 121274368, + "heapTotal": 96174080, + "heapUsed": 28168392, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 523.933083, + "rss": 135741440, + "heapTotal": 99794944, + "heapUsed": 32038824, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 567.489416, + "rss": 156827648, + "heapTotal": 106348544, + "heapUsed": 38424024, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 598.908, + "rss": 160595968, + "heapTotal": 110051328, + "heapUsed": 42221320, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 634.888333, + "rss": 170278912, + "heapTotal": 181878784, + "heapUsed": 46720192, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 670.023, + "rss": 185548800, + "heapTotal": 185810944, + "heapUsed": 50929816, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "after", + "elapsedMs": 735.424125, + "rss": 202817536, + "heapTotal": 147144704, + "heapUsed": 11727688, + "external": 2455679, + "arrayBuffers": 143819 + } + ], + "sampledRetainedPeakDeltaBytes": 46005872, + "finalRetainedDeltaBytes": 6803744, + "correctness": { + "claimQueries": 100000, + "committedKeys": null, + "ownedKeys": 100000, + "countedEvents": 100000, + "inputTokens": 10000000, + "outputTokens": 1000000, + "totalTokens": 11000000, + "hasDeferredClaims": false, + "projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a" + } + }, + { + "round": 2, + "order": ["current", "baseline"], + "arm": "current", + "stride": 10, + "duringSampleCount": 5, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 2.7139999999999986, + "rss": 54902784, + "heapTotal": 7929856, + "heapUsed": 4938792, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 39.88775, + "rss": 66011136, + "heapTotal": 10813440, + "heapUsed": 5552696, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 75.154917, + "rss": 66748416, + "heapTotal": 10993664, + "heapUsed": 5767816, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 110.337708, + "rss": 67108864, + "heapTotal": 11255808, + "heapUsed": 5912424, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 147.088458, + "rss": 68534272, + "heapTotal": 11419648, + "heapUsed": 6234112, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 179.94975, + "rss": 68550656, + "heapTotal": 11681792, + "heapUsed": 6230488, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "after", + "elapsedMs": 205.10170799999997, + "rss": 70860800, + "heapTotal": 10813440, + "heapUsed": 5992168, + "external": 2455471, + "arrayBuffers": 143611 + } + ], + "sampledRetainedPeakDeltaBytes": 1295320, + "finalRetainedDeltaBytes": 1053376, + "correctness": { + "claimQueries": 100000, + "committedKeys": 10000, + "ownedKeys": 10000, + "countedEvents": 10000, + "inputTokens": 1000000, + "outputTokens": 100000, + "totalTokens": 1100000, + "hasDeferredClaims": true, + "projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac" + } + }, + { + "round": 2, + "order": ["current", "baseline"], + "arm": "baseline", + "stride": 10, + "duringSampleCount": 4, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 3.1553749999999994, + "rss": 54804480, + "heapTotal": 7929856, + "heapUsed": 4923976, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 41.432542, + "rss": 66666496, + "heapTotal": 11337728, + "heapUsed": 6206048, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 75.186042, + "rss": 71827456, + "heapTotal": 16760832, + "heapUsed": 7309048, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 111.10141699999998, + "rss": 74629120, + "heapTotal": 17809408, + "heapUsed": 8403400, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 146.308042, + "rss": 83525632, + "heapTotal": 27148288, + "heapUsed": 9674064, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "after", + "elapsedMs": 172.545917, + "rss": 87998464, + "heapTotal": 24182784, + "heapUsed": 5945896, + "external": 2455679, + "arrayBuffers": 143819 + } + ], + "sampledRetainedPeakDeltaBytes": 4750088, + "finalRetainedDeltaBytes": 1021920, + "correctness": { + "claimQueries": 100000, + "committedKeys": null, + "ownedKeys": 10000, + "countedEvents": 10000, + "inputTokens": 1000000, + "outputTokens": 100000, + "totalTokens": 1100000, + "hasDeferredClaims": true, + "projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac" + } + } + ] +} diff --git a/docs/bug-reproductions/usage-scan-append/codex-parser-memory.json b/docs/bug-reproductions/usage-scan-append/codex-parser-memory.json new file mode 100644 index 00000000000..e94c4562017 --- /dev/null +++ b/docs/bug-reproductions/usage-scan-append/codex-parser-memory.json @@ -0,0 +1,879 @@ +{ + "schemaVersion": 1, + "recordedAt": "2026-10-09T18:39:49.816Z", + "command": "node config/scripts/codex-usage-parser-memory-benchmark.mjs --baseline-ref 51e7181850b022bae2d41091fe1229accef4bd50 --events 100000 --pairs 2 --sample-ms 35 --output docs/bug-reproductions/usage-scan-append/codex-parser-memory.json", + "environment": { + "node": "v24.20.0", + "v8": "13.6.233.17-node.53", + "platform": "darwin", + "architecture": "arm64", + "osRelease": "25.3.0", + "cpuModel": "Apple M4 Max", + "availableParallelism": 16, + "totalMemoryBytes": 137438953472, + "esbuild": "0.28.2", + "timeZone": "America/Los_Angeles" + }, + "instrumentation": { + "metric": "sampled retained heapUsed after two forced full GCs; not absolute peak memory", + "sampleIntervalMs": 35, + "isolation": "one fresh Node process for each arm, scenario and round", + "finalResult": "held by a global reference through the final sample", + "observerMemory": "retained snapshot records contribute small duration-dependent overhead", + "baseline": "all bundled project source modules read from the pinned baseline commit", + "current": "all bundled project source modules read and hashed once before any worker runs", + "timing": "elapsed values include forced GC and are not a speed benchmark" + }, + "instrumentationSha256": { + "config/scripts/codex-usage-parser-memory-benchmark.mjs": "434e9818f8d8b132603912b104560a000e9e2635a2564cf65ae7e87cff483a1a", + "config/scripts/script-child-process.mjs": "a7441428bf7b5c9a897b6a3c057c50f46acca24c4f218a4b849de5aec53c4173", + "config/scripts/counterbalanced-benchmark-schedule.mjs": "bc35b40e954fa7ddec0225ed375bd3b113c2e05b8554bc50734d14f89987294d" + }, + "baselineCommit": "51e7181850b022bae2d41091fe1229accef4bd50", + "sources": { + "baseline": { + "origin": "51e7181850b022bae2d41091fe1229accef4bd50", + "files": { + "src/main/codex-usage/codex-model-pricing.ts": "05416b11916778dc2c69edfccfb277b0a2fe6ef7e88e0d8c9bdf030f2dc7ea1d", + "src/main/codex-usage/codex-rollout-file-parse.ts": "9ef58c665491ea709166d9580913bb1791a1748435cb2af1a5bb41dfe47df792", + "src/main/codex-usage/codex-rollout-resume-state.ts": "de254c8fbca043c311350adf5466d5e510c4894b9c0d9e1a086a204a8c949f6f", + "src/main/codex-usage/codex-usage-aggregation.ts": "82774021e25bfcf413dbc2faad8d0cb92310a3ddeac87ba640abe4254f19d082", + "src/main/codex-usage/codex-usage-event-attribution.ts": "1b3c37111bb5ff9a28fb9a97d3e40d0c76026b6b5be2d55a0c512b68d5944b02", + "src/main/codex-usage/codex-usage-record-parser.ts": "b73ecd62b6ed20ce9cd7e21ca8e78e8a3f7d9a635311f2d8d56f9d64042a1661", + "src/main/codex-usage/codex-usage-token-delta.ts": "fb3e64b17eef3d8e23fafbcb2ce4a1ebfe44cab1091faa979d8c0269f294d300", + "src/main/usage/jsonl-line-offsets.ts": "ea7f58e248e8c332bfe6d18294261b1941816eb38f079e9a2c0c9b15c06820c5", + "src/main/usage/usage-calendar-range.ts": "fe4c4eb09aaefcae6d2698e113e134663826c3ed9ca3ef960f3047a95686b1dd", + "src/main/usage/usage-event-aggregation.ts": "68caf65824d32c55b706fd78ee90d583439f4b86445862cd75d10f3f4ef122d1", + "src/main/usage/usage-event-attribution.ts": "e7a909437d836c34939e2709ede99794f81cc8746c6c55b1bbdd36bf352e7664", + "src/main/usage/usage-path-comparison.ts": "07be51caa36e43962fac4cb66c1e0bfd4700f58aeb76eae32cc524fff88625d8", + "src/main/usage/usage-record-coercion.ts": "1b7f9fd8a0c01fbe25f82171d9ceefcbc717e87df3e321fc9514c59c9cfbd7da", + "src/main/usage/usage-rollup-merge.ts": "67cc062058439d659f77a8b47ce1b0ad3e2c323a83d2332aaa75e399de98b433", + "src/main/usage/usage-rollup-records.ts": "eff570d174439c5786175081ffe3ac2aedd960ebfc7e317f9f3cfc4091f27dab", + "src/main/usage/usage-session-breakdown-index.ts": "51725b3480ed078506beeec3105bd95402836e10478bd681cdff95fed080f81d" + }, + "bundleSha256": "aaf0b5266d4511752229a998db9789c583ca3cd2c0b7bcc4afe03c465ef6e564" + }, + "current": { + "origin": "worktree at bundle build", + "files": { + "src/main/codex-usage/codex-model-pricing.ts": "05416b11916778dc2c69edfccfb277b0a2fe6ef7e88e0d8c9bdf030f2dc7ea1d", + "src/main/codex-usage/codex-rollout-file-parse.ts": "6e4937105fc9cf8457b79ac16bcfc71f706f22a87e036166bd461aeea8c07fdc", + "src/main/codex-usage/codex-rollout-resume-state.ts": "aa028d2155b7744c0d961c5649537faa20c39bfe404dea8d9c4c1464d11a0e39", + "src/main/codex-usage/codex-usage-aggregation.ts": "82774021e25bfcf413dbc2faad8d0cb92310a3ddeac87ba640abe4254f19d082", + "src/main/codex-usage/codex-usage-event-attribution.ts": "1b3c37111bb5ff9a28fb9a97d3e40d0c76026b6b5be2d55a0c512b68d5944b02", + "src/main/codex-usage/codex-usage-record-parser.ts": "b73ecd62b6ed20ce9cd7e21ca8e78e8a3f7d9a635311f2d8d56f9d64042a1661", + "src/main/codex-usage/codex-usage-token-delta.ts": "fb3e64b17eef3d8e23fafbcb2ce4a1ebfe44cab1091faa979d8c0269f294d300", + "src/main/usage/jsonl-file-checkpoint.ts": "8eb579f356e1bb23b36319f00b3d4603fff80a963222dcb8a22331319cab3a03", + "src/main/usage/jsonl-file-snapshot.ts": "1bd98ac194f73097cd6c204238fe1db01e1e0d60a604e93c1d7841725971fe0d", + "src/main/usage/jsonl-line-offsets.ts": "f119465fad3771acd72b128853e84eed9165f0f74b75743ad662aa5bab991273", + "src/main/usage/usage-calendar-range.ts": "fe4c4eb09aaefcae6d2698e113e134663826c3ed9ca3ef960f3047a95686b1dd", + "src/main/usage/usage-event-aggregation.ts": "31adda2e8e0a32756416fb49b2c3e02c707703bfe76434aabc95c61a7efd180b", + "src/main/usage/usage-event-attribution.ts": "e7a909437d836c34939e2709ede99794f81cc8746c6c55b1bbdd36bf352e7664", + "src/main/usage/usage-path-comparison.ts": "07be51caa36e43962fac4cb66c1e0bfd4700f58aeb76eae32cc524fff88625d8", + "src/main/usage/usage-record-coercion.ts": "1b7f9fd8a0c01fbe25f82171d9ceefcbc717e87df3e321fc9514c59c9cfbd7da", + "src/main/usage/usage-rollup-merge.ts": "67cc062058439d659f77a8b47ce1b0ad3e2c323a83d2332aaa75e399de98b433", + "src/main/usage/usage-rollup-records.ts": "eff570d174439c5786175081ffe3ac2aedd960ebfc7e317f9f3cfc4091f27dab", + "src/main/usage/usage-session-breakdown-index.ts": "51725b3480ed078506beeec3105bd95402836e10478bd681cdff95fed080f81d" + }, + "bundleSha256": "9e56032701d379c5e1b5482c49c5734153fbc2735010d0cbc3bc82e44662fe25" + } + }, + "fixture": { + "events": 100000, + "bytes": 20200079, + "sha256": "fa36fd10e36653020c0997890c0c9d9e75f40ee4040adf3a8d79735c4fbe20d5" + }, + "schedule": [ + ["baseline", "current"], + ["current", "baseline"] + ], + "measurements": [ + { + "round": 1, + "order": ["baseline", "current"], + "arm": "baseline", + "stride": 1, + "duringSampleCount": 12, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 2.196832999999998, + "rss": 54837248, + "heapTotal": 7929856, + "heapUsed": 4923952, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 41.750583000000006, + "rss": 85114880, + "heapTotal": 29507584, + "heapUsed": 11458104, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 77.58875, + "rss": 100073472, + "heapTotal": 52396032, + "heapUsed": 17544544, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 115.21895800000001, + "rss": 114868224, + "heapTotal": 92504064, + "heapUsed": 24514776, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 154.876542, + "rss": 137347072, + "heapTotal": 98222080, + "heapUsed": 30184872, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 186.526458, + "rss": 152600576, + "heapTotal": 102416384, + "heapUsed": 34527664, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 223.591625, + "rss": 159465472, + "heapTotal": 109232128, + "heapUsed": 41191320, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 275.32012499999996, + "rss": 171245568, + "heapTotal": 181878784, + "heapUsed": 46842960, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 329.858125, + "rss": 171409408, + "heapTotal": 182140928, + "heapUsed": 46988472, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 372.46854199999996, + "rss": 171507712, + "heapTotal": 182403072, + "heapUsed": 47139504, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 395.50424999999996, + "rss": 171720704, + "heapTotal": 182403072, + "heapUsed": 47282504, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 430.291167, + "rss": 174358528, + "heapTotal": 185024512, + "heapUsed": 49947720, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 460.099042, + "rss": 176521216, + "heapTotal": 186859520, + "heapUsed": 51775528, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "after", + "elapsedMs": 521.9577919999999, + "rss": 198000640, + "heapTotal": 147144704, + "heapUsed": 11731728, + "external": 2455679, + "arrayBuffers": 143819 + } + ], + "sampledRetainedPeakDeltaBytes": 46851576, + "finalRetainedDeltaBytes": 6807776, + "correctness": { + "claimQueries": 100000, + "committedKeys": null, + "ownedKeys": 100000, + "countedEvents": 100000, + "inputTokens": 10000000, + "outputTokens": 1000000, + "totalTokens": 11000000, + "hasDeferredClaims": false, + "projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a" + } + }, + { + "round": 1, + "order": ["baseline", "current"], + "arm": "current", + "stride": 1, + "duringSampleCount": 9, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 2.388917000000003, + "rss": 54624256, + "heapTotal": 7929856, + "heapUsed": 4935392, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 43.105542, + "rss": 69894144, + "heapTotal": 11681792, + "heapUsed": 6351040, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 75.529417, + "rss": 75300864, + "heapTotal": 16728064, + "heapUsed": 7330792, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 110.00737500000001, + "rss": 76808192, + "heapTotal": 18169856, + "heapUsed": 8640192, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 147.38125000000002, + "rss": 78299136, + "heapTotal": 18432000, + "heapUsed": 9275016, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 181.302042, + "rss": 83001344, + "heapTotal": 27344896, + "heapUsed": 9940712, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 215.42712500000002, + "rss": 88145920, + "heapTotal": 29442048, + "heapUsed": 11844616, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 250.383958, + "rss": 89227264, + "heapTotal": 29966336, + "heapUsed": 12414208, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 285.865792, + "rss": 92225536, + "heapTotal": 30752768, + "heapUsed": 13073640, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 339.514083, + "rss": 95862784, + "heapTotal": 32079872, + "heapUsed": 14447032, + "external": 2455471, + "arrayBuffers": 143611 + }, + { + "phase": "after", + "elapsedMs": 349.316458, + "rss": 93257728, + "heapTotal": 29442048, + "heapUsed": 11764560, + "external": 2455471, + "arrayBuffers": 143611 + } + ], + "sampledRetainedPeakDeltaBytes": 9511640, + "finalRetainedDeltaBytes": 6829168, + "correctness": { + "claimQueries": 100000, + "committedKeys": 100000, + "ownedKeys": 100000, + "countedEvents": 100000, + "inputTokens": 10000000, + "outputTokens": 1000000, + "totalTokens": 11000000, + "hasDeferredClaims": false, + "projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a" + } + }, + { + "round": 1, + "order": ["baseline", "current"], + "arm": "baseline", + "stride": 10, + "duringSampleCount": 3, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 2.742457999999999, + "rss": 54558720, + "heapTotal": 7667712, + "heapUsed": 4924000, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 40.256542, + "rss": 67108864, + "heapTotal": 15794176, + "heapUsed": 6373000, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 75.06275, + "rss": 72105984, + "heapTotal": 17022976, + "heapUsed": 7649648, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 111.00004200000001, + "rss": 81936384, + "heapTotal": 26198016, + "heapUsed": 8795656, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "after", + "elapsedMs": 153.181625, + "rss": 88260608, + "heapTotal": 24444928, + "heapUsed": 5941160, + "external": 2455679, + "arrayBuffers": 143819 + } + ], + "sampledRetainedPeakDeltaBytes": 3871656, + "finalRetainedDeltaBytes": 1017160, + "correctness": { + "claimQueries": 100000, + "committedKeys": null, + "ownedKeys": 10000, + "countedEvents": 10000, + "inputTokens": 1000000, + "outputTokens": 100000, + "totalTokens": 1100000, + "hasDeferredClaims": true, + "projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac" + } + }, + { + "round": 1, + "order": ["baseline", "current"], + "arm": "current", + "stride": 10, + "duringSampleCount": 4, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 2.448875000000001, + "rss": 54788096, + "heapTotal": 7929856, + "heapUsed": 4935416, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 39.214, + "rss": 66109440, + "heapTotal": 10813440, + "heapUsed": 5545328, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 74.328625, + "rss": 66650112, + "heapTotal": 10993664, + "heapUsed": 5766000, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 109.42824999999999, + "rss": 66830336, + "heapTotal": 11255808, + "heapUsed": 5927144, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 144.325792, + "rss": 69926912, + "heapTotal": 11419648, + "heapUsed": 6228632, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "after", + "elapsedMs": 165.405959, + "rss": 72056832, + "heapTotal": 10551296, + "heapUsed": 5981088, + "external": 2455471, + "arrayBuffers": 143611 + } + ], + "sampledRetainedPeakDeltaBytes": 1293216, + "finalRetainedDeltaBytes": 1045672, + "correctness": { + "claimQueries": 100000, + "committedKeys": 10000, + "ownedKeys": 10000, + "countedEvents": 10000, + "inputTokens": 1000000, + "outputTokens": 100000, + "totalTokens": 1100000, + "hasDeferredClaims": true, + "projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac" + } + }, + { + "round": 2, + "order": ["current", "baseline"], + "arm": "current", + "stride": 1, + "duringSampleCount": 8, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 2.7266249999999985, + "rss": 54542336, + "heapTotal": 7929856, + "heapUsed": 4935464, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 41.696167, + "rss": 69419008, + "heapTotal": 11157504, + "heapUsed": 6326664, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 76.023, + "rss": 75857920, + "heapTotal": 16465920, + "heapUsed": 7366984, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 111.78200000000001, + "rss": 77578240, + "heapTotal": 17907712, + "heapUsed": 8640264, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 146.21325000000002, + "rss": 78397440, + "heapTotal": 18169856, + "heapUsed": 9204112, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 181.093667, + "rss": 84312064, + "heapTotal": 27344896, + "heapUsed": 9868584, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 216.906709, + "rss": 89128960, + "heapTotal": 29442048, + "heapUsed": 11800592, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 251.848084, + "rss": 89964544, + "heapTotal": 29966336, + "heapUsed": 12519232, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 286.69679199999996, + "rss": 92831744, + "heapTotal": 30752768, + "heapUsed": 13286328, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "after", + "elapsedMs": 315.54558399999996, + "rss": 93945856, + "heapTotal": 29179904, + "heapUsed": 11758736, + "external": 2455471, + "arrayBuffers": 143611 + } + ], + "sampledRetainedPeakDeltaBytes": 8350864, + "finalRetainedDeltaBytes": 6823272, + "correctness": { + "claimQueries": 100000, + "committedKeys": 100000, + "ownedKeys": 100000, + "countedEvents": 100000, + "inputTokens": 10000000, + "outputTokens": 1000000, + "totalTokens": 11000000, + "hasDeferredClaims": false, + "projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a" + } + }, + { + "round": 2, + "order": ["current", "baseline"], + "arm": "baseline", + "stride": 1, + "duringSampleCount": 8, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 2.759208000000001, + "rss": 54788096, + "heapTotal": 7929856, + "heapUsed": 4923952, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 41.3715, + "rss": 85540864, + "heapTotal": 28983296, + "heapUsed": 11037688, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 77.633541, + "rss": 103530496, + "heapTotal": 52133888, + "heapUsed": 17544376, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 114.46741599999999, + "rss": 115310592, + "heapTotal": 92504064, + "heapUsed": 24639432, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 152.397541, + "rss": 140820480, + "heapTotal": 98484224, + "heapUsed": 30596424, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 191.927833, + "rss": 154828800, + "heapTotal": 105037824, + "heapUsed": 36979976, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 222.867541, + "rss": 160776192, + "heapTotal": 110313472, + "heapUsed": 42347640, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 266.556791, + "rss": 172752896, + "heapTotal": 183451648, + "heapUsed": 48259976, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 297.627125, + "rss": 183664640, + "heapTotal": 187383808, + "heapUsed": 52060344, + "external": 2455471, + "arrayBuffers": 143611 + }, + { + "phase": "after", + "elapsedMs": 354.97375, + "rss": 195674112, + "heapTotal": 147406848, + "heapUsed": 11724288, + "external": 2455679, + "arrayBuffers": 143819 + } + ], + "sampledRetainedPeakDeltaBytes": 47136392, + "finalRetainedDeltaBytes": 6800336, + "correctness": { + "claimQueries": 100000, + "committedKeys": null, + "ownedKeys": 100000, + "countedEvents": 100000, + "inputTokens": 10000000, + "outputTokens": 1000000, + "totalTokens": 11000000, + "hasDeferredClaims": false, + "projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a" + } + }, + { + "round": 2, + "order": ["current", "baseline"], + "arm": "current", + "stride": 10, + "duringSampleCount": 4, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 2.1205, + "rss": 54591488, + "heapTotal": 7929856, + "heapUsed": 4935416, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 38.77175, + "rss": 65896448, + "heapTotal": 10813440, + "heapUsed": 5596672, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 74.407583, + "rss": 66158592, + "heapTotal": 10993664, + "heapUsed": 5771488, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 109.03608299999999, + "rss": 66912256, + "heapTotal": 11255808, + "heapUsed": 5941256, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "during", + "elapsedMs": 143.820708, + "rss": 69861376, + "heapTotal": 11419648, + "heapUsed": 6247120, + "external": 2586751, + "arrayBuffers": 274891 + }, + { + "phase": "after", + "elapsedMs": 161.987125, + "rss": 71991296, + "heapTotal": 10551296, + "heapUsed": 5976440, + "external": 2455471, + "arrayBuffers": 143611 + } + ], + "sampledRetainedPeakDeltaBytes": 1311704, + "finalRetainedDeltaBytes": 1041024, + "correctness": { + "claimQueries": 100000, + "committedKeys": 10000, + "ownedKeys": 10000, + "countedEvents": 10000, + "inputTokens": 1000000, + "outputTokens": 100000, + "totalTokens": 1100000, + "hasDeferredClaims": true, + "projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac" + } + }, + { + "round": 2, + "order": ["current", "baseline"], + "arm": "baseline", + "stride": 10, + "duringSampleCount": 3, + "finalOnlyMeasurement": false, + "baseline": { + "phase": "before", + "elapsedMs": 2.370624999999997, + "rss": 54919168, + "heapTotal": 7929856, + "heapUsed": 4924000, + "external": 2455431, + "arrayBuffers": 143611 + }, + "samples": [ + { + "phase": "during", + "elapsedMs": 40.106750000000005, + "rss": 67092480, + "heapTotal": 15532032, + "heapUsed": 6373000, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 74.903292, + "rss": 71827456, + "heapTotal": 16760832, + "heapUsed": 7649648, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "during", + "elapsedMs": 110.34537499999999, + "rss": 81395712, + "heapTotal": 26722304, + "heapUsed": 8851816, + "external": 2586543, + "arrayBuffers": 274683 + }, + { + "phase": "after", + "elapsedMs": 149.984917, + "rss": 86966272, + "heapTotal": 24444928, + "heapUsed": 5941160, + "external": 2455679, + "arrayBuffers": 143819 + } + ], + "sampledRetainedPeakDeltaBytes": 3927816, + "finalRetainedDeltaBytes": 1017160, + "correctness": { + "claimQueries": 100000, + "committedKeys": null, + "ownedKeys": 10000, + "countedEvents": 10000, + "inputTokens": 1000000, + "outputTokens": 100000, + "totalTokens": 1100000, + "hasDeferredClaims": true, + "projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac" + } + } + ] +} diff --git a/docs/bug-reproductions/usage-scan-append/codex-reader.md b/docs/bug-reproductions/usage-scan-append/codex-reader.md new file mode 100644 index 00000000000..83ada82617f --- /dev/null +++ b/docs/bug-reproductions/usage-scan-append/codex-reader.md @@ -0,0 +1,103 @@ +# Verified transcript reads and Codex usage memory + +Codex could verify one file generation, reopen a replacement, and publish old +totals under the replacement's metadata. A short parse could also publish a +partial result as a complete file. Unchanged refreshes then reused those totals. + +The shared JSONL reader now pins verification, parsing and checkpoint creation +to one descriptor and the initial file size. It proves complete range consumption +and validates generation and byte evidence before publishing. Stable short reads +propagate errors; only evidence that the file changed permits a retry. Each failed +attempt closes its handle. A successful attempt closes before ownership publication +and aggregate work. Metadata-less caches reparse once. + +Codex folds owned records into the existing session/day accumulator while reading. +It commits ownership after validation, and lets unchanged forks reclaim keys that +an owner actually loses. Deferred records still advance cumulative token context. +This removes intermediate event arrays while preserving all history and results. + +## Reproduce and verify + +```sh +ORCA_BACKGROUND_LAUNCH=1 pnpm test src/main/codex-usage src/main/usage +ORCA_BACKGROUND_LAUNCH=1 node config/scripts/codex-usage-parser-memory-benchmark.mjs --baseline-ref 51e7181850b022bae2d41091fe1229accef4bd50 --events 100000 --pairs 2 --sample-ms 35 --output docs/bug-reproductions/usage-scan-append/codex-parser-memory.json +``` + +The original [memory comparison](./codex-parser-memory.json) bundles the full parser +dependency graph from the named baseline and current worktree. Two counterbalanced +rounds use fresh Node workers and record source/tool hashes and raw forced-GC +snapshots. Every arm preserves identical projections, owned keys and token totals. + +| Owned records | Baseline sampled retained heap | Current sampled retained heap | +| ------------------------- | -----------------------------: | ----------------------------: | +| 100,000 | 44.7โ€“45.0 MiB | 8.0โ€“9.1 MiB | +| 10,000, with 90% deferred | 3.69โ€“3.75 MiB | 1.23โ€“1.25 MiB | + +Final result retention stays approximately 6.49 โ†’ 6.51 MiB for all owned records: +the result still contains the full history. Timer samples miss synchronous peaks, +and forced GC affects timing; these figures measure sampled retained heap rather +than total process memory or speed. + +Real-file regressions cover replacement, truncation, rewrite, suffix resume, +partial tails, stable short reads, failed ownership publication, old-cache upgrade +and handle cleanup. Local checks used macOS; CI covers Linux and Windows. +The shared aggregation compatibility suites also passed for Muse and OpenCode. + +Head and committed-boundary digests support the append-only contract. They cannot +prove arbitrary middle-only rewrites during growth. Small changed files receive +a full verification reread; unchanged files read no transcript bytes. + +The companion Claude PR adopts this reader. The original integrated validation +and benchmarks were recorded from a combined worktree; production source remains +identical when split into PRs. + +## Adversarial hardening + +Tiny files now use bounded descriptor reads and exact byte comparison for both +passes. This avoids creating a stream without reducing verification. Exact bigint +device/inode values prevent large Windows file IDs from colliding after numeric +rounding; unsafe persisted offsets fall back to a complete parse. + +Discovery distinguishes a directory that disappeared from one that cannot be +read. Missing children leave readable siblings available. Permission and I/O +errors abort the refresh, retaining the previous verified totals and showing the +existing error state rather than publishing incomplete totals. + +A [counterbalanced ten-round comparison](./shared-reader-small-files.json) of the published reader and hardened +reader over 1,000 tiny files measured median scanner cold time 85.024 โ†’ 80.529 ms +and changed time 91.512 โ†’ 84.092 ms. Both arms read the same bytes twice; streams +fell from 1,000 to zero. Unchanged scans read zero bytes in both arms and measured +7.808 โ†’ 8.241 ms. These page-cache-warm macOS timings are observations, not CI +thresholds or a claim about the original pre-verification reader. + +Regression contracts exercise short reads, exact comparison, unsafe offsets, +large file identities, fractional timestamps, and discovery failures. The +original retained-heap artifact above predates this hardening; its source hashes +describe that earlier measurement, rather than the final reader. + +## Further risk reduction + +The [refreshed memory comparison](./codex-parser-memory-final.json) captures the +final parser and reader source. Its two counterbalanced rounds use eight fresh +Node workers and preserve exact output parity with the original baseline. + +| Owned records | Original sampled retained heap | Final sampled retained heap | +| ------------------------- | -----------------------------: | --------------------------: | +| 100,000 | 43.87โ€“44.54 MiB | 8.07โ€“8.30 MiB | +| 10,000, with 90% deferred | 4.53โ€“4.58 MiB | 1.231โ€“1.235 MiB | + +The median reductions are approximately 81.5% and 72.9%. Final output retention +stays similar or slightly higher: 6.49 โ†’ 6.53โ€“6.55 MiB for owned events, and +0.975 โ†’ 1.005โ€“1.014 MiB for deferred events. These are forced-GC samples, not +absolute peaks, full-app memory, or speed measurements. + +Real-file lifetime contracts cover both tiny and streamed transcripts. They +prove the descriptor is closed before an ownership callback replaces the path, +the verified old projection keeps its captured metadata, and the next parse +observes the replacement. Failure cases check cleanup when validation fails or +the first close attempt rejects before reaching the native close. + +This shortens Windows replacement contention to reading and validation; parsing +can still include asynchronous attribution while the descriptor is pinned. +Ordinary append writes remain compatible. Tiny verification still reads the full +file twice, and bounded append evidence retains its append-only assumption. diff --git a/docs/bug-reproductions/usage-scan-append/shared-reader-small-files.json b/docs/bug-reproductions/usage-scan-append/shared-reader-small-files.json new file mode 100644 index 00000000000..b0e20292b90 --- /dev/null +++ b/docs/bug-reproductions/usage-scan-append/shared-reader-small-files.json @@ -0,0 +1,984 @@ +{ + "schemaVersion": 1, + "scenario": "1000-small-files", + "scope": "Shared JSONL reader and file metadata hardening, exercised through the combined Claude scanner; the current Claude projection implementation is identical in both arms.", + "capturedAtUtc": "2026-10-09", + "environment": { + "node": "v24.20.0", + "platform": "darwin", + "osRelease": "25.3.0", + "arch": "arm64", + "cpuModel": "Apple M4 Max" + }, + "rounds": 10, + "schedule": "Alternating before/after and after/before; five rounds in each order; tests and builds held during this final capture.", + "baselineCommit": "03a2ecaa00138ee44ad0dbb9dbd957cfe1a5e3f8", + "baselineModules": [ + "src/main/claude-usage/transcript-record-parser.ts", + "src/main/usage/jsonl-file-checkpoint.ts", + "src/main/usage/jsonl-file-snapshot.ts", + "src/main/usage/jsonl-line-offsets.ts" + ], + "baselineHashesVerifiedAgainstGit": true, + "currentSourceHashesVerifiedAgainstFrozenWorktree": true, + "sourceFingerprintLabels": { + "HEAD": "The four pinned published baseline modules supplied as an overlay; this is the existing loader label, not the current Git HEAD.", + "worktree": "Frozen combined-worktree modules, held identical in both arms except the four baseline overlays." + }, + "methodology": { + "cold": "1,000 freshly written tiny transcripts; empty scanner cache; filesystem page cache warm.", + "changed": "Append a duplicate usage row with higher token maxima to each file, then scan from the unchanged cache.", + "warm": "Reuse each arm's cold persisted cache without changing files; no transcript content reads.", + "durationMs": "Production scan only, excluding fixture writes and cache serialization.", + "totalDurationMs": "Production scan plus JSON cache serialization, excluding hydration and fixture writes.", + "bytesRead": "All raw bytes reported by the existing benchmark loader, including verification reads.", + "readCount": "Logical filesystem API read records, including streams; this does not claim an OS syscall count.", + "parity": "Exact sessions, daily aggregates, and per-file projections in cold/changed phases across arms, excluding volatile stat values and additive generation/resume metadata.", + "median": "Average of the middle two values for the ten samples; the temporary capture script's upper-middle summaries are retained separately.", + "reproduction": [ + "Use loadClaudeUsageBenchmarkScanner and measureClaudeUsageBenchmarkScan from config/scripts/claude-usage-benchmark-scanner.mjs in the combined Claude/shared-reader worktree.", + "Load only the four baselineModules from baselineCommit into the before source-overlay map; load every other module once into a shared worktree-source map used by both arms.", + "Create an isolated agent home containing 1,000 JSONL files, one keyed assistant usage row per file, with input_tokens=100 and output_tokens=10.", + "Measure a cold scan, an unchanged scan from its cache, then append the same keyed row with input_tokens=200 and measure the changed scan. Exclude batched fixture writes from timers.", + "Alternate arm order for ten rounds, assert exact projected behavior parity for cold and changed phases, and assert that every unchanged scan reads zero transcript bytes.", + "Compare byte counts and stream counts separately from duration, and record all raw samples and source fingerprints." + ] + }, + "captureScriptSha256": "d4bac98e4bd15f0ba4760646b9c63cd69160b4f5df7d09ae7b22b101ea0fc9c5", + "captureScriptStatus": "Local measurement adapter reused the existing loader; no separate benchmark framework or production tool was added.", + "toolingSha256": { + "config/scripts/claude-usage-benchmark-scanner.mjs": "ffb6e5defb921f2ab35726a716f5e731dec439ead53cc2f3a192c7bef4df45d3", + "config/scripts/counterbalanced-benchmark-schedule.mjs": "bc35b40e954fa7ddec0225ed375bd3b113c2e05b8554bc50734d14f89987294d" + }, + "historyBytes": 307780, + "changedSnapshotBytes": 615560, + "exactColdChangedProjectionParity": true, + "medians": { + "before": { + "cold": { + "durationMs": 85.02441700000008, + "cacheSerializationMs": 1.705395499999895, + "totalDurationMs": 86.68222949999995, + "bytesRead": 615560.0, + "directReadBytes": 307780.0, + "readCount": 2000.0, + "streamCount": 1000.0, + "cachedStateBytes": 1508259.0 + }, + "warm": { + "durationMs": 7.808062500000233, + "cacheSerializationMs": 1.4102915000000849, + "totalDurationMs": 9.195042000000285, + "bytesRead": 0.0, + "directReadBytes": 0.0, + "readCount": 0.0, + "streamCount": 0.0, + "cachedStateBytes": 1508259.0 + }, + "changed": { + "durationMs": 91.5124165000002, + "cacheSerializationMs": 1.5766254999998637, + "totalDurationMs": 93.02370800000017, + "bytesRead": 1231120.0, + "directReadBytes": 615560.0, + "readCount": 2000.0, + "streamCount": 1000.0, + "cachedStateBytes": 1508232.0 + } + }, + "after": { + "cold": { + "durationMs": 80.5293959999999, + "cacheSerializationMs": 1.6749375000002829, + "totalDurationMs": 82.21989599999984, + "bytesRead": 615560.0, + "directReadBytes": 615560.0, + "readCount": 2000.0, + "streamCount": 0.0, + "cachedStateBytes": 1508252.0 + }, + "warm": { + "durationMs": 8.240729500000043, + "cacheSerializationMs": 1.4088955000003125, + "totalDurationMs": 9.655500500000016, + "bytesRead": 0.0, + "directReadBytes": 0.0, + "readCount": 0.0, + "streamCount": 0.0, + "cachedStateBytes": 1508252.0 + }, + "changed": { + "durationMs": 84.09189549999974, + "cacheSerializationMs": 1.587229000000093, + "totalDurationMs": 85.75574999999981, + "bytesRead": 1231120.0, + "directReadBytes": 1231120.0, + "readCount": 2000.0, + "streamCount": 0.0, + "cachedStateBytes": 1508243.0 + } + } + }, + "durationChangePercent": { + "cold": -5.286741336903466, + "warm": 5.541285049905742, + "changed": -8.108758662274473 + }, + "samples": [ + { + "arm": "before", + "round": 0, + "cold": { + "durationMs": 392.049709, + "cacheSerializationMs": 1.6989579999999478, + "totalDurationMs": 393.74866699999995, + "bytesRead": 615560, + "directReadBytes": 307780, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508277 + }, + "warm": { + "durationMs": 48.59562500000004, + "cacheSerializationMs": 1.523916999999983, + "totalDurationMs": 50.119542000000024, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508277 + }, + "changed": { + "durationMs": 233.39462500000002, + "cacheSerializationMs": 1.4007910000000265, + "totalDurationMs": 234.79541600000005, + "bytesRead": 1231120, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508233 + } + }, + { + "arm": "after", + "round": 0, + "cold": { + "durationMs": 132.50262499999985, + "cacheSerializationMs": 1.6758330000000115, + "totalDurationMs": 134.17845799999986, + "bytesRead": 615560, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508243 + }, + "warm": { + "durationMs": 13.144000000000005, + "cacheSerializationMs": 1.4084580000001097, + "totalDurationMs": 14.552458000000115, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508243 + }, + "changed": { + "durationMs": 111.28033299999993, + "cacheSerializationMs": 1.5340830000000096, + "totalDurationMs": 112.81441599999994, + "bytesRead": 1231120, + "directReadBytes": 1231120, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508271 + } + }, + { + "arm": "after", + "round": 1, + "cold": { + "durationMs": 174.1105, + "cacheSerializationMs": 1.588083000000097, + "totalDurationMs": 175.6985830000001, + "bytesRead": 615560, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508241 + }, + "warm": { + "durationMs": 12.503875000000107, + "cacheSerializationMs": 1.357582999999977, + "totalDurationMs": 13.861458000000084, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508241 + }, + "changed": { + "durationMs": 127.85537499999987, + "cacheSerializationMs": 1.5461250000000746, + "totalDurationMs": 129.40149999999994, + "bytesRead": 1231120, + "directReadBytes": 1231120, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508289 + } + }, + { + "arm": "before", + "round": 1, + "cold": { + "durationMs": 126.94487499999991, + "cacheSerializationMs": 1.7684590000001208, + "totalDurationMs": 128.71333400000003, + "bytesRead": 615560, + "directReadBytes": 307780, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508223 + }, + "warm": { + "durationMs": 8.401833000000124, + "cacheSerializationMs": 1.3886669999999413, + "totalDurationMs": 9.790500000000065, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508223 + }, + "changed": { + "durationMs": 114.26195800000005, + "cacheSerializationMs": 1.8519999999998618, + "totalDurationMs": 116.11395799999991, + "bytesRead": 1231120, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508231 + } + }, + { + "arm": "before", + "round": 2, + "cold": { + "durationMs": 91.58558300000004, + "cacheSerializationMs": 1.4894590000003518, + "totalDurationMs": 93.0750420000004, + "bytesRead": 615560, + "directReadBytes": 307780, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508261 + }, + "warm": { + "durationMs": 8.029250000000047, + "cacheSerializationMs": 1.4045000000000982, + "totalDurationMs": 9.433750000000146, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508261 + }, + "changed": { + "durationMs": 95.63120799999979, + "cacheSerializationMs": 1.6717079999998532, + "totalDurationMs": 97.30291599999964, + "bytesRead": 1231120, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508229 + } + }, + { + "arm": "after", + "round": 2, + "cold": { + "durationMs": 109.59825000000001, + "cacheSerializationMs": 2.067250000000058, + "totalDurationMs": 111.66550000000007, + "bytesRead": 615560, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508283 + }, + "warm": { + "durationMs": 13.671583000000282, + "cacheSerializationMs": 1.3706250000000182, + "totalDurationMs": 15.0422080000003, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508283 + }, + "changed": { + "durationMs": 108.32254200000034, + "cacheSerializationMs": 1.7173749999997199, + "totalDurationMs": 110.03991700000006, + "bytesRead": 1231120, + "directReadBytes": 1231120, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508251 + } + }, + { + "arm": "after", + "round": 3, + "cold": { + "durationMs": 80.29837499999985, + "cacheSerializationMs": 1.6784579999998641, + "totalDurationMs": 81.97683299999971, + "bytesRead": 615560, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508213 + }, + "warm": { + "durationMs": 8.142833000000337, + "cacheSerializationMs": 1.4027909999999792, + "totalDurationMs": 9.545624000000316, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508213 + }, + "changed": { + "durationMs": 88.40525000000025, + "cacheSerializationMs": 1.6283330000001115, + "totalDurationMs": 90.03358300000036, + "bytesRead": 1231120, + "directReadBytes": 1231120, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508237 + } + }, + { + "arm": "before", + "round": 3, + "cold": { + "durationMs": 75.91833399999996, + "cacheSerializationMs": 1.7118329999998423, + "totalDurationMs": 77.6301669999998, + "bytesRead": 615560, + "directReadBytes": 307780, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508261 + }, + "warm": { + "durationMs": 7.40179100000023, + "cacheSerializationMs": 1.40166599999975, + "totalDurationMs": 8.80345699999998, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508261 + }, + "changed": { + "durationMs": 86.82458300000008, + "cacheSerializationMs": 1.5110839999997552, + "totalDurationMs": 88.33566699999983, + "bytesRead": 1231120, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508229 + } + }, + { + "arm": "before", + "round": 4, + "cold": { + "durationMs": 86.88720899999998, + "cacheSerializationMs": 1.7541670000000522, + "totalDurationMs": 88.64137600000004, + "bytesRead": 615560, + "directReadBytes": 307780, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508281 + }, + "warm": { + "durationMs": 7.466167000000041, + "cacheSerializationMs": 1.4322910000000775, + "totalDurationMs": 8.898458000000119, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508281 + }, + "changed": { + "durationMs": 91.41150000000016, + "cacheSerializationMs": 1.5426669999997102, + "totalDurationMs": 92.95416699999987, + "bytesRead": 1231120, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508227 + } + }, + { + "arm": "after", + "round": 4, + "cold": { + "durationMs": 80.76041699999996, + "cacheSerializationMs": 1.702541999999994, + "totalDurationMs": 82.46295899999996, + "bytesRead": 615560, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508227 + }, + "warm": { + "durationMs": 8.741708000000017, + "cacheSerializationMs": 1.4854580000001079, + "totalDurationMs": 10.227166000000125, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508227 + }, + "changed": { + "durationMs": 77.78320800000029, + "cacheSerializationMs": 1.4414170000000013, + "totalDurationMs": 79.22462500000029, + "bytesRead": 1231120, + "directReadBytes": 1231120, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508195 + } + }, + { + "arm": "after", + "round": 5, + "cold": { + "durationMs": 75.24266699999998, + "cacheSerializationMs": 1.6534579999997732, + "totalDurationMs": 76.89612499999976, + "bytesRead": 615560, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508281 + }, + "warm": { + "durationMs": 8.298541999999998, + "cacheSerializationMs": 1.406333999999788, + "totalDurationMs": 9.704875999999786, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508281 + }, + "changed": { + "durationMs": 76.96762499999977, + "cacheSerializationMs": 1.446374999999989, + "totalDurationMs": 78.41399999999976, + "bytesRead": 1231120, + "directReadBytes": 1231120, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508247 + } + }, + { + "arm": "before", + "round": 5, + "cold": { + "durationMs": 83.16162500000019, + "cacheSerializationMs": 1.561457999999675, + "totalDurationMs": 84.72308299999986, + "bytesRead": 615560, + "directReadBytes": 307780, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508211 + }, + "warm": { + "durationMs": 7.586875000000418, + "cacheSerializationMs": 1.3694590000000062, + "totalDurationMs": 8.956334000000425, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508211 + }, + "changed": { + "durationMs": 81.44929200000024, + "cacheSerializationMs": 1.6840000000001965, + "totalDurationMs": 83.13329200000044, + "bytesRead": 1231120, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508301 + } + }, + { + "arm": "before", + "round": 6, + "cold": { + "durationMs": 83.09058300000015, + "cacheSerializationMs": 1.5731669999995574, + "totalDurationMs": 84.66374999999971, + "bytesRead": 615560, + "directReadBytes": 307780, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508219 + }, + "warm": { + "durationMs": 7.298624999999447, + "cacheSerializationMs": 1.4347920000000158, + "totalDurationMs": 8.733416999999463, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508219 + }, + "changed": { + "durationMs": 84.96458299999995, + "cacheSerializationMs": 1.6105840000000171, + "totalDurationMs": 86.57516699999996, + "bytesRead": 1231120, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508219 + } + }, + { + "arm": "after", + "round": 6, + "cold": { + "durationMs": 70.85174999999981, + "cacheSerializationMs": 1.4307080000007772, + "totalDurationMs": 72.28245800000059, + "bytesRead": 615560, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508265 + }, + "warm": { + "durationMs": 7.037790999999743, + "cacheSerializationMs": 1.4527090000001408, + "totalDurationMs": 8.490499999999884, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508265 + }, + "changed": { + "durationMs": 71.76158299999952, + "cacheSerializationMs": 1.6893340000005992, + "totalDurationMs": 73.45091700000012, + "bytesRead": 1231120, + "directReadBytes": 1231120, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508197 + } + }, + { + "arm": "after", + "round": 7, + "cold": { + "durationMs": 71.874417, + "cacheSerializationMs": 1.7616250000000946, + "totalDurationMs": 73.63604200000009, + "bytesRead": 615560, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508241 + }, + "warm": { + "durationMs": 7.877457999999933, + "cacheSerializationMs": 1.4134170000006634, + "totalDurationMs": 9.290875000000597, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508241 + }, + "changed": { + "durationMs": 71.59512500000073, + "cacheSerializationMs": 1.6312079999997877, + "totalDurationMs": 73.22633300000052, + "bytesRead": 1231120, + "directReadBytes": 1231120, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508171 + } + }, + { + "arm": "before", + "round": 7, + "cold": { + "durationMs": 76.68987500000003, + "cacheSerializationMs": 1.752542000000176, + "totalDurationMs": 78.4424170000002, + "bytesRead": 615560, + "directReadBytes": 307780, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508263 + }, + "warm": { + "durationMs": 6.472415999999612, + "cacheSerializationMs": 1.423917000000074, + "totalDurationMs": 7.896332999999686, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508263 + }, + "changed": { + "durationMs": 93.93520900000021, + "cacheSerializationMs": 1.528958999999304, + "totalDurationMs": 95.46416799999952, + "bytesRead": 1231120, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508253 + } + }, + { + "arm": "before", + "round": 8, + "cold": { + "durationMs": 99.58229100000062, + "cacheSerializationMs": 2.150957999999264, + "totalDurationMs": 101.73324899999989, + "bytesRead": 615560, + "directReadBytes": 307780, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508241 + }, + "warm": { + "durationMs": 8.437750000000051, + "cacheSerializationMs": 1.4160830000000715, + "totalDurationMs": 9.853833000000122, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508241 + }, + "changed": { + "durationMs": 91.61333300000024, + "cacheSerializationMs": 1.4799160000002303, + "totalDurationMs": 93.09324900000047, + "bytesRead": 1231120, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508269 + } + }, + { + "arm": "after", + "round": 8, + "cold": { + "durationMs": 118.9740839999995, + "cacheSerializationMs": 1.6740420000005543, + "totalDurationMs": 120.64812600000005, + "bytesRead": 615560, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508267 + }, + "warm": { + "durationMs": 7.485000000000582, + "cacheSerializationMs": 1.4093330000005153, + "totalDurationMs": 8.894333000001097, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508267 + }, + "changed": { + "durationMs": 83.58533299999999, + "cacheSerializationMs": 1.4950419999995574, + "totalDurationMs": 85.08037499999955, + "bytesRead": 1231120, + "directReadBytes": 1231120, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508251 + } + }, + { + "arm": "after", + "round": 9, + "cold": { + "durationMs": 76.86124999999993, + "cacheSerializationMs": 1.6329170000008162, + "totalDurationMs": 78.49416700000074, + "bytesRead": 615560, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508261 + }, + "warm": { + "durationMs": 8.182917000000089, + "cacheSerializationMs": 1.4232080000001588, + "totalDurationMs": 9.606125000000247, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508261 + }, + "changed": { + "durationMs": 84.59845799999948, + "cacheSerializationMs": 1.8326670000005834, + "totalDurationMs": 86.43112500000007, + "bytesRead": 1231120, + "directReadBytes": 1231120, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508239 + } + }, + { + "arm": "before", + "round": 9, + "cold": { + "durationMs": 79.30645899999945, + "cacheSerializationMs": 1.5558329999994385, + "totalDurationMs": 80.86229199999889, + "bytesRead": 615560, + "directReadBytes": 307780, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508257 + }, + "warm": { + "durationMs": 8.131749999999556, + "cacheSerializationMs": 1.381416999999601, + "totalDurationMs": 9.513166999999157, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508257 + }, + "changed": { + "durationMs": 90.45725000000039, + "cacheSerializationMs": 1.6439999999993233, + "totalDurationMs": 92.10124999999971, + "bytesRead": 1231120, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508269 + } + } + ], + "capturedUpperMiddleSummaries": { + "before": { + "cold": { + "durationMs": 86.88720899999998, + "cacheSerializationMs": 1.7118329999998423, + "totalDurationMs": 88.64137600000004, + "bytesRead": 615560, + "directReadBytes": 307780, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508261 + }, + "warm": { + "durationMs": 8.029250000000047, + "cacheSerializationMs": 1.4160830000000715, + "totalDurationMs": 9.433750000000146, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508261 + }, + "changed": { + "durationMs": 91.61333300000024, + "cacheSerializationMs": 1.6105840000000171, + "totalDurationMs": 93.09324900000047, + "bytesRead": 1231120, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 1000, + "cachedStateBytes": 1508233 + } + }, + "after": { + "cold": { + "durationMs": 80.76041699999996, + "cacheSerializationMs": 1.6758330000000115, + "totalDurationMs": 82.46295899999996, + "bytesRead": 615560, + "directReadBytes": 615560, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508261 + }, + "warm": { + "durationMs": 8.298541999999998, + "cacheSerializationMs": 1.4093330000005153, + "totalDurationMs": 9.704875999999786, + "bytesRead": 0, + "directReadBytes": 0, + "readCount": 0, + "streamCount": 0, + "cachedStateBytes": 1508261 + }, + "changed": { + "durationMs": 84.59845799999948, + "cacheSerializationMs": 1.6283330000001115, + "totalDurationMs": 86.43112500000007, + "bytesRead": 1231120, + "directReadBytes": 1231120, + "readCount": 2000, + "streamCount": 0, + "cachedStateBytes": 1508247 + } + } + }, + "sourceFingerprints": { + "before": { + "src/main/claude-usage/scanner.ts": { + "source": "worktree", + "sha256": "ef42add20f63152cf2ebe106e0794119b9c388c763736c8450b559ae7c2418f5" + }, + "src/main/claude-usage/transcript-file-discovery.ts": { + "source": "worktree", + "sha256": "54c84d0b99b45c0fa13e589a1bcec2b479cb01e8b17bcbddbb65a4bc85a03401" + }, + "src/main/claude-usage/transcript-projection-integrity.ts": { + "source": "worktree", + "sha256": "f4efccd6f1be0603a90174f66ac961d07ea5bf1c1fbebe015b0537deee1c58f2" + }, + "src/main/claude-usage/transcript-record-parser.ts": { + "source": "HEAD", + "sha256": "d4fab89434c1b6c60480a561310542246be34ef86bc13da0340c1dfdbb269854" + }, + "src/main/claude-usage/transcript-resume-projection.ts": { + "source": "worktree", + "sha256": "b2ed0a589b60fdccb02c72fa234368853d32be080a84ee3bcc93f5834478a153" + }, + "src/main/claude-usage/transcript-token-checkpoint.ts": { + "source": "worktree", + "sha256": "e96cc4aa9b899cad90aee462f053f483f5bb010dea99de1ee11a9f8fd8a2d0f2" + }, + "src/main/claude-usage/transcript-usage-projection.ts": { + "source": "worktree", + "sha256": "56e2fe017f806670595dae766b6291e09b05b321b83e4b83e1ae5166bab2041c" + }, + "src/main/claude-usage/usage-aggregation.ts": { + "source": "worktree", + "sha256": "be4dad6f41eaf720252137b16b9a96a2cfdd2e73b05eee097993653ce6c5e509" + }, + "src/main/claude-usage/worktree-attribution.ts": { + "source": "worktree", + "sha256": "7450e5ca7d8dcb66d1f5003a3e445c29f05c1416c6cd63aeb3f5f74804bc6213" + }, + "src/main/usage/jsonl-file-checkpoint.ts": { + "source": "HEAD", + "sha256": "8eb579f356e1bb23b36319f00b3d4603fff80a963222dcb8a22331319cab3a03" + }, + "src/main/usage/jsonl-file-snapshot.ts": { + "source": "HEAD", + "sha256": "1bd98ac194f73097cd6c204238fe1db01e1e0d60a604e93c1d7841725971fe0d" + }, + "src/main/usage/jsonl-line-offsets.ts": { + "source": "HEAD", + "sha256": "f119465fad3771acd72b128853e84eed9165f0f74b75743ad662aa5bab991273" + } + }, + "after": { + "src/main/claude-usage/scanner.ts": { + "source": "worktree", + "sha256": "ef42add20f63152cf2ebe106e0794119b9c388c763736c8450b559ae7c2418f5" + }, + "src/main/claude-usage/transcript-file-discovery.ts": { + "source": "worktree", + "sha256": "54c84d0b99b45c0fa13e589a1bcec2b479cb01e8b17bcbddbb65a4bc85a03401" + }, + "src/main/claude-usage/transcript-projection-integrity.ts": { + "source": "worktree", + "sha256": "f4efccd6f1be0603a90174f66ac961d07ea5bf1c1fbebe015b0537deee1c58f2" + }, + "src/main/claude-usage/transcript-record-parser.ts": { + "source": "worktree", + "sha256": "6598c02e38ea1776ef5ef926ca424dbc5880f27e7ff03b651cf8f60b791996f6" + }, + "src/main/claude-usage/transcript-resume-projection.ts": { + "source": "worktree", + "sha256": "b2ed0a589b60fdccb02c72fa234368853d32be080a84ee3bcc93f5834478a153" + }, + "src/main/claude-usage/transcript-token-checkpoint.ts": { + "source": "worktree", + "sha256": "e96cc4aa9b899cad90aee462f053f483f5bb010dea99de1ee11a9f8fd8a2d0f2" + }, + "src/main/claude-usage/transcript-usage-projection.ts": { + "source": "worktree", + "sha256": "56e2fe017f806670595dae766b6291e09b05b321b83e4b83e1ae5166bab2041c" + }, + "src/main/claude-usage/usage-aggregation.ts": { + "source": "worktree", + "sha256": "be4dad6f41eaf720252137b16b9a96a2cfdd2e73b05eee097993653ce6c5e509" + }, + "src/main/claude-usage/worktree-attribution.ts": { + "source": "worktree", + "sha256": "7450e5ca7d8dcb66d1f5003a3e445c29f05c1416c6cd63aeb3f5f74804bc6213" + }, + "src/main/usage/jsonl-file-checkpoint.ts": { + "source": "worktree", + "sha256": "b7f307564989dabaf1dfda095a26c7bd9f0683e0efcabe79a010c7b58fa1a2d4" + }, + "src/main/usage/jsonl-file-snapshot.ts": { + "source": "worktree", + "sha256": "b1b8869db92c08c762160acba7570aabff25879649cb6ffa3826912ee35deffe" + }, + "src/main/usage/jsonl-line-offsets.ts": { + "source": "worktree", + "sha256": "f3574adfad4549c6330e2ac52760c91be5a8cbb9bf8e1beb440e3226087074db" + } + } + }, + "limits": [ + "One macOS host and page-cache-warm fixtures; durations are evidence for this scenario, not a platform-independent performance contract.", + "This comparison is against the published incremental reader implementation, not the pre-PR full-scan implementation.", + "Both arms still read each small cold/changed snapshot twice; unchanged scans read zero transcript bytes.", + "Warm duration rose by about 0.43 ms over 1,000 files in this capture; exact bigint metadata adds CPU work without adding stat calls.", + "Large-file append-only checkpoints retain their documented limitation for arbitrary middle rewrites during growth." + ] +} diff --git a/src/main/codex-usage/codex-rollout-file-lifetime.test.ts b/src/main/codex-usage/codex-rollout-file-lifetime.test.ts new file mode 100644 index 00000000000..6fc7c16236a --- /dev/null +++ b/src/main/codex-usage/codex-rollout-file-lifetime.test.ts @@ -0,0 +1,145 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { mkdtempSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs' +import type * as NodeFsPromises from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { parseCodexUsageFile } from './codex-rollout-file-parse' + +const { openedHandles, closedHandles, failNextClose } = vi.hoisted(() => ({ + openedHandles: new Array(), + closedHandles: new Set(), + failNextClose: { current: false } +})) + +vi.mock('node:fs/promises', async () => { + const actual = await vi.importActual('node:fs/promises') + return { + ...actual, + open: async (...args: Parameters) => { + const handle = await actual.open(...args) + openedHandles.push(handle) + const closeHandle = handle.close.bind(handle) + vi.spyOn(handle, 'close').mockImplementation(async () => { + if (failNextClose.current) { + failNextClose.current = false + throw new Error('descriptor close failed') + } + await closeHandle() + closedHandles.add(handle) + }) + return handle + } + } +}) + +function rollout(sessionId: string, recordCount: number): string { + const records = [JSON.stringify({ type: 'session_meta', payload: { id: sessionId } })] + for (let index = 0; index < recordCount; index++) { + records.push( + JSON.stringify({ + timestamp: new Date(Date.UTC(2026, 4, 26, 12, index)).toISOString(), + type: 'event_msg', + payload: { + type: 'token_count', + info: { + model: 'gpt-5-codex', + last_token_usage: { input_tokens: 1, total_tokens: 1 }, + total_token_usage: { input_tokens: index + 1, total_tokens: index + 1 } + } + } + }) + ) + } + return `${records.join('\n')}\n` +} + +let directory: string + +beforeEach(() => { + directory = mkdtempSync(join(tmpdir(), 'orca-codex-rollout-lifetime-')) + openedHandles.length = 0 + closedHandles.clear() + failNextClose.current = false +}) + +afterEach(() => { + rmSync(directory, { recursive: true, force: true }) + vi.clearAllMocks() +}) + +describe('Codex rollout descriptor lifetime', () => { + it.each([3, 100])( + 'closes the verified %i-record rollout before publishing ownership', + async (recordCount) => { + const rolloutPath = join(directory, 'rollout.jsonl') + const replacementPath = join(directory, 'replacement.tmp') + writeFileSync(rolloutPath, rollout('verified-session', recordCount)) + const originalStat = statSync(rolloutPath, { bigint: true }) + writeFileSync(replacementPath, rollout('replacement-session', 2)) + let committedKeys = 0 + + const verified = await parseCodexUsageFile(rolloutPath, () => null, { + commitEventKey: () => { + expect(closedHandles.size).toBe(openedHandles.length) + expect(openedHandles.every((handle) => handle.fd === -1)).toBe(true) + if (committedKeys++ === 0) { + renameSync(replacementPath, rolloutPath) + } + } + }) + + expect(committedKeys).toBe(recordCount) + expect(verified.sessions.map(({ sessionId, eventCount }) => [sessionId, eventCount])).toEqual( + [['verified-session', recordCount]] + ) + expect(verified.size).toBe(Number(originalStat.size)) + expect(verified.physicalFileId).toBe( + originalStat.ino === 0n ? null : `${originalStat.dev}:${originalStat.ino}` + ) + expect(verified.parseResumeState !== null).toBe(recordCount === 100) + const refreshed = await parseCodexUsageFile( + rolloutPath, + () => null, + verified.parseResumeState + ? { resume: { state: verified.parseResumeState, previous: verified } } + : {} + ) + expect( + refreshed.sessions.map(({ sessionId, eventCount }) => [sessionId, eventCount]) + ).toEqual([['replacement-session', 2]]) + expect(closedHandles.size).toBe(openedHandles.length) + } + ) + + it('keeps the descriptor closed when publishing ownership fails', async () => { + const rolloutPath = join(directory, 'rollout.jsonl') + writeFileSync(rolloutPath, rollout('failed-claims', 3)) + + await expect( + parseCodexUsageFile(rolloutPath, () => null, { + commitEventKey: () => { + expect(closedHandles.size).toBe(openedHandles.length) + throw new Error('ownership publication failed') + } + }) + ).rejects.toThrow('ownership publication failed') + expect(openedHandles).toHaveLength(1) + expect(closedHandles.size).toBe(1) + expect(openedHandles[0]?.fd).toBe(-1) + }) + + it('publishes no ownership if closing the verified descriptor fails', async () => { + const rolloutPath = join(directory, 'rollout.jsonl') + writeFileSync(rolloutPath, rollout('close-failure', 3)) + const commitEventKey = vi.fn() + failNextClose.current = true + + await expect(parseCodexUsageFile(rolloutPath, () => null, { commitEventKey })).rejects.toThrow( + 'descriptor close failed' + ) + expect(commitEventKey).not.toHaveBeenCalled() + expect(openedHandles).toHaveLength(1) + expect(closedHandles.size).toBe(1) + expect(openedHandles[0]?.fd).toBe(-1) + }) +}) diff --git a/src/main/codex-usage/codex-rollout-file-parse.ts b/src/main/codex-usage/codex-rollout-file-parse.ts index 01755ee69aa..4e3ebe154c9 100644 --- a/src/main/codex-usage/codex-rollout-file-parse.ts +++ b/src/main/codex-usage/codex-rollout-file-parse.ts @@ -1,6 +1,11 @@ import { basename } from 'node:path' -import { stat } from 'node:fs/promises' +import { readJsonlFileSnapshot, type JsonlFileSnapshot } from '../usage/jsonl-file-snapshot' import { readJsonlLinesFromOffset } from '../usage/jsonl-line-offsets' +import { + openJsonlFileReader, + validateJsonlFileReader, + jsonlPhysicalFileId +} from '../usage/jsonl-file-checkpoint' import { attributeCodexUsageEvent } from './codex-usage-event-attribution' import type { UsageWorktreeResolver } from '../usage/usage-worktree-resolver' import { parseCodexUsageRecord, type CodexUsageParseContext } from './codex-usage-record-parser' @@ -10,7 +15,6 @@ import { resolveCodexRolloutResume } from './codex-rollout-resume-state' import type { - CodexUsageAttributedEvent, CodexUsageDailyAggregate, CodexUsageParseResumeState, CodexUsagePersistedFile, @@ -24,17 +28,24 @@ const { finalizeSessions, mergeSessions, mergeDailyAggregates, sortDailyAggregat export type CodexRolloutParseOptions = { /** Suffix-only parse for a diverged legacy copied-session bridge. */ legacySourceSkipBytes?: number - claimEventKey?: (eventKey: string) => boolean + canClaimEventKey?: (eventKey: string) => boolean + commitEventKey?: (eventKey: string) => void /** Resume point verified by the caller, with the cached projection to extend. */ resume?: { state: CodexUsageParseResumeState; previous: CodexUsagePersistedFile } } export async function getProcessedFileInfo(filePath: string): Promise { - const fileStat = await stat(filePath) + const fileStat = await readJsonlFileSnapshot(filePath) + return processedFileInfo(filePath, fileStat) +} + +function processedFileInfo(filePath: string, fileStat: JsonlFileSnapshot): CodexUsageProcessedFile { return { path: filePath, mtimeMs: fileStat.mtimeMs, - size: fileStat.size + size: fileStat.size, + ctimeMs: fileStat.ctimeMs, + physicalFileId: jsonlPhysicalFileId(fileStat) } } @@ -86,106 +97,104 @@ export async function parseCodexUsageFile( resolveWorktree: UsageWorktreeResolver, options: CodexRolloutParseOptions = {} ): Promise { - // Why: the caller verified this resume point while walking the directory, and - // every file discovered or parsed since then has run in between. Re-verify - // here, against the file about to be read, or a rollout replaced in that gap - // gets the cached session id, cwd, model and running totals stitched onto an - // unrelated file's records โ€” and `processedFile` below re-stats to the new - // size, so the reuse path then freezes the corrupted projection. - if ( - options.resume && - (await resolveCodexRolloutResume(filePath, options.resume.previous)) === null - ) { - return parseCodexUsageFile(filePath, resolveWorktree, { ...options, resume: undefined }) - } + let parseOptions = options + for (;;) { + const reader = await openJsonlFileReader(filePath) + try { + // Verification, parsing and checkpointing must all read the same opened file. + if ( + parseOptions.resume && + (await resolveCodexRolloutResume(filePath, parseOptions.resume.previous, reader)) === null + ) { + parseOptions = { ...parseOptions, resume: undefined } + continue + } - const processedFile = await getProcessedFileInfo(filePath) - const legacySourceSkipBytes = options.legacySourceSkipBytes ?? 0 - const startOffset = options.resume?.state.parsedBytes ?? legacySourceSkipBytes - const context = createParseContext(filePath, options) + const legacySourceSkipBytes = parseOptions.legacySourceSkipBytes ?? 0 + const startOffset = parseOptions.resume?.state.parsedBytes ?? legacySourceSkipBytes + const context = createParseContext(filePath, parseOptions) + const accumulator = codexUsageAggregation.createAccumulator() + const ownedEventKeys = new Set() + let hasDeferredClaims = false + let parsedBytes = startOffset + let resumeContext = context + let partialTailProducedEvent = false - const events: CodexUsageAttributedEvent[] = [] - const ownedEventKeys = new Set() - let hasDeferredClaims = false - let parsedBytes = startOffset - // Points at the context as of `parsedBytes`, which excludes a partial tail. - let resumeContext = context - let partialTailProducedEvent = false - - for await (const { line, endOffset, terminated } of readJsonlLinesFromOffset( - filePath, - startOffset - )) { - if (!terminated) { - // Only the final fragment can be unterminated, and the next scan re-reads - // it, so its context edits must not leak into the persisted resume point. - resumeContext = { ...context } - } - const parsed = parseCodexUsageRecord(line, context) - if (terminated) { - parsedBytes = endOffset - } else if (parsed) { - partialTailProducedEvent = true - } - if (!parsed) { - continue - } - // Why: fork/resume rollouts start with a copied prefix of the parent file. - // Events another file already owns are dropped here, but the record still - // advanced context.previousTotals above, so later deltas stay correct. - if (options.claimEventKey && !options.claimEventKey(parsed.eventKey)) { - hasDeferredClaims = true - continue - } - ownedEventKeys.add(parsed.eventKey) - const attributed = await attributeCodexUsageEvent(parsed, resolveWorktree) - if (attributed) { - events.push(attributed) - } - } - - // A counted-but-unterminated tail would be counted again on resume, and a - // legacy suffix offset is recomputed per scan, so neither may be resumed. A - // prefix under the resumable floor is turned away by the builder itself. - const resumeStateSuppressed = partialTailProducedEvent || legacySourceSkipBytes > 0 - const parseResumeState = resumeStateSuppressed - ? null - : await buildCodexRolloutResumeState( + for await (const { line, endOffset, terminated } of readJsonlLinesFromOffset( filePath, - parsedBytes, - resumeContext, - // Already verified against the file at the top of this scan. - options.resume?.state.headDigest ?? null - ) + startOffset, + reader + )) { + if (!terminated) { + // The next scan rereads this tail, including its context changes. + resumeContext = { ...context } + } + const parsed = parseCodexUsageRecord(line, context) + if (terminated) { + parsedBytes = endOffset + } else if (parsed) { + partialTailProducedEvent = true + } + if (!parsed) { + continue + } + // Deferred records still advance cumulative totals without retaining their copies. + if (parseOptions.canClaimEventKey && !parseOptions.canClaimEventKey(parsed.eventKey)) { + hasDeferredClaims = true + continue + } + ownedEventKeys.add(parsed.eventKey) + const attributed = await attributeCodexUsageEvent(parsed, resolveWorktree) + if (attributed) { + accumulator.add(attributed) + } + } - // Why: a resume point only exists past the resumable floor and `parsedBytes` - // only grows, so the builder's other null โ€” a prefix too short to be worth - // resuming โ€” is unreachable here and this null means a short read: the file - // shrank past the prefix this parse merged history for, after the - // re-verification above and during the read. `processedFile` already re-stat'd - // to the smaller size, so persisting that pair would let the next scan reuse a - // pre-truncation total forever. An unterminated tail proves the file still - // runs past the resume offset, so it cannot be this case. - if (options.resume && !resumeStateSuppressed && parseResumeState === null) { - return parseCodexUsageFile(filePath, resolveWorktree, { ...options, resume: undefined }) - } + // A counted unterminated row would be counted again from the committed offset. + const resumeStateSuppressed = partialTailProducedEvent || legacySourceSkipBytes > 0 + const parseResumeState = resumeStateSuppressed + ? null + : await buildCodexRolloutResumeState( + filePath, + parsedBytes, + resumeContext, + parseOptions.resume?.state.headDigest ?? null, + reader + ) - const appended = codexUsageAggregation.aggregate(events) - const previous = options.resume?.previous - if (!previous) { - return { - ...processedFile, - ...appended, - ownedEventKeys: [...ownedEventKeys], - hasDeferredClaims, - parseResumeState + if ( + !(await validateJsonlFileReader(reader, parseOptions.resume?.state)) || + (parseOptions.resume && !resumeStateSuppressed && parseResumeState === null) + ) { + parseOptions = { ...parseOptions, resume: undefined } + continue + } + + await reader.handle.close() + + // Discarded reads cannot claim keys that would hide records in another file. + for (const eventKey of ownedEventKeys) { + parseOptions.commitEventKey?.(eventKey) + } + + const processedFile = processedFileInfo(filePath, reader.stats) + const appended = accumulator.finalize() + const previous = parseOptions.resume?.previous + const committedKeys = previous ? new Set(previous.ownedEventKeys) : ownedEventKeys + if (previous) { + for (const key of ownedEventKeys) { + committedKeys.add(key) + } + } + return { + ...processedFile, + ...(previous ? mergeRolloutProjections(previous, appended) : appended), + ownedEventKeys: [...committedKeys], + hasDeferredClaims: previous?.hasDeferredClaims || hasDeferredClaims, + parseResumeState + } + } finally { + await reader.handle.close() } } - return { - ...processedFile, - ...mergeRolloutProjections(previous, appended), - ownedEventKeys: [...new Set([...previous.ownedEventKeys, ...ownedEventKeys])], - hasDeferredClaims: previous.hasDeferredClaims || hasDeferredClaims, - parseResumeState - } } diff --git a/src/main/codex-usage/codex-rollout-resume-state.ts b/src/main/codex-usage/codex-rollout-resume-state.ts index 368d50bddf9..b531ae74c9c 100644 --- a/src/main/codex-usage/codex-rollout-resume-state.ts +++ b/src/main/codex-usage/codex-rollout-resume-state.ts @@ -1,183 +1,46 @@ -/** - * Decides whether a grown rollout can be parsed from where the last scan - * stopped. A wrong answer here silently corrupts usage totals, so every check - * fails closed: anything unproven falls back to a full reparse. - */ -import { createReadStream } from 'node:fs' -import { stat } from 'node:fs/promises' -import { createHash } from 'node:crypto' import type { CodexUsageParseContext } from './codex-usage-record-parser' import type { CodexUsageParseResumeState, CodexUsagePersistedFile } from './types' +import { + buildJsonlFileCheckpoint, + resolveJsonlFileCheckpoint, + type JsonlFileReader +} from '../usage/jsonl-file-checkpoint' -/** Bytes hashed immediately before the resume offset. Large enough to span a - * whole token_count record, small enough that verifying it is free next to - * re-reading a multi-megabyte rollout. */ -const BOUNDARY_WINDOW_BYTES = 4096 - -/** Bytes hashed at the very start of the parsed prefix. The boundary window can - * only speak for the bytes next to the resume offset, so without this a - * same-length rewrite that leaves the tail intact resumes over changed - * history. Every realistic rotation or rewrite of a rollout replaces the - * leading session_meta line, which lands in this window. */ -const HEAD_WINDOW_BYTES = 4096 - -/** - * Shortest prefix worth resuming over. A resumed scan verifies the prefix - * twice โ€” once when the scanner plans the resume, once against the file it is - * about to read โ€” and then records the moved boundary, so it pays five windows - * whatever the file size. A cold reparse pays the prefix itself plus the two - * windows it records. Below this length reading the whole file is cheaper, and - * for a prefix short enough that the two windows overlap it is far cheaper, - * because each verification then rehashes the entire prefix. - * - * Measured on the byte oracle: a 12,191 B prefix costs 21,234 B resumed against - * 21,137 B cold; at 13,699 B it is 21,234 B against 22,645 B. - */ -export const MIN_RESUMABLE_PREFIX_BYTES = 3 * BOUNDARY_WINDOW_BYTES - -/** Below the floor the two windows could also overlap, so every offset that - * reaches the digest reads is guaranteed to give them a disjoint layout. */ -export function isResumablePrefixLength(parsedBytes: number): boolean { - return parsedBytes > MIN_RESUMABLE_PREFIX_BYTES -} - -async function readWindowDigest( - filePath: string, - start: number, - endExclusive: number -): Promise { - const expectedBytes = endExclusive - start - const hash = createHash('sha256') - let readBytes = 0 - const stream = createReadStream(filePath, { start, end: endExclusive - 1 }) - for await (const chunk of stream) { - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) - hash.update(buffer) - readBytes += buffer.length - } - // A short read means the file no longer reaches the offset we recorded. - return readBytes === expectedBytes ? `${expectedBytes}:${hash.digest('hex')}` : null -} - -type PrefixDigests = { headDigest: string; boundaryDigest: string } - -/** - * Both window digests for a prefix of `parsedBytes`, or null if the file no - * longer reaches that offset. The window layout is a pure function of - * `parsedBytes`, so a later verification hashes exactly the same ranges. Only - * called for a prefix past `MIN_RESUMABLE_PREFIX_BYTES`, so the two windows are - * always disjoint and always their full size. - * - * `carriedHeadDigest` lets a caller that already verified the head window this - * scan skip re-reading those bytes. Deferring to it is self-correcting: if the - * head did change after that read, the next scan compares against the carried - * value and falls back to a full reparse. - */ -async function readPrefixDigests( - filePath: string, - parsedBytes: number, - carriedHeadDigest: string | null = null -): Promise { - const headDigest = carriedHeadDigest?.startsWith(`${HEAD_WINDOW_BYTES}:`) - ? carriedHeadDigest - : await readWindowDigest(filePath, 0, HEAD_WINDOW_BYTES) - if (headDigest === null) { - return null - } - const boundaryDigest = await readWindowDigest( - filePath, - parsedBytes - BOUNDARY_WINDOW_BYTES, - parsedBytes - ) - return boundaryDigest === null ? null : { headDigest, boundaryDigest } -} - -async function readPhysicalFileId(filePath: string): Promise { - try { - const fileStat = await stat(filePath) - return fileStat.ino === 0 ? null : `${fileStat.dev}:${fileStat.ino}` - } catch { - return null - } -} - -function isUsableResumeState( - resume: CodexUsageParseResumeState | null | undefined -): resume is CodexUsageParseResumeState { - return ( - resume != null && - Number.isInteger(resume.parsedBytes) && - resume.parsedBytes >= 0 && - typeof resume.boundaryDigest === 'string' && - // State persisted before the head window existed cannot be verified. - typeof resume.headDigest === 'string' && - // Caches written before the floor existed can hold a shorter prefix. - isResumablePrefixLength(resume.parsedBytes) && - typeof resume.sessionId === 'string' - ) -} +export { MIN_RESUMABLE_PREFIX_BYTES, isResumablePrefixLength } from '../usage/jsonl-file-checkpoint' export async function buildCodexRolloutResumeState( filePath: string, parsedBytes: number, context: CodexUsageParseContext, - verifiedHeadDigest: string | null = null + verifiedHeadDigest: string | null = null, + reader?: JsonlFileReader ): Promise { - if (!isResumablePrefixLength(parsedBytes)) { - return null - } - const digests = await readPrefixDigests(filePath, parsedBytes, verifiedHeadDigest) - if (digests === null) { - return null - } - return { + const checkpoint = await buildJsonlFileCheckpoint( + filePath, parsedBytes, - boundaryDigest: digests.boundaryDigest, - headDigest: digests.headDigest, - physicalFileId: await readPhysicalFileId(filePath), - sessionId: context.sessionId, - sessionCwd: context.sessionCwd, - currentCwd: context.currentCwd, - currentModel: context.currentModel, - previousTotals: context.previousTotals - } + verifiedHeadDigest, + reader + ) + return checkpoint === null + ? null + : { + ...checkpoint, + sessionId: context.sessionId, + sessionCwd: context.sessionCwd, + currentCwd: context.currentCwd, + currentModel: context.currentModel, + previousTotals: context.previousTotals + } } -/** - * Returns the resume point only when the recorded prefix still looks like the - * file's prefix. Deliberately does not consult any timestamp: filesystems vary - * in clock granularity, so a rewrite can land under the mtime, ctime or - * birthtime the cache already holds. `physicalFileId` is a cheap extra catch - * rather than a rotation check โ€” ext4 and overlayfs hand a recreated path the - * inode the old file freed, so it only fires on filesystems that allocate a - * fresh one. Truncation needs no separate check: a file that no longer reaches - * the offset cannot produce the recorded digests. - * - * Bounded by design, so for a prefix larger than the two windows together this - * cannot prove every byte between them is intact. - */ export async function resolveCodexRolloutResume( filePath: string, - previous: CodexUsagePersistedFile | undefined + previous: CodexUsagePersistedFile | undefined, + reader?: JsonlFileReader ): Promise { const resume = previous?.parseResumeState - if (!isUsableResumeState(resume)) { + if (!resume || typeof resume.sessionId !== 'string') { return null } - const physicalFileId = await readPhysicalFileId(filePath) - if ( - resume.physicalFileId !== null && - physicalFileId !== null && - resume.physicalFileId !== physicalFileId - ) { - return null - } - const digests = await readPrefixDigests(filePath, resume.parsedBytes) - if (digests === null) { - return null - } - return digests.boundaryDigest === resume.boundaryDigest && - digests.headDigest === resume.headDigest - ? resume - : null + return (await resolveJsonlFileCheckpoint(filePath, resume, reader)) === null ? null : resume } diff --git a/src/main/codex-usage/codex-session-file-discovery.ts b/src/main/codex-usage/codex-session-file-discovery.ts index bf2595fc340..1ea1cd10162 100644 --- a/src/main/codex-usage/codex-session-file-discovery.ts +++ b/src/main/codex-usage/codex-session-file-discovery.ts @@ -5,6 +5,7 @@ import { getOrcaManagedCodexHomePath, getSystemCodexHomePath } from '../codex/co import { getCodexAccountHomeSessionDirectories } from '../codex/codex-account-home-discovery' import { getLegacyCopiedCodexSessionBridgeScanPreference } from '../codex/codex-session-bridge' import { normalizeFsPath } from '../usage/usage-path-comparison' +import { isDefinitiveAbsence } from '../../shared/definitive-filesystem-absence' const YIELD_EVERY_DISCOVERY_ENTRIES = 100 @@ -25,7 +26,12 @@ async function walkJsonlFiles( dirPath: string, progress: { entriesVisited: number } = { entriesVisited: 0 } ): Promise { - const entries = await readdir(dirPath, { withFileTypes: true }) + const entries = await readdir(dirPath, { withFileTypes: true }).catch((error: unknown) => { + if (isDefinitiveAbsence(error)) { + return [] + } + throw error + }) const files: string[] = [] for (const entry of entries) { @@ -79,11 +85,7 @@ function hasLegacyCopiedSessionBridgeMarkers(): boolean { export async function listCodexSessionFiles(): Promise { const files: string[] = [] for (const dirPath of getCodexSessionDirectories()) { - try { - appendDiscoveredFiles(files, await walkJsonlFiles(dirPath)) - } catch { - // Missing or unreadable history in one home should not hide the other. - } + appendDiscoveredFiles(files, await walkJsonlFiles(dirPath)) } return dedupeCodexSessionFileAliases(files, hasLegacyCopiedSessionBridgeMarkers()) } diff --git a/src/main/codex-usage/scanner-checkpoint-races.test.ts b/src/main/codex-usage/scanner-checkpoint-races.test.ts new file mode 100644 index 00000000000..920ba0412b7 --- /dev/null +++ b/src/main/codex-usage/scanner-checkpoint-races.test.ts @@ -0,0 +1,333 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + appendFileSync, + mkdirSync, + mkdtempSync, + renameSync, + rmSync, + statSync, + writeFileSync +} from 'node:fs' +import type * as NodeFsPromises from 'node:fs/promises' +import type * as NodeOs from 'node:os' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' + +const { homedirMock, getPathMock, parseReads, handles, mutation } = vi.hoisted(() => { + const parseReads: { path: string; start: number; bytes: number }[] = [] + const mutation: { + beforeParse: ((path: string) => void) | null + duringParse: ((path: string) => void) | null + endBeforeEOF: number | null + zeroDescriptorRead: boolean + } = { + beforeParse: null, + duringParse: null, + endBeforeEOF: null, + zeroDescriptorRead: false + } + return { + homedirMock: vi.fn<() => string>(), + getPathMock: vi.fn<(name: string) => string>(), + parseReads, + handles: { opened: 0, closed: 0 }, + mutation + } +}) + +vi.mock('electron', () => ({ app: { getPath: getPathMock } })) +vi.mock('node:os', async () => ({ + ...(await vi.importActual('node:os')), + homedir: homedirMock +})) + +vi.mock('node:fs/promises', async () => { + const actual = await vi.importActual('node:fs/promises') + return { + ...actual, + open: async (...args: Parameters) => { + const handle = await actual.open(...args) + handles.opened++ + const createStream = handle.createReadStream.bind(handle) + const readDescriptor = handle.read.bind(handle) + const close = handle.close.bind(handle) + vi.spyOn(handle, 'read').mockImplementation(async (...readArgs) => { + const result = await readDescriptor(...readArgs) + if (mutation.zeroDescriptorRead) { + mutation.zeroDescriptorRead = false + return { ...result, bytesRead: 0 } + } + return result + }) + let closed = false + vi.spyOn(handle, 'close').mockImplementation(async () => { + await close() + if (!closed) { + closed = true + handles.closed++ + } + }) + vi.spyOn(handle, 'createReadStream').mockImplementation((options) => { + const start = options?.start ?? 0 + const end = options?.end + const parseRead = end !== undefined && end - start + 1 !== 4096 + const filePath = String(args[0]) + if (parseRead) { + mutation.beforeParse?.(filePath) + } + const forcedEnd = parseRead ? mutation.endBeforeEOF : null + if (forcedEnd !== null) { + mutation.endBeforeEOF = null + } + const stream = createStream(forcedEnd === null ? options : { ...options, end: forcedEnd }) + if (parseRead) { + const read = { path: filePath, start, bytes: 0 } + parseReads.push(read) + stream.on('data', (chunk: string | Buffer) => { + read.bytes += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length + }) + stream.once('data', () => mutation.duringParse?.(filePath)) + } + return stream + }) + return handle + } + } +}) + +import { parseCodexUsageFile } from './codex-rollout-file-parse' +import { scanCodexUsageFiles } from './scanner' + +let root: string +let rolloutPath: string + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orca-codex-checkpoint-races-')) + homedirMock.mockReturnValue(root) + getPathMock.mockReturnValue(join(root, 'user-data')) + vi.stubEnv('ORCA_USER_DATA_PATH', join(root, 'user-data')) + vi.stubEnv('XDG_CONFIG_HOME', join(root, '.config')) + vi.stubEnv('CODEX_HOME', join(root, '.codex')) + const sessions = join(root, '.codex', 'sessions') + mkdirSync(sessions, { recursive: true }) + rolloutPath = join(sessions, 'rollout.jsonl') + parseReads.length = 0 + handles.opened = 0 + handles.closed = 0 + mutation.beforeParse = null + mutation.duringParse = null + mutation.endBeforeEOF = null + mutation.zeroDescriptorRead = false +}) + +afterEach(() => { + rmSync(root, { recursive: true, force: true }) + vi.unstubAllEnvs() + expect(handles.closed).toBe(handles.opened) +}) + +function record(index: number, baseline: number): string { + return `${JSON.stringify({ + timestamp: `2026-05-26T12:${String(index % 60).padStart(2, '0')}:00.000Z`, + type: 'event_msg', + payload: { + type: 'token_count', + info: { + model: 'gpt-5-codex', + last_token_usage: { input_tokens: 1, total_tokens: 1 }, + total_token_usage: { + input_tokens: baseline + index + 1, + total_tokens: baseline + index + 1 + } + } + } + })}\n` +} + +function records(from: number, to: number, baseline = 1000): string { + return Array.from({ length: to - from }, (_, index) => record(from + index, baseline)).join('') +} + +function transcript(sessionId: string, count: number, baseline = 1000): string { + const metadata = `${JSON.stringify({ type: 'session_meta', payload: { id: sessionId, cwd: root } })}\n` + const padding = `${JSON.stringify({ type: 'tool-result', content: 'x'.repeat(16_384) })}\n` + return metadata + padding + records(0, count, baseline) +} + +function offsets(): number[] { + return parseReads.filter((read) => read.path === rolloutPath).map((read) => read.start) +} + +describe('Codex checkpoint file races', () => { + it.each(['rewrite', 'rotate'])( + 'discards a %s after verification before reading the suffix', + async (kind) => { + writeFileSync(rolloutPath, transcript('session-old', 40)) + const first = await scanCodexUsageFiles([], []) + const offset = first.processedFiles[0]?.parseResumeState?.parsedBytes ?? 0 + expect(offset).toBeGreaterThan(12_288) + appendFileSync(rolloutPath, records(40, 42)) + const replacement = transcript('session-new', 60, 2000) + mutation.beforeParse = (filePath) => { + mutation.beforeParse = null + if (kind === 'rotate') { + const replacementPath = join(root, 'replacement.jsonl') + writeFileSync(replacementPath, replacement) + renameSync(replacementPath, filePath) + } else { + writeFileSync(filePath, replacement) + } + } + parseReads.length = 0 + + const second = await scanCodexUsageFiles([], first.processedFiles) + expect(mutation.beforeParse).toBeNull() + expect(offsets()).toEqual([offset, 0]) + expect(parseReads.every((read) => read.bytes > 0)).toBe(true) + const cold = await scanCodexUsageFiles([], []) + expect(second.sessions).toEqual(cold.sessions) + expect(second.dailyAggregates).toEqual(cold.dailyAggregates) + expect(second.sessions[0]?.sessionId).toBe('session-new') + const third = await scanCodexUsageFiles([], second.processedFiles) + expect(third.processedFiles[0]).toBe(second.processedFiles[0]) + expect(third.sessions).toEqual(cold.sessions) + } + ) + + it('discards cold-read events before claiming their keys when the file changes during parsing', async () => { + writeFileSync(rolloutPath, transcript('session-old', 40)) + mutation.duringParse = (filePath) => { + mutation.duringParse = null + writeFileSync(filePath, transcript('session-new', 60, 2000)) + } + const commitEventKey = vi.fn() + + const parsed = await parseCodexUsageFile(rolloutPath, () => null, { commitEventKey }) + expect(offsets()).toEqual([0, 0]) + expect(parsed.sessions[0]?.sessionId).toBe('session-new') + expect(parsed.sessions[0]?.eventCount).toBe(60) + expect(commitEventKey).toHaveBeenCalledTimes(60) + const cold = await parseCodexUsageFile(rolloutPath, () => null) + expect(parsed.ownedEventKeys).toEqual(cold.ownedEventKeys) + expect(parsed.sessions).toEqual(cold.sessions) + }) + + it('leaves an append after the captured EOF for the next scan', async () => { + writeFileSync(rolloutPath, transcript('session-stable', 40)) + const capturedSize = statSync(rolloutPath).size + mutation.duringParse = (filePath) => { + mutation.duringParse = null + appendFileSync(filePath, records(40, 41)) + } + + const first = await scanCodexUsageFiles([], []) + expect(first.sessions[0]?.eventCount).toBe(40) + expect(first.processedFiles[0]?.size).toBe(capturedSize) + expect(first.processedFiles[0]?.parseResumeState?.parsedBytes).toBe(capturedSize) + const second = await scanCodexUsageFiles([], first.processedFiles) + expect(second.sessions[0]?.eventCount).toBe(41) + const cold = await scanCodexUsageFiles([], []) + expect(second.sessions).toEqual(cold.sessions) + expect(second.dailyAggregates).toEqual(cold.dailyAggregates) + }) + + it('releases old owner claims and lets an unchanged deferred fork reclaim them', async () => { + const prefix = transcript('session-old', 40) + const forkPath = join(dirname(rolloutPath), 'zzzz-fork.jsonl') + writeFileSync(rolloutPath, prefix) + writeFileSync(forkPath, prefix + records(40, 41)) + const first = await scanCodexUsageFiles([], []) + const fork = first.processedFiles.find((file) => file.path === forkPath) + expect(fork?.hasDeferredClaims).toBe(true) + expect(fork?.ownedEventKeys).toHaveLength(1) + appendFileSync(rolloutPath, records(40, 42)) + mutation.beforeParse = (filePath) => { + if (filePath === rolloutPath) { + mutation.beforeParse = null + writeFileSync(filePath, transcript('session-new', 60, 2000)) + } + } + parseReads.length = 0 + + const second = await scanCodexUsageFiles([], first.processedFiles) + expect(parseReads.filter((read) => read.path === forkPath).map((read) => read.start)).toEqual([ + 0 + ]) + expect( + second.processedFiles.find((file) => file.path === forkPath)?.ownedEventKeys + ).toHaveLength(41) + const cold = await scanCodexUsageFiles([], []) + expect(second.sessions).toEqual(cold.sessions) + expect(second.dailyAggregates).toEqual(cold.dailyAggregates) + const third = await scanCodexUsageFiles([], second.processedFiles) + expect(third.sessions).toEqual(cold.sessions) + }) + + it('keeps an unchanged deferred fork cached when a short owner merely appends', async () => { + const prefix = records(0, 3) + const forkPath = join(dirname(rolloutPath), 'zzzz-fork.jsonl') + writeFileSync(rolloutPath, prefix) + writeFileSync(forkPath, prefix + records(3, 4)) + const first = await scanCodexUsageFiles([], []) + const fork = first.processedFiles.find((file) => file.path === forkPath) + expect(fork?.hasDeferredClaims).toBe(true) + expect(first.processedFiles[0]?.parseResumeState).toBeNull() + appendFileSync(rolloutPath, records(4, 5)) + parseReads.length = 0 + + const second = await scanCodexUsageFiles([], first.processedFiles) + expect(second.processedFiles.find((file) => file.path === forkPath)).toBe(fork) + expect(parseReads.some((read) => read.path === forkPath)).toBe(false) + expect(second.dailyAggregates.reduce((sum, row) => sum + row.totalTokens, 0)).toBe(5) + }) + + it('rejects an unchanged file stream that stops early before claiming partial history', async () => { + writeFileSync(rolloutPath, transcript('session-stable', 40)) + mutation.endBeforeEOF = Buffer.byteLength(transcript('session-stable', 10)) - 1 + const commitEventKey = vi.fn() + + await expect(parseCodexUsageFile(rolloutPath, () => null, { commitEventKey })).rejects.toThrow( + 'JSONL snapshot stream ended before the unchanged file boundary.' + ) + expect(mutation.endBeforeEOF).toBeNull() + expect(offsets()).toEqual([0]) + expect(commitEventKey).not.toHaveBeenCalled() + const cold = await scanCodexUsageFiles([], []) + expect(cold.sessions[0]?.eventCount).toBe(40) + }) + + it('leaves the previous scan intact when an unchanged suffix stream stops early', async () => { + const prefix = transcript('session-stable', 40) + writeFileSync(rolloutPath, prefix) + const previous = await scanCodexUsageFiles([], []) + appendFileSync(rolloutPath, records(40, 42)) + mutation.endBeforeEOF = Buffer.byteLength(prefix + records(40, 41)) - 1 + parseReads.length = 0 + + await expect(scanCodexUsageFiles([], previous.processedFiles)).rejects.toThrow( + 'JSONL snapshot stream ended before the unchanged file boundary.' + ) + expect(mutation.endBeforeEOF).toBeNull() + expect(offsets()).toEqual([Buffer.byteLength(prefix)]) + expect(previous.sessions[0]?.eventCount).toBe(40) + expect(previous.processedFiles[0]?.size).toBe(Buffer.byteLength(prefix)) + const recovered = await scanCodexUsageFiles([], previous.processedFiles) + expect(recovered.sessions[0]?.eventCount).toBe(42) + }) + + it('rejects a stable zero-byte read of the skipped prefix in a short legacy source', async () => { + writeFileSync(rolloutPath, records(0, 3)) + mutation.zeroDescriptorRead = true + const commitEventKey = vi.fn() + const legacySourceSkipBytes = Buffer.byteLength(records(0, 1)) + + await expect( + parseCodexUsageFile(rolloutPath, () => null, { legacySourceSkipBytes, commitEventKey }) + ).rejects.toThrow('JSONL snapshot read ended before the unchanged file boundary.') + expect(mutation.zeroDescriptorRead).toBe(false) + expect(offsets()).toEqual([]) + expect(commitEventKey).not.toHaveBeenCalled() + const recovered = await parseCodexUsageFile(rolloutPath, () => null, { legacySourceSkipBytes }) + expect(recovered.sessions[0]?.eventCount).toBe(2) + }) +}) diff --git a/src/main/codex-usage/scanner-file-generation.test.ts b/src/main/codex-usage/scanner-file-generation.test.ts new file mode 100644 index 00000000000..6a9c976c7ad --- /dev/null +++ b/src/main/codex-usage/scanner-file-generation.test.ts @@ -0,0 +1,192 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + statSync, + utimesSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type * as JsonlLineOffsets from '../usage/jsonl-line-offsets' + +const fixture = vi.hoisted(() => { + const parseOffsets: number[] = [] + return { root: '', parseOffsets } +}) + +vi.mock('../codex/codex-home-paths', () => ({ + getOrcaManagedCodexHomePath: () => join(fixture.root, 'runtime'), + getSystemCodexHomePath: () => join(fixture.root, 'system') +})) + +vi.mock('../codex/codex-account-home-discovery', () => ({ + getCodexAccountHomeSessionDirectories: () => [] +})) + +vi.mock('../codex/codex-session-bridge', () => ({ + getLegacyCopiedCodexSessionBridgeScanPreference: () => null +})) + +vi.mock('../usage/jsonl-line-offsets', async () => { + const actual = await vi.importActual('../usage/jsonl-line-offsets') + return { + ...actual, + readJsonlLinesFromOffset: async function* ( + ...args: Parameters + ) { + fixture.parseOffsets.push(args[1]) + yield* actual.readJsonlLinesFromOffset(...args) + } + } +}) + +import { scanCodexUsageFiles } from './scanner' + +const preservedMtime = new Date('2026-05-26T12:00:00.000Z') +let rolloutPath: string + +beforeEach(() => { + fixture.root = mkdtempSync(join(tmpdir(), 'orca-codex-file-generation-')) + const sessions = join(fixture.root, 'runtime', 'sessions') + mkdirSync(sessions, { recursive: true }) + rolloutPath = join(sessions, 'rollout.jsonl') + fixture.parseOffsets.length = 0 +}) + +afterEach(() => { + rmSync(fixture.root, { recursive: true, force: true }) +}) + +function transcript(sessionId: string, count: number, changedIndex = -1): string { + const metadata = JSON.stringify({ + type: 'session_meta', + payload: { id: sessionId, cwd: fixture.root } + }) + const records = Array.from({ length: count }, (_, index) => { + const tokens = index === changedIndex ? 20 : 10 + return JSON.stringify({ + timestamp: new Date(preservedMtime.getTime() + index * 60_000).toISOString(), + type: 'event_msg', + payload: { + type: 'token_count', + info: { + model: 'gpt-5-codex', + last_token_usage: { input_tokens: tokens, total_tokens: tokens } + } + } + }) + }) + return `${[metadata, ...records].join('\n')}\n` +} + +function writePreservingMtime(filePath: string, content: string): void { + writeFileSync(filePath, content) + utimesSync(filePath, preservedMtime, preservedMtime) +} + +function totalTokens(rows: { totalTokens: number }[]): number { + return rows.reduce((sum, row) => sum + row.totalTokens, 0) +} + +describe('Codex cached rollout generation', () => { + it.each([1, 80])( + 'reparses a same-size preserved-mtime atomic replacement containing %i events', + async (count) => { + const original = transcript('session-old', count) + const replacement = transcript('session-new', count, Math.floor(count / 2)) + expect(Buffer.byteLength(replacement)).toBe(Buffer.byteLength(original)) + writePreservingMtime(rolloutPath, original) + const before = statSync(rolloutPath, { bigint: true }) + const first = await scanCodexUsageFiles([], []) + expect(totalTokens(first.dailyAggregates)).toBe(count * 10) + expect(Boolean(first.processedFiles[0]?.parseResumeState)).toBe(count > 1) + + const replacementPath = join(fixture.root, 'replacement.jsonl') + writePreservingMtime(replacementPath, replacement) + renameSync(replacementPath, rolloutPath) + const after = statSync(rolloutPath, { bigint: true }) + expect(after.size).toBe(before.size) + expect(after.mtimeNs).toBe(before.mtimeNs) + if (before.ino !== 0n && after.ino !== 0n) { + expect(`${after.dev}:${after.ino}`).not.toBe(`${before.dev}:${before.ino}`) + } + fixture.parseOffsets.length = 0 + + const second = await scanCodexUsageFiles([], first.processedFiles) + expect(totalTokens(second.dailyAggregates)).toBe(count * 10 + 10) + expect(second.sessions[0]?.sessionId).toBe('session-new') + expect(fixture.parseOffsets).toEqual([0]) + const cold = await scanCodexUsageFiles([], []) + expect(second.sessions).toEqual(cold.sessions) + expect(second.dailyAggregates).toEqual(cold.dailyAggregates) + fixture.parseOffsets.length = 0 + const third = await scanCodexUsageFiles([], second.processedFiles) + expect(third.processedFiles[0]).toBe(second.processedFiles[0]) + expect(fixture.parseOffsets).toEqual([]) + } + ) + + it('cold-reparses a same-inode middle rewrite when size and mtime are preserved', async () => { + const original = transcript('session-stable', 80) + const replacement = transcript('session-stable', 80, 40) + expect(Buffer.byteLength(replacement)).toBe(Buffer.byteLength(original)) + expect(replacement.slice(0, 4096)).toBe(original.slice(0, 4096)) + expect(replacement.slice(-4096)).toBe(original.slice(-4096)) + writePreservingMtime(rolloutPath, original) + const before = statSync(rolloutPath) + const first = await scanCodexUsageFiles([], []) + expect(totalTokens(first.dailyAggregates)).toBe(800) + expect(first.processedFiles[0]?.parseResumeState?.parsedBytes).toBe(before.size) + + await new Promise((resolve) => setTimeout(resolve, 20)) + writePreservingMtime(rolloutPath, replacement) + const after = statSync(rolloutPath) + expect(after.ino).toBe(before.ino) + expect(after.dev).toBe(before.dev) + expect(after.size).toBe(before.size) + expect(after.mtimeMs).toBe(before.mtimeMs) + expect(after.ctimeMs).not.toBe(before.ctimeMs) + expect(readFileSync(rolloutPath, 'utf8')).toBe(replacement) + fixture.parseOffsets.length = 0 + + const second = await scanCodexUsageFiles([], first.processedFiles) + expect(totalTokens(second.dailyAggregates)).toBe(810) + expect(fixture.parseOffsets).toEqual([0]) + expect(second.processedFiles[0]?.ctimeMs).toBe(after.ctimeMs) + const cold = await scanCodexUsageFiles([], []) + expect(second.sessions).toEqual(cold.sessions) + expect(second.dailyAggregates).toEqual(cold.dailyAggregates) + }) + + it('cold-reparses legacy metadata once even when the old checkpoint remains valid', async () => { + writePreservingMtime(rolloutPath, transcript('session-legacy', 80)) + const first = await scanCodexUsageFiles([], []) + expect(first.processedFiles[0]?.parseResumeState?.parsedBytes).toBeGreaterThan(12_288) + const legacy = first.processedFiles.map((file) => { + const oldFile = { ...file } + delete oldFile.physicalFileId + delete oldFile.ctimeMs + return oldFile + }) + fixture.parseOffsets.length = 0 + + const second = await scanCodexUsageFiles([], legacy) + expect(fixture.parseOffsets).toEqual([0]) + expect(second.sessions).toEqual(first.sessions) + expect(second.dailyAggregates).toEqual(first.dailyAggregates) + const stats = statSync(rolloutPath) + expect(second.processedFiles[0]?.physicalFileId).toBe( + stats.ino === 0 ? null : `${stats.dev}:${stats.ino}` + ) + expect(second.processedFiles[0]?.ctimeMs).toBe(stats.ctimeMs) + fixture.parseOffsets.length = 0 + + const third = await scanCodexUsageFiles([], second.processedFiles) + expect(third.processedFiles[0]).toBe(second.processedFiles[0]) + expect(fixture.parseOffsets).toEqual([]) + }) +}) diff --git a/src/main/codex-usage/scanner-file-identity.test.ts b/src/main/codex-usage/scanner-file-identity.test.ts index f0d85076457..69301c54a2f 100644 --- a/src/main/codex-usage/scanner-file-identity.test.ts +++ b/src/main/codex-usage/scanner-file-identity.test.ts @@ -31,6 +31,22 @@ vi.mock('node:fs/promises', async () => { ? Object.assign(stats, { dev: Number(identity.dev), ino: Number(identity.ino) }) : stats }, + open: async (...args: Parameters) => { + const handle = await actual.open(...args) + const identity = fixture.identities.get(String(args[0])) + const stat = handle.stat.bind(handle) + vi.spyOn(handle, 'stat').mockImplementation(async (options) => { + if (options?.bigint) { + const stats = await stat({ bigint: true }) + return identity ? Object.assign(stats, identity) : stats + } + const stats = await stat() + return identity + ? Object.assign(stats, { dev: Number(identity.dev), ino: Number(identity.ino) }) + : stats + }) + return handle + }, realpath: async (filePath: PathLike) => { if (fixture.realpathErrors.has(String(filePath))) { throw new Error('Fixture realpath unavailable') diff --git a/src/main/codex-usage/scanner-incremental-append.test.ts b/src/main/codex-usage/scanner-incremental-append.test.ts index f4abec1f1c4..b00054894ec 100644 --- a/src/main/codex-usage/scanner-incremental-append.test.ts +++ b/src/main/codex-usage/scanner-incremental-append.test.ts @@ -2,19 +2,22 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { tmpdir } from 'node:os' import type * as NodeOs from 'node:os' import type * as NodeFs from 'node:fs' +import type * as NodeFsPromises from 'node:fs/promises' +import type * as JsonlLineOffsets from '../usage/jsonl-line-offsets' import { join } from 'node:path' -const { getPathMock, homedirMock, streamReads, onStreamOpen } = vi.hoisted(() => { - const streamReads: { path: string; bytes: number; start: number; bounded: boolean }[] = [] +const { getPathMock, homedirMock, streamReads, onStreamOpen, parsingPaths } = vi.hoisted(() => { + const streamReads: { path: string; bytes: number; start: number; digest: boolean }[] = [] // Seam for mutating the tree mid-scan, between two files' parse reads. - const onStreamOpen: { current: ((path: string, bounded: boolean) => void) | null } = { + const onStreamOpen: { current: ((path: string, digest: boolean) => void) | null } = { current: null } return { getPathMock: vi.fn<(name: string) => string>(), homedirMock: vi.fn<() => string>(), streamReads, - onStreamOpen + onStreamOpen, + parsingPaths: new Set() } }) @@ -44,16 +47,84 @@ vi.mock('node:fs', async () => { const filePath = String(path) const range = typeof options === 'object' && options !== null ? options : {} const start = range.start ?? 0 - const size = actual.statSync(filePath).size - const stop = range.end === undefined ? size : Math.min(size, range.end + 1) - streamReads.push({ - path: filePath, - bytes: Math.max(0, stop - start), - start, - bounded: range.end !== undefined + const read = { path: filePath, bytes: 0, start, digest: !parsingPaths.has(filePath) } + streamReads.push(read) + onStreamOpen.current?.(filePath, read.digest) + const stream = actual.createReadStream(path, options) + stream.on('data', (chunk: string | Buffer) => { + read.bytes += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length }) - onStreamOpen.current?.(filePath, range.end !== undefined) - return actual.createReadStream(path, options) + return stream + } + } +}) + +vi.mock('node:fs/promises', async () => { + const actual = await vi.importActual('node:fs/promises') + return { + ...actual, + open: async (...args: Parameters) => { + const handle = await actual.open(...args) + const createStream = handle.createReadStream.bind(handle) + const readWindow = handle.read.bind(handle) + let streamActive = false + vi.spyOn(handle, 'read').mockImplementation(async (...readArgs) => { + if (streamActive) { + return readWindow(...readArgs) + } + const filePath = String(args[0]) + const digest = !parsingPaths.has(filePath) + onStreamOpen.current?.(filePath, digest) + const result = await readWindow(...readArgs) + const position = readArgs.at(-1) + streamReads.push({ + path: filePath, + bytes: result.bytesRead, + start: typeof position === 'number' ? position : 0, + digest + }) + return result + }) + vi.spyOn(handle, 'createReadStream').mockImplementation((options) => { + const filePath = String(args[0]) + const read = { + path: filePath, + bytes: 0, + start: options?.start ?? 0, + digest: !parsingPaths.has(filePath) + } + streamReads.push(read) + onStreamOpen.current?.(filePath, read.digest) + streamActive = true + const stream = createStream(options) + const finish = (): void => { + streamActive = false + } + stream.once('end', finish) + stream.once('close', finish) + stream.on('data', (chunk: string | Buffer) => { + read.bytes += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length + }) + return stream + }) + return handle + } + } +}) + +vi.mock('../usage/jsonl-line-offsets', async () => { + const actual = await vi.importActual('../usage/jsonl-line-offsets') + return { + ...actual, + readJsonlLinesFromOffset: async function* ( + ...args: Parameters + ) { + parsingPaths.add(args[0]) + try { + yield* actual.readJsonlLinesFromOffset(...args) + } finally { + parsingPaths.delete(args[0]) + } } } }) @@ -62,7 +133,7 @@ import { mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync, appendFileSync import { scanCodexUsageFiles } from './scanner' import type { CodexUsagePersistedFile } from './types' -/** Mirrors BOUNDARY_WINDOW_BYTES in codex-rollout-resume-state.ts. */ +/** Mirrors the checkpoint window in jsonl-file-checkpoint.ts. */ const BOUNDARY_WINDOW_BYTES = 4096 /** Enough records (~377 B each) to put a prefix past MIN_RESUMABLE_PREFIX_BYTES. @@ -148,12 +219,10 @@ function bytesReadFor(filePath: string): number { .reduce((total, entry) => total + entry.bytes, 0) } -/** Offsets at which this file's parse reads opened, in order. Bounded reads are - * digest windows; an unbounded one at 0 is a full reparse and at the recorded - * offset is a resume, so this says exactly which path a scan took. */ +/** Digest and parse streams both have end bounds; the iterator marks parse reads. */ function parseReadOffsets(filePath: string): number[] { return streamReads - .filter((entry) => entry.path === filePath && !entry.bounded) + .filter((entry) => entry.path === filePath && !entry.digest) .map((entry) => entry.start) } @@ -195,6 +264,7 @@ beforeEach(() => { mkdirSync(sessionsDir, { recursive: true }) streamReads.length = 0 onStreamOpen.current = null + parsingPaths.clear() }) afterEach(() => { @@ -231,14 +301,13 @@ describe('scanCodexUsageFiles incremental append', () => { expect(totalTokens(second.dailyAggregates)).toBe(202) expect(second.sessions[0]?.eventCount).toBe(202) - // The defect: the scanner restarts at byte 0 and re-reads the whole file. - // The fix reads the appended bytes plus five bounded windows: a head and a - // boundary window to plan the resume, both again at the point of use so a - // rollout replaced in between cannot be stitched onto, then the moved - // boundary to record the new resume point. + // Eleven windows prove the planned prefix, opened snapshot and final checkpoint. expect(reparsedFromStart(rolloutPath)).toBe(false) expect(bytesReadFor(rolloutPath)).toBeLessThan(sizeBeforeAppend) - expect(bytesReadFor(rolloutPath)).toBeLessThanOrEqual(appendedBytes + 5 * BOUNDARY_WINDOW_BYTES) + expect(bytesReadFor(rolloutPath)).toBeGreaterThan(appendedBytes) + expect(bytesReadFor(rolloutPath)).toBeLessThanOrEqual( + appendedBytes + 11 * BOUNDARY_WINDOW_BYTES + ) }) it('carries cumulative token totals across the resume boundary', async () => { @@ -379,10 +448,8 @@ describe('scanCodexUsageFiles incremental append', () => { appendFileSync(rolloutPath, usageRecordRange(RESUMABLE_RECORDS, RESUMABLE_RECORDS + 2), 'utf-8') const truncated = `${sessionMeta('session-shrinker')}${usageRecordRange(0, 5)}` - onStreamOpen.current = (path, bounded) => { - // Bounded reads are the digest windows; the unbounded one is the parse - // read, which opens after every check this scan is going to make. - if (path === rolloutPath && !bounded) { + onStreamOpen.current = (path, digest) => { + if (path === rolloutPath && !digest) { onStreamOpen.current = null writeFileSync(path, truncated, 'utf-8') } @@ -429,10 +496,10 @@ describe('scanCodexUsageFiles incremental append', () => { const replacement = `${sessionMeta('session-other')}${usageRecordRange(200, 260)}` // Past the recorded offset, so every digest window still reads its full size. expect(Buffer.byteLength(replacement)).toBeGreaterThan(resumeOffset) - onStreamOpen.current = (path, bounded) => { + onStreamOpen.current = (path, digest) => { // Lands while the earlier-sorted rollout is being parsed, which is after // the scanner's discovery loop verified every file's prefix. - if (path === driverPath && !bounded) { + if (path === driverPath && !digest) { onStreamOpen.current = null writeFileSync(targetPath, replacement, 'utf-8') } diff --git a/src/main/codex-usage/scanner.ts b/src/main/codex-usage/scanner.ts index 449e8c2613d..f0eaf84d087 100644 --- a/src/main/codex-usage/scanner.ts +++ b/src/main/codex-usage/scanner.ts @@ -61,12 +61,19 @@ export async function scanCodexUsageFiles( const previous = previousByPath.get(filePath) // When an owner disappears, only deferred-claim files need reparse. const mustReclaimDeferred = lostOwnerPath && previous?.hasDeferredClaims !== false + const hasGenerationMetadata = + previous && + typeof previous.ctimeMs === 'number' && + (typeof previous.physicalFileId === 'string' || previous.physicalFileId === null) const canReuse = !mustReclaimDeferred && legacySourceSkipBytes === 0 && previous && + hasGenerationMetadata && previous.mtimeMs === fileInfo.mtimeMs && previous.size === fileInfo.size && + previous.ctimeMs === fileInfo.ctimeMs && + previous.physicalFileId === fileInfo.physicalFileId && Array.isArray(previous.ownedEventKeys) && typeof previous.hasDeferredClaims === 'boolean' if (canReuse) { @@ -75,7 +82,13 @@ export async function scanCodexUsageFiles( // Why: rollouts are append-only and grow all day, so re-reading each one // from byte 0 dominated scans (#20940). A reclaim or a legacy suffix // offset still needs the whole file, so neither may resume. - if (!mustReclaimDeferred && legacySourceSkipBytes === 0 && previous) { + if ( + !mustReclaimDeferred && + legacySourceSkipBytes === 0 && + previous && + hasGenerationMetadata && + fileInfo.size > previous.size + ) { const state = await resolveCodexRolloutResume(filePath, previous) if (state) { resumeByPath.set(filePath, { state, previous }) @@ -97,7 +110,7 @@ export async function scanCodexUsageFiles( // stay deterministic. const eventOwnerByKey = new Map() for (const filePath of files) { - const retained = reusedByPath.get(filePath) ?? resumeByPath.get(filePath)?.previous + const retained = previousByPath.get(filePath) for (const eventKey of retained?.ownedEventKeys ?? []) { // First retained claim wins so conflicting projections stay deterministic. if (!eventOwnerByKey.has(eventKey)) { @@ -107,20 +120,37 @@ export async function scanCodexUsageFiles( } const parsedByPath = new Map() - for (const [index, filePath] of pathsToParse.entries()) { + let lostOwnerKeys = lostOwnerPath + const parseAndPublish = async ( + filePath: string, + resume?: CodexRolloutResumePlan + ): Promise => { + const previous = + parsedByPath.get(filePath) ?? reusedByPath.get(filePath) ?? previousByPath.get(filePath) const processed = await parseCodexUsageFile(filePath, resolveWorktree, { legacySourceSkipBytes: legacySourceSkipBytesByPath.get(filePath) ?? 0, - resume: resumeByPath.get(filePath), - claimEventKey: (eventKey) => { + resume, + canClaimEventKey: (eventKey) => { const owner = eventOwnerByKey.get(eventKey) - if (owner !== undefined && owner !== filePath) { - return false - } + return owner === undefined || owner === filePath + }, + commitEventKey: (eventKey) => { eventOwnerByKey.set(eventKey, filePath) - return true } }) + const currentKeys = new Set(processed.ownedEventKeys) + for (const key of previous?.ownedEventKeys ?? []) { + if (eventOwnerByKey.get(key) === filePath && !currentKeys.has(key)) { + eventOwnerByKey.delete(key) + lostOwnerKeys = true + } + } + reusedByPath.delete(filePath) parsedByPath.set(filePath, processed) + } + + for (const [index, filePath] of pathsToParse.entries()) { + await parseAndPublish(filePath, resumeByPath.get(filePath)) // Why: Codex session history can grow large, and scans run on the Electron // main process. Yield regularly so opening Settings does not stall while @@ -131,6 +161,21 @@ export async function scanCodexUsageFiles( } } + // A replaced owner can release records that an unchanged fork previously deferred. + while (lostOwnerKeys) { + lostOwnerKeys = false + for (const [index, filePath] of files.entries()) { + const current = parsedByPath.get(filePath) ?? reusedByPath.get(filePath) + if (current?.hasDeferredClaims) { + await parseAndPublish(filePath) + onFilesScanned?.(1) + } + if ((index + 1) % YIELD_EVERY_FILES === 0) { + await yieldToEventLoop() + } + } + } + const processedFiles: CodexUsagePersistedFile[] = [] const sessionsById = new Map() const dailyByKey = new Map() diff --git a/src/main/codex-usage/session-file-discovery-errors.test.ts b/src/main/codex-usage/session-file-discovery-errors.test.ts new file mode 100644 index 00000000000..afe827f978b --- /dev/null +++ b/src/main/codex-usage/session-file-discovery-errors.test.ts @@ -0,0 +1,91 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import type * as FsPromises from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +const state = vi.hoisted(() => ({ managed: '', system: '', blocked: '', code: '' })) + +vi.mock('../codex/codex-home-paths', () => ({ + getOrcaManagedCodexHomePath: () => state.managed, + getSystemCodexHomePath: () => state.system +})) + +vi.mock('../codex/codex-account-home-discovery', () => ({ + getCodexAccountHomeSessionDirectories: () => [] +})) + +vi.mock('../codex/codex-session-bridge', () => ({ + getLegacyCopiedCodexSessionBridgeScanPreference: () => null +})) + +vi.mock('node:fs/promises', async () => { + const actual = await vi.importActual('node:fs/promises') + return { + ...actual, + readdir: async (...args: Parameters) => { + if (String(args[0]) === state.blocked) { + throw Object.assign(new Error(`Cannot discover ${state.blocked}`), { code: state.code }) + } + return actual.readdir(...args) + } + } +}) + +import { listCodexSessionFiles } from './codex-session-file-discovery' + +let root: string +let sessions: string + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-codex-discovery-error-')) + state.managed = join(root, 'managed') + state.system = join(root, 'system') + state.blocked = '' + state.code = '' + sessions = join(state.managed, 'sessions') + await mkdir(sessions, { recursive: true }) + await mkdir(join(state.system, 'sessions'), { recursive: true }) +}) + +afterEach(async () => { + await rm(root, { recursive: true, force: true }) +}) + +describe('Codex history discovery failures', () => { + it.each(['EACCES', 'EPERM', 'EIO', 'UNKNOWN'])( + 'rejects an inaccessible root (%s)', + async (code) => { + state.blocked = sessions + state.code = code + await expect(listCodexSessionFiles()).rejects.toMatchObject({ code }) + } + ) + + it('rejects an unreadable child instead of publishing an incomplete history list', async () => { + await writeFile(join(sessions, 'kept.jsonl'), '') + state.blocked = join(sessions, 'inaccessible') + await mkdir(state.blocked) + state.code = 'EACCES' + await expect(listCodexSessionFiles()).rejects.toMatchObject({ code: 'EACCES' }) + }) + + it.each(['ENOENT', 'ENOTDIR'])( + 'keeps sibling files when a child disappears (%s)', + async (code) => { + const sibling = join(sessions, 'kept.jsonl') + await writeFile(sibling, '') + state.blocked = join(sessions, 'removed') + await mkdir(state.blocked) + state.code = code + await expect(listCodexSessionFiles()).resolves.toEqual([sibling]) + } + ) + + it('accepts absent homes while preserving the available history', async () => { + const sibling = join(sessions, 'kept.jsonl') + await writeFile(sibling, '') + await rm(state.system, { recursive: true }) + await expect(listCodexSessionFiles()).resolves.toEqual([sibling]) + }) +}) diff --git a/src/main/codex-usage/types.ts b/src/main/codex-usage/types.ts index 2aa0f2687b9..27de1965a5e 100644 --- a/src/main/codex-usage/types.ts +++ b/src/main/codex-usage/types.ts @@ -12,6 +12,9 @@ export type CodexUsageProcessedFile = { path: string mtimeMs: number size: number + /** Absent in older caches; identifies the file that produced this projection. */ + physicalFileId?: string | null + ctimeMs?: number } /** Everything needed to resume parsing a grown rollout where the last scan diff --git a/src/main/usage/jsonl-file-checkpoint.ts b/src/main/usage/jsonl-file-checkpoint.ts new file mode 100644 index 00000000000..27d754b4098 --- /dev/null +++ b/src/main/usage/jsonl-file-checkpoint.ts @@ -0,0 +1,309 @@ +import { createReadStream } from 'node:fs' +import { open, type FileHandle } from 'node:fs/promises' +import { createHash } from 'node:crypto' +import { + readJsonlFileSnapshot, + readJsonlHandleSnapshot, + readJsonlSnapshotRange, + releaseJsonlReadStream, + sameJsonlFileSnapshot, + verifyJsonlSnapshotContents, + type JsonlFileSnapshot +} from './jsonl-file-snapshot' + +const CHECKPOINT_WINDOW_BYTES = 4096 + +// Below this measured crossover, verifying the prefix costs more than rereading it. +export const MIN_RESUMABLE_PREFIX_BYTES = 3 * CHECKPOINT_WINDOW_BYTES + +export type JsonlFileCheckpoint = { + parsedBytes: number + boundaryDigest: string + headDigest: string + physicalFileId: string | null +} + +export type JsonlFileReader = { + path: string + handle: FileHandle + stats: JsonlFileSnapshot + snapshotCheckpoint: JsonlFileCheckpoint | null + snapshotChunks: Buffer[] | null + snapshotBytes: number + streamStartOffset: number + streamBytes: number + onChunk: (chunk: Buffer) => void + prepareRead: (startOffset: number) => Promise +} + +export function isResumablePrefixLength(parsedBytes: number): boolean { + return parsedBytes > MIN_RESUMABLE_PREFIX_BYTES +} + +async function readWindowDigest( + filePath: string, + start: number, + endExclusive: number, + reader?: JsonlFileReader +): Promise { + const expectedBytes = endExclusive - start + const hash = createHash('sha256') + let readBytes = 0 + if (expectedBytes === 0) { + return `0:${hash.digest('hex')}` + } + if (reader) { + const buffer = await readJsonlSnapshotRange( + filePath, + reader.handle, + reader.stats, + start, + endExclusive + ) + if (!buffer) { + return null + } + hash.update(buffer) + return `${buffer.length}:${hash.digest('hex')}` + } + const range = { start, end: endExclusive - 1 } + const stream = createReadStream(filePath, range) + try { + for await (const chunk of stream) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) + hash.update(buffer) + readBytes += buffer.length + } + } finally { + await releaseJsonlReadStream(stream) + } + return readBytes === expectedBytes ? `${expectedBytes}:${hash.digest('hex')}` : null +} + +async function readPrefixDigests( + filePath: string, + parsedBytes: number, + carriedHeadDigest: string | null = null, + reader?: JsonlFileReader +): Promise<{ headDigest: string; boundaryDigest: string } | null> { + const headBytes = Math.min(CHECKPOINT_WINDOW_BYTES, parsedBytes) + const headDigest = carriedHeadDigest?.startsWith(`${headBytes}:`) + ? carriedHeadDigest + : await readWindowDigest(filePath, 0, headBytes, reader) + if (headDigest === null) { + return null + } + const boundaryDigest = await readWindowDigest( + filePath, + Math.max(0, parsedBytes - CHECKPOINT_WINDOW_BYTES), + parsedBytes, + reader + ) + return boundaryDigest === null ? null : { headDigest, boundaryDigest } +} + +export function jsonlPhysicalFileId(stats: { + dev: number | bigint + ino: number | bigint +}): string | null { + return stats.ino === 0 || stats.ino === 0n ? null : `${stats.dev}:${stats.ino}` +} + +export async function openJsonlFileReader(filePath: string): Promise { + while (true) { + const handle = await open(filePath, 'r') + let retained = false + try { + const stats = await readJsonlHandleSnapshot(handle) + const reader: JsonlFileReader = { + path: filePath, + handle, + stats, + snapshotCheckpoint: null, + snapshotChunks: isResumablePrefixLength(stats.size) ? null : [], + snapshotBytes: 0, + streamStartOffset: 0, + streamBytes: 0, + onChunk: (chunk) => { + reader.streamBytes += chunk.length + if (reader.snapshotChunks) { + reader.snapshotChunks.push(chunk) + reader.snapshotBytes += chunk.length + } + }, + prepareRead: async (startOffset) => { + reader.streamStartOffset = Math.min(startOffset, stats.size) + if (!reader.snapshotChunks || reader.snapshotBytes !== 0 || startOffset <= 0) { + return + } + const skipped = await readJsonlSnapshotRange( + filePath, + handle, + stats, + 0, + Math.min(startOffset, stats.size) + ) + if (!skipped) { + return + } + reader.snapshotChunks.push(skipped) + reader.snapshotBytes += skipped.length + } + } + if (!isResumablePrefixLength(stats.size)) { + retained = true + return reader + } + const digests = await readPrefixDigests(filePath, stats.size, null, reader) + if (!digests) { + continue + } + reader.snapshotCheckpoint = { + parsedBytes: stats.size, + ...digests, + physicalFileId: jsonlPhysicalFileId(stats) + } + retained = true + return reader + } finally { + if (!retained) { + await handle.close() + } + } + } +} + +async function readPhysicalFileId(filePath: string): Promise { + try { + const fileStat = await readJsonlFileSnapshot(filePath) + return jsonlPhysicalFileId(fileStat) + } catch { + return null + } +} + +export async function buildJsonlFileCheckpoint( + filePath: string, + parsedBytes: number, + verifiedHeadDigest: string | null = null, + reader?: JsonlFileReader +): Promise { + if (!isResumablePrefixLength(parsedBytes)) { + return null + } + if (reader?.snapshotCheckpoint?.parsedBytes === parsedBytes) { + return reader.snapshotCheckpoint + } + const digests = await readPrefixDigests( + filePath, + parsedBytes, + verifiedHeadDigest ?? reader?.snapshotCheckpoint?.headDigest, + reader + ) + return digests === null + ? null + : { + parsedBytes, + ...digests, + physicalFileId: reader + ? jsonlPhysicalFileId(reader.stats) + : await readPhysicalFileId(filePath) + } +} + +// Append-only evidence catches rotation, truncation and head/tail rewrites; it cannot prove the middle. +export async function resolveJsonlFileCheckpoint( + filePath: string, + checkpoint: JsonlFileCheckpoint | null | undefined, + reader?: JsonlFileReader +): Promise { + if ( + !checkpoint || + !Number.isSafeInteger(checkpoint.parsedBytes) || + !isResumablePrefixLength(checkpoint.parsedBytes) || + (reader && checkpoint.parsedBytes > reader.stats.size) || + typeof checkpoint.boundaryDigest !== 'string' || + typeof checkpoint.headDigest !== 'string' + ) { + return null + } + const currentFileId = reader + ? jsonlPhysicalFileId(reader.stats) + : await readPhysicalFileId(filePath) + if ( + checkpoint.physicalFileId !== null && + currentFileId !== null && + checkpoint.physicalFileId !== currentFileId + ) { + return null + } + const digests = await readPrefixDigests( + filePath, + checkpoint.parsedBytes, + reader?.snapshotCheckpoint?.headDigest, + reader + ) + return digests?.boundaryDigest === checkpoint.boundaryDigest && + digests?.headDigest === checkpoint.headDigest + ? checkpoint + : null +} + +export async function validateJsonlFileReader( + reader: JsonlFileReader, + originalCheckpoint?: JsonlFileCheckpoint | null +): Promise { + const [current, pathStats] = await Promise.all([ + readJsonlHandleSnapshot(reader.handle), + readJsonlFileSnapshot(reader.path) + ]) + if (reader.streamBytes !== reader.stats.size - reader.streamStartOffset) { + if ( + sameJsonlFileSnapshot(reader.stats, current) && + sameJsonlFileSnapshot(reader.stats, pathStats) + ) { + throw new Error('JSONL snapshot stream ended before the unchanged file boundary.') + } + return false + } + if ( + current.dev !== reader.stats.dev || + current.ino !== reader.stats.ino || + pathStats.dev !== current.dev || + pathStats.ino !== current.ino || + current.size < reader.stats.size || + (current.size === reader.stats.size && + (current.mtimeMs !== reader.stats.mtimeMs || current.ctimeMs !== reader.stats.ctimeMs)) + ) { + return false + } + const snapshot = reader.snapshotCheckpoint + if (!snapshot) { + if (!reader.snapshotChunks || reader.snapshotBytes !== reader.stats.size) { + return false + } + const chunks = reader.snapshotChunks + reader.snapshotChunks = null + return verifyJsonlSnapshotContents(reader.path, reader.handle, reader.stats, chunks) + } + const snapshotDigests = await readPrefixDigests(reader.path, snapshot.parsedBytes, null, reader) + if ( + snapshotDigests?.headDigest !== snapshot.headDigest || + snapshotDigests?.boundaryDigest !== snapshot.boundaryDigest + ) { + return false + } + if (!originalCheckpoint || originalCheckpoint.parsedBytes === snapshot.parsedBytes) { + return true + } + const originalBoundary = await readWindowDigest( + reader.path, + Math.max(0, originalCheckpoint.parsedBytes - CHECKPOINT_WINDOW_BYTES), + originalCheckpoint.parsedBytes, + reader + ) + return ( + originalCheckpoint.headDigest === snapshotDigests.headDigest && + originalCheckpoint.boundaryDigest === originalBoundary + ) +} diff --git a/src/main/usage/jsonl-file-reader.test.ts b/src/main/usage/jsonl-file-reader.test.ts new file mode 100644 index 00000000000..67dbd32ac0f --- /dev/null +++ b/src/main/usage/jsonl-file-reader.test.ts @@ -0,0 +1,253 @@ +import { appendFile, mkdtemp, rename, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + MIN_RESUMABLE_PREFIX_BYTES, + jsonlPhysicalFileId, + openJsonlFileReader, + resolveJsonlFileCheckpoint, + validateJsonlFileReader, + type JsonlFileReader +} from './jsonl-file-checkpoint' +import { readJsonlLinesFromOffset, type JsonlLineAtOffset } from './jsonl-line-offsets' + +let directory: string +let filePath: string +let reader: JsonlFileReader | undefined + +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'orca-jsonl-reader-')) + filePath = join(directory, 'transcript.jsonl') +}) + +afterEach(async () => { + await reader?.handle.close() + reader = undefined + await rm(directory, { recursive: true, force: true }) +}) + +async function openFixture(content: string): Promise { + await writeFile(filePath, content) + reader = await openJsonlFileReader(filePath) + return reader +} + +async function collect(active: JsonlFileReader, start = 0): Promise { + const lines: JsonlLineAtOffset[] = [] + for await (const line of readJsonlLinesFromOffset(filePath, start, active)) { + lines.push(line) + } + return lines +} + +function positionalReadArgs(args: readonly unknown[]): [Buffer, number, number, number] { + const [buffer, offset, length, position] = args + if ( + !Buffer.isBuffer(buffer) || + typeof offset !== 'number' || + typeof length !== 'number' || + typeof position !== 'number' + ) { + throw new Error('Expected positional descriptor read arguments.') + } + return [buffer, offset, length, position] +} + +describe('pinned small JSONL reader', () => { + it('encodes large physical identifiers exactly and preserves numeric caller compatibility', () => { + expect(jsonlPhysicalFileId({ dev: 1n, ino: 9_007_199_254_740_992n })).toBe('1:9007199254740992') + expect(jsonlPhysicalFileId({ dev: 1n, ino: 9_007_199_254_740_993n })).toBe('1:9007199254740993') + expect(jsonlPhysicalFileId({ dev: 3, ino: 4 })).toBe('3:4') + expect(jsonlPhysicalFileId({ dev: 3, ino: 0 })).toBeNull() + expect(jsonlPhysicalFileId({ dev: 3n, ino: 0n })).toBeNull() + }) + + it.each([Number.NaN, Infinity, -1, 1.5, Number.MAX_SAFE_INTEGER + 1])( + 'rejects an invalid persisted offset %s before attempting a range read', + async (parsedBytes) => { + await expect( + resolveJsonlFileCheckpoint(filePath, { + parsedBytes, + headDigest: '4096:invalid', + boundaryDigest: '4096:invalid', + physicalFileId: null + }) + ).resolves.toBeNull() + } + ) + + it('verifies an empty snapshot without creating a stream or issuing a read', async () => { + const active = await openFixture('') + const read = vi.spyOn(active.handle, 'read') + const stream = vi.spyOn(active.handle, 'createReadStream') + + expect(await collect(active)).toEqual([]) + expect(await validateJsonlFileReader(active)).toBe(true) + expect(stream).not.toHaveBeenCalled() + expect(read).not.toHaveBeenCalled() + }) + + it('reads and verifies all bytes twice without creating a stream', async () => { + const contents = 'ไผš๐Ÿ™‚\r\n\r\nnext\nๅฐพ๐Ÿ™‚' + const active = await openFixture(contents) + const read = vi.spyOn(active.handle, 'read') + const stream = vi.spyOn(active.handle, 'createReadStream') + + expect(await collect(active)).toEqual([ + { line: 'ไผš๐Ÿ™‚', endOffset: Buffer.byteLength('ไผš๐Ÿ™‚\r\n'), terminated: true }, + { line: '', endOffset: Buffer.byteLength('ไผš๐Ÿ™‚\r\n\r\n'), terminated: true }, + { line: 'next', endOffset: Buffer.byteLength('ไผš๐Ÿ™‚\r\n\r\nnext\n'), terminated: true }, + { line: 'ๅฐพ๐Ÿ™‚', endOffset: Buffer.byteLength(contents), terminated: false } + ]) + expect(await validateJsonlFileReader(active)).toBe(true) + expect(stream).not.toHaveBeenCalled() + expect(read).toHaveBeenCalledTimes(2) + expect( + (await Promise.all(read.mock.results.map((result) => result.value))).map( + (result) => result.bytesRead + ) + ).toEqual([Buffer.byteLength(contents), Buffer.byteLength(contents)]) + }) + + it('accounts for a skipped legacy prefix in the full snapshot proof', async () => { + const contents = 'legacyไผš\r\nnew๐Ÿ™‚\r\n' + const active = await openFixture(contents) + const offset = Buffer.byteLength('legacyไผš\r\n') + const read = vi.spyOn(active.handle, 'read') + + expect(await collect(active, offset)).toEqual([ + { line: 'new๐Ÿ™‚', endOffset: Buffer.byteLength(contents), terminated: true } + ]) + expect(await validateJsonlFileReader(active)).toBe(true) + expect(active.streamBytes).toBe(Buffer.byteLength(contents) - offset) + expect(active.snapshotBytes).toBe(Buffer.byteLength(contents)) + expect(read).toHaveBeenCalledTimes(3) + }) + + it('accumulates short positional reads without exposing gaps or duplicate bytes', async () => { + const contents = 'ไผš๐Ÿ™‚\r\nnext\n็ต‚๐Ÿ™‚' + const active = await openFixture(contents) + const read = active.handle.read.bind(active.handle) + vi.spyOn(active.handle, 'read').mockImplementation(async (...args) => { + const [buffer, offset, length, position] = positionalReadArgs(args) + return read(buffer, offset, Math.min(length, 3), position) + }) + + expect(await collect(active)).toEqual([ + { line: 'ไผš๐Ÿ™‚', endOffset: Buffer.byteLength('ไผš๐Ÿ™‚\r\n'), terminated: true }, + { line: 'next', endOffset: Buffer.byteLength('ไผš๐Ÿ™‚\r\nnext\n'), terminated: true }, + { line: '็ต‚๐Ÿ™‚', endOffset: Buffer.byteLength(contents), terminated: false } + ]) + expect(await validateJsonlFileReader(active)).toBe(true) + }) + + it('rejects a stable premature EOF instead of publishing partial history', async () => { + const active = await openFixture('one\ntwo\n') + const read = active.handle.read.bind(active.handle) + vi.spyOn(active.handle, 'read').mockImplementationOnce(async (...args) => { + const result = await read(...args) + return { ...result, bytesRead: 0 } + }) + + await expect(collect(active)).rejects.toThrow(/snapshot read.*unchanged file boundary/) + expect(active.streamBytes).toBe(0) + expect(active.snapshotBytes).toBe(0) + }) + + it('rejects an unchanged EOF after a positive short read without yielding a partial line', async () => { + const active = await openFixture('one\ntwo\n') + const read = active.handle.read.bind(active.handle) + vi.spyOn(active.handle, 'read') + .mockImplementationOnce(async (...args) => { + const [buffer, offset, length, position] = positionalReadArgs(args) + return read(buffer, offset, Math.min(length, 2), position) + }) + .mockImplementationOnce(async (...args) => { + const result = await read(...args) + return { ...result, bytesRead: 0 } + }) + + await expect(collect(active)).rejects.toThrow(/snapshot read.*unchanged file boundary/) + expect(active.streamBytes).toBe(0) + expect(active.snapshotBytes).toBe(0) + }) + + it('rejects a zero-byte verification read even after parsing every line', async () => { + const active = await openFixture('one\ntwo\n') + await collect(active) + const read = active.handle.read.bind(active.handle) + vi.spyOn(active.handle, 'read').mockImplementationOnce(async (...args) => { + const result = await read(...args) + return { ...result, bytesRead: 0 } + }) + + await expect(validateJsonlFileReader(active)).rejects.toThrow( + /snapshot read.*unchanged file boundary/ + ) + }) + + it('reads a skipped legacy prefix again when its proof changes during parsing', async () => { + const active = await openFixture('old\nnext\n') + await collect(active, 4) + await writeFile(filePath, 'NEW\nnext\nappend\n') + + expect(await validateJsonlFileReader(active)).toBe(false) + }) + + it('leaves an append after the captured boundary for the next scan', async () => { + const original = 'oneไผš๐Ÿ™‚\r\n' + const active = await openFixture(original) + await appendFile(filePath, 'two๐Ÿ™‚\n') + + expect(await collect(active)).toEqual([ + { line: 'oneไผš๐Ÿ™‚', endOffset: Buffer.byteLength(original), terminated: true } + ]) + expect(await validateJsonlFileReader(active)).toBe(true) + expect(active.streamBytes).toBe(Buffer.byteLength(original)) + }) + + it('rejects a full small-file middle rewrite even when the source also grows', async () => { + const original = 'head\nold\ntail\n' + const active = await openFixture(original) + await collect(active) + await writeFile(filePath, 'head\nNEW\ntail\nappend\n') + + expect(await validateJsonlFileReader(active)).toBe(false) + }) + + it('keeps the pinned handle but rejects path rotation', async () => { + const original = 'one\n' + const active = await openFixture(original) + const replacement = join(directory, 'replacement.jsonl') + await writeFile(replacement, 'two\n') + await rename(filePath, join(directory, 'rotated.jsonl')) + await rename(replacement, filePath) + + expect(await collect(active)).toEqual([{ line: 'one', endOffset: 4, terminated: true }]) + expect(await validateJsonlFileReader(active)).toBe(false) + }) + + it('invalidates a truncation encountered by the descriptor loop', async () => { + const active = await openFixture('one\ntwo\n') + await writeFile(filePath, 'one\n') + + expect(await collect(active)).toEqual([]) + expect(await validateJsonlFileReader(active)).toBe(false) + }) + + it('uses the bounded buffer at the crossover and streams larger snapshots', async () => { + const active = await openFixture(`${'x'.repeat(MIN_RESUMABLE_PREFIX_BYTES - 1)}\n`) + const stream = vi.spyOn(active.handle, 'createReadStream') + await collect(active) + expect(await validateJsonlFileReader(active)).toBe(true) + expect(stream).not.toHaveBeenCalled() + await active.handle.close() + reader = undefined + const large = await openFixture(`${'x'.repeat(MIN_RESUMABLE_PREFIX_BYTES)}\n`) + const largeStream = vi.spyOn(large.handle, 'createReadStream') + await collect(large) + expect(await validateJsonlFileReader(large)).toBe(true) + expect(largeStream).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/usage/jsonl-file-snapshot.test.ts b/src/main/usage/jsonl-file-snapshot.test.ts new file mode 100644 index 00000000000..188d35ba5f8 --- /dev/null +++ b/src/main/usage/jsonl-file-snapshot.test.ts @@ -0,0 +1,92 @@ +import { mkdtemp, open, rm, stat, utimes, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + normalizeJsonlFileSnapshot, + readJsonlFileSnapshot, + readJsonlHandleSnapshot, + sameJsonlFileSnapshot +} from './jsonl-file-snapshot' + +let directory: string +let filePath: string + +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'orca-jsonl-snapshot-')) + filePath = join(directory, 'transcript.jsonl') + await writeFile(filePath, 'one\n') +}) + +afterEach(async () => { + await rm(directory, { recursive: true, force: true }) +}) + +describe('JSONL file snapshot normalization', () => { + it.each(['dev', 'ino'] as const)( + 'distinguishes exact %s identifiers that collide as JavaScript numbers', + async (field) => { + const stats = await stat(filePath, { bigint: true }) + const lower = { ...stats, [field]: 9_007_199_254_740_992n } + const upper = { ...stats, [field]: 9_007_199_254_740_993n } + + expect(Number(lower[field])).toBe(Number(upper[field])) + expect(normalizeJsonlFileSnapshot(lower)[field]).not.toBe( + normalizeJsonlFileSnapshot(upper)[field] + ) + expect( + sameJsonlFileSnapshot(normalizeJsonlFileSnapshot(lower), normalizeJsonlFileSnapshot(upper)) + ).toBe(false) + } + ) + + it.each([0.123456789, 1.999999999, 1_784_029_260.125, 2_147_483_647.875])( + 'preserves numeric stat timestamps at fractional epoch seconds %s', + async (seconds) => { + await utimes(filePath, seconds, seconds) + const numeric = await stat(filePath) + const snapshot = await readJsonlFileSnapshot(filePath) + const handle = await open(filePath, 'r') + try { + expect(snapshot.mtimeMs).toBe(numeric.mtimeMs) + expect(snapshot.ctimeMs).toBe(numeric.ctimeMs) + expect(snapshot.size).toBe(numeric.size) + expect(await readJsonlHandleSnapshot(handle)).toEqual(snapshot) + } finally { + await handle.close() + } + } + ) + + it('retains fractional nanoseconds instead of truncating bigint milliseconds', async () => { + const stats = await stat(filePath, { bigint: true }) + const snapshot = normalizeJsonlFileSnapshot({ + ...stats, + mtimeNs: 1_784_029_260_123_456_789n, + ctimeNs: 1_784_029_260_987_654_321n + }) + + expect(snapshot.mtimeMs).toBe(1_784_029_260_000 + 123.456789) + expect(snapshot.ctimeMs).toBe(1_784_029_260_000 + 987.654321) + expect(snapshot.mtimeMs).not.toBe(1_784_029_260_123) + expect(snapshot.ctimeMs).not.toBe(1_784_029_260_987) + }) + + it('accepts zero and the largest safe byte offset', async () => { + const stats = await stat(filePath, { bigint: true }) + + expect(normalizeJsonlFileSnapshot({ ...stats, size: 0n }).size).toBe(0) + expect( + normalizeJsonlFileSnapshot({ ...stats, size: BigInt(Number.MAX_SAFE_INTEGER) }).size + ).toBe(Number.MAX_SAFE_INTEGER) + }) + + it.each([-1n, BigInt(Number.MAX_SAFE_INTEGER) + 1n, 1n << 1024n])( + 'rejects a file size that is not a safe byte offset: %s', + async (size) => { + const stats = await stat(filePath, { bigint: true }) + + expect(() => normalizeJsonlFileSnapshot({ ...stats, size })).toThrow(/safe byte offset/) + } + ) +}) diff --git a/src/main/usage/jsonl-file-snapshot.ts b/src/main/usage/jsonl-file-snapshot.ts new file mode 100644 index 00000000000..5012a03990b --- /dev/null +++ b/src/main/usage/jsonl-file-snapshot.ts @@ -0,0 +1,113 @@ +import type { BigIntStats, ReadStream } from 'node:fs' +import { stat, type FileHandle } from 'node:fs/promises' + +const SNAPSHOT_FIELDS = ['dev', 'ino', 'size', 'mtimeMs', 'ctimeMs'] as const + +export type JsonlFileSnapshot = { + dev: bigint + ino: bigint + size: number + mtimeMs: number + ctimeMs: number +} + +export async function releaseJsonlReadStream(stream: ReadStream): Promise { + if (!stream.closed) { + await new Promise((resolve) => { + stream.once('close', resolve) + stream.destroy() + }) + } +} + +function nanosecondsToMilliseconds(value: bigint): number { + return Number(value / 1_000_000_000n) * 1000 + Number(value % 1_000_000_000n) / 1_000_000 +} + +export function normalizeJsonlFileSnapshot( + stats: Pick +): JsonlFileSnapshot { + const size = Number(stats.size) + if (!Number.isSafeInteger(size) || size < 0) { + throw new RangeError('JSONL snapshot size cannot be represented as a safe byte offset.') + } + return { + dev: stats.dev, + ino: stats.ino, + size, + mtimeMs: nanosecondsToMilliseconds(stats.mtimeNs), + ctimeMs: nanosecondsToMilliseconds(stats.ctimeNs) + } +} + +export async function readJsonlFileSnapshot(filePath: string): Promise { + return normalizeJsonlFileSnapshot(await stat(filePath, { bigint: true })) +} + +export async function readJsonlHandleSnapshot(handle: FileHandle): Promise { + return normalizeJsonlFileSnapshot(await handle.stat({ bigint: true })) +} + +export function sameJsonlFileSnapshot(left: JsonlFileSnapshot, right: JsonlFileSnapshot): boolean { + return SNAPSHOT_FIELDS.every((field) => left[field] === right[field]) +} + +export async function rejectUnchangedJsonlFileRead( + filePath: string, + handle: FileHandle, + initial: JsonlFileSnapshot +): Promise { + const [current, pathStats] = await Promise.all([ + readJsonlHandleSnapshot(handle), + readJsonlFileSnapshot(filePath) + ]) + if (sameJsonlFileSnapshot(initial, current) && sameJsonlFileSnapshot(initial, pathStats)) { + throw new Error('JSONL snapshot read ended before the unchanged file boundary.') + } +} + +export async function readJsonlSnapshotRange( + filePath: string, + handle: FileHandle, + initial: JsonlFileSnapshot, + start: number, + endExclusive: number +): Promise { + const buffer = Buffer.allocUnsafe(endExclusive - start) + let consumed = 0 + while (consumed < buffer.length) { + const { bytesRead } = await handle.read( + buffer, + consumed, + buffer.length - consumed, + start + consumed + ) + if (bytesRead === 0) { + await rejectUnchangedJsonlFileRead(filePath, handle, initial) + return null + } + consumed += bytesRead + } + return buffer +} + +export async function verifyJsonlSnapshotContents( + filePath: string, + handle: FileHandle, + initial: JsonlFileSnapshot, + pieces: readonly Buffer[] +): Promise { + const current = await readJsonlSnapshotRange(filePath, handle, initial, 0, initial.size) + if (!current) { + return false + } + let consumed = 0 + for (const piece of pieces) { + const end = consumed + piece.length + if (end > current.length || piece.compare(current, consumed, end) !== 0) { + return false + } + consumed = end + } + return consumed === current.length +} diff --git a/src/main/usage/jsonl-line-offsets.ts b/src/main/usage/jsonl-line-offsets.ts index cea99cd4162..5a6d8230145 100644 --- a/src/main/usage/jsonl-line-offsets.ts +++ b/src/main/usage/jsonl-line-offsets.ts @@ -5,6 +5,8 @@ * CRLF transcript would otherwise drift one byte per line. */ import { createReadStream } from 'node:fs' +import { isResumablePrefixLength, type JsonlFileReader } from './jsonl-file-checkpoint' +import { readJsonlSnapshotRange, releaseJsonlReadStream } from './jsonl-file-snapshot' const LINE_FEED = 0x0a const CARRIAGE_RETURN = 0x0d @@ -23,17 +25,61 @@ function decodeLine(pieces: Buffer[]): string { return raw.toString('utf-8', 0, end) } +async function* readJsonlChunks( + filePath: string, + startOffset: number, + reader?: Pick +): AsyncGenerator { + if (reader && !isResumablePrefixLength(reader.stats.size)) { + // Tiny snapshots avoid stream setup while keeping the same pinned-range proof. + const chunk = await readJsonlSnapshotRange( + filePath, + reader.handle, + reader.stats, + startOffset, + reader.stats.size + ) + if (chunk) { + yield chunk + } + return + } + const stream = reader + ? reader.handle.createReadStream({ + start: startOffset, + end: reader.stats.size - 1, + autoClose: false + }) + : createReadStream(filePath, { start: startOffset }) + try { + for await (const chunk of stream) { + yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) + } + } finally { + if (!reader) { + await releaseJsonlReadStream(stream) + } + } +} + export async function* readJsonlLinesFromOffset( filePath: string, - startOffset: number + startOffset: number, + reader?: Pick & { + onChunk?: JsonlFileReader['onChunk'] + prepareRead?: JsonlFileReader['prepareRead'] + } ): AsyncGenerator { - const stream = createReadStream(filePath, { start: startOffset }) + await reader?.prepareRead?.(startOffset) + if (reader && reader.stats.size <= startOffset) { + return + } const pending: Buffer[] = [] let pendingBytes = 0 let endOffset = startOffset - for await (const rawChunk of stream) { - const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk) + for await (const chunk of readJsonlChunks(filePath, startOffset, reader)) { + reader?.onChunk?.(chunk) let searchFrom = 0 for (;;) { const lineFeedIndex = chunk.indexOf(LINE_FEED, searchFrom) diff --git a/src/main/usage/usage-breakdown-scaling.test.ts b/src/main/usage/usage-breakdown-scaling.test.ts index 2412d5d097d..40f51ab2888 100644 --- a/src/main/usage/usage-breakdown-scaling.test.ts +++ b/src/main/usage/usage-breakdown-scaling.test.ts @@ -55,6 +55,145 @@ it('folds events without rescanning accumulated location breakdowns', () => { expect(result.dailyAggregates).toHaveLength(1000) }) +it('folds streamed events immediately with the batch counters, metrics and metadata', () => { + const [seed] = events(1) + const first: Event = { + ...seed, + sessionId: 'mixed', + timestamp: '2026-09-09T00:00:00Z', + day: '2026-09-09', + projectKey: 'z', + projectLabel: 'Z', + repoId: 'repo-z', + worktreeId: 'tree-z', + model: 'large', + inputTokens: 3, + cachedInputTokens: 1, + outputTokens: 5, + reasoningOutputTokens: 2, + totalTokens: 8, + cost: 2 + } + const input: Event[] = [ + first, + { + ...seed, + sessionId: 'unknown', + timestamp: '2026-09-10T00:00:00Z', + day: '2026-09-10', + model: null + }, + { + ...first, + timestamp: '2026-09-07T00:00:00Z', + day: '2026-09-07', + projectKey: 'a', + projectLabel: 'A', + repoId: 'repo-a', + worktreeId: 'tree-a', + model: 'small', + inputTokens: 7, + cachedInputTokens: 3, + outputTokens: 11, + reasoningOutputTokens: 5, + totalTokens: 18, + cost: 5 + }, + { + ...first, + timestamp: '2026-09-09T12:00:00Z', + projectLabel: 'Changed label', + repoId: 'changed-repo', + worktreeId: 'changed-tree', + inputTokens: 13, + cachedInputTokens: 7, + outputTokens: 17, + reasoningOutputTokens: 11, + totalTokens: 30, + cost: 7 + } + ] + const expected = aggregation.aggregate(input) + const accumulator = aggregation.createAccumulator() + for (const event of input) { + accumulator.add(event) + event.totalTokens = 999 + event.cost = 999 + event.model = 'changed' + event.projectLabel = 'Changed after adding' + } + const result = accumulator.finalize() + expect(result).toEqual(expected) + expect(result.sessions.map((session) => session.sessionId)).toEqual(['unknown', 'mixed']) + expect(result.sessions[0].primaryModel).toBe('Unknown model') + const mixed = result.sessions[1] + expect(mixed).toMatchObject({ + firstTimestamp: '2026-09-07T00:00:00Z', + lastTimestamp: '2026-09-09T12:00:00Z', + primaryModel: 'Mixed models', + hasMixedModels: true, + primaryProjectLabel: 'Multiple locations', + hasMixedLocations: true, + primaryRepoId: 'repo-z', + primaryWorktreeId: 'tree-z', + eventCount: 3, + totalInputTokens: 23, + totalCachedInputTokens: 11, + totalOutputTokens: 33, + totalReasoningOutputTokens: 18, + totalTokens: 56, + cost: 14 + }) + expect(mixed.locationBreakdown).toMatchObject([ + { locationKey: 'z', projectLabel: 'Z', totalTokens: 38, cost: 9 }, + { locationKey: 'a', projectLabel: 'A', totalTokens: 18, cost: 5 } + ]) + expect(mixed.modelBreakdown).toMatchObject([ + { modelKey: 'large', totalTokens: 38, cost: 9 }, + { modelKey: 'small', totalTokens: 18, cost: 5 } + ]) + expect(mixed.locationModelBreakdown).toMatchObject([ + { locationKey: 'z', modelKey: 'large', totalTokens: 38, cost: 9 }, + { locationKey: 'a', modelKey: 'small', totalTokens: 18, cost: 5 } + ]) + expect(result.dailyAggregates).toMatchObject([ + { day: '2026-09-07', projectKey: 'a', totalTokens: 18, cost: 5 }, + { day: '2026-09-09', projectKey: 'z', projectLabel: 'Z', totalTokens: 38, cost: 9 }, + { day: '2026-09-10', projectKey: 'path-0', totalTokens: 3, cost: 0.5 } + ]) +}) + +it('keeps first encounter order when streamed rollup sort keys tie', () => { + const accumulator = aggregation.createAccumulator() + for (const [index, event] of events(4).entries()) { + accumulator.add({ ...event, sessionId: `session-${index % 2}`, projectLabel: 'Same label' }) + } + const result = accumulator.finalize() + expect(result.sessions.map((session) => session.sessionId)).toEqual(['session-0', 'session-1']) + expect(result.sessions[0].locationBreakdown.map((entry) => entry.locationKey)).toEqual([ + 'path-0', + 'path-2' + ]) + expect(result.sessions[0].modelBreakdown.map((entry) => entry.modelKey)).toEqual([ + 'model-0', + 'model-2' + ]) + expect(result.dailyAggregates.map((entry) => entry.projectKey)).toEqual([ + 'path-0', + 'path-1', + 'path-2', + 'path-3' + ]) +}) + +it('finalizes an empty stream once and rejects subsequent additions', () => { + const accumulator = aggregation.createAccumulator() + const result = accumulator.finalize() + expect(result).toEqual({ sessions: [], dailyAggregates: [] }) + expect(accumulator.finalize()).toBe(result) + expect(() => accumulator.add(events(1)[0])).toThrow('Cannot add usage events after finalization') +}) + it('merges rollups using first-match indexes without mutating source breakdowns', () => { const source = aggregation.aggregate(events(1000)).sessions[0] const existing = structuredClone(source) diff --git a/src/main/usage/usage-event-aggregation.ts b/src/main/usage/usage-event-aggregation.ts index d58a854fb37..1e7b9fc0c64 100644 --- a/src/main/usage/usage-event-aggregation.ts +++ b/src/main/usage/usage-event-aggregation.ts @@ -25,6 +25,11 @@ export type UsageEventAggregationOptions = { cloneSessionForMerge(session: UsageSession): UsageSession } +export type UsageEventAggregationResult = { + sessions: UsageSession[] + dailyAggregates: UsageDailyAggregate[] +} + export function createUsageEventAggregation< TEvent extends UsageAttributedEventFields, TMetric extends object @@ -214,15 +219,17 @@ export function createUsageEventAggregation< ) } - function aggregate(events: TEvent[]): { - sessions: UsageSession[] - dailyAggregates: UsageDailyAggregate[] - } { + /** Finalization closes the stream and releases its lookup indexes. */ + function createAccumulator() { const sessionsById = new Map>() const dailyByKey = new Map>() const breakdownsBySession = new Map>() + let result: UsageEventAggregationResult | null = null - for (const event of events) { + function add(event: TEvent): void { + if (result) { + throw new Error('Cannot add usage events after finalization') + } const eventMetric = metric.fromEvent(event) const session = sessionsById.get(event.sessionId) ?? createEmptySession(event) if (!sessionsById.has(event.sessionId)) { @@ -269,14 +276,34 @@ export function createUsageEventAggregation< metric.fold(daily, eventMetric) } - return { - sessions: finalizeSessions(sessionsById), - dailyAggregates: sortDailyAggregates(dailyByKey) + function finalize(): UsageEventAggregationResult { + if (result) { + return result + } + result = { + sessions: finalizeSessions(sessionsById), + dailyAggregates: sortDailyAggregates(dailyByKey) + } + sessionsById.clear() + dailyByKey.clear() + breakdownsBySession.clear() + return result } + + return { add, finalize } + } + + function aggregate(events: TEvent[]) { + const accumulator = createAccumulator() + for (const event of events) { + accumulator.add(event) + } + return accumulator.finalize() } return { aggregate, + createAccumulator, finalizeSessions, sortDailyAggregates, mergeSessions: (