mirror of
https://github.com/stablyai/orca.git
synced 2026-10-31 00:57:45 +00:00
Reduce memory used to count Codex usage and avoid partial totals (#26966)
This commit is contained in:
@@ -16,6 +16,10 @@ on:
|
||||
# without these a rename lands green and only breaks the next nightly.
|
||||
- 'src/main/sqlite/sync-database.test.ts'
|
||||
- 'src/main/runtime/orchestration/db/row-column-lists.test.ts'
|
||||
- 'src/main/codex-usage/codex-rollout-file-lifetime.test.ts'
|
||||
- 'src/main/codex-usage/scanner-incremental-append.test.ts'
|
||||
- 'src/main/usage/jsonl-file-reader.test.ts'
|
||||
- 'src/main/usage/jsonl-file-snapshot.test.ts'
|
||||
- 'src/relay/fs-path-metadata-symlink-concurrency.test.ts'
|
||||
- 'src/renderer/src/components/editor/rich-markdown-list-tokenizers.test.ts'
|
||||
- 'src/renderer/src/components/editor/rich-markdown-lowlight-cache.test.ts'
|
||||
|
||||
@@ -16,8 +16,9 @@ manual review. A warning identifies repeated setup, not proof of visible lag.
|
||||
|
||||
`pnpm test:perf:contracts` runs the explicit selection in
|
||||
`vitest.performance.config.ts`: SQLite statement reuse and schema parity, relay
|
||||
filesystem concurrency, tokenizer rejection, highlighting cache, queued
|
||||
cancellation, terminal backing-memory retention and detector fixtures. Missing
|
||||
filesystem concurrency, incremental Codex scans and reader lifetimes, pinned JSONL
|
||||
readers and snapshots, tokenizer rejection, highlighting cache, queued cancellation,
|
||||
terminal backing-memory retention and detector fixtures. Missing
|
||||
listed files fail configuration loading. Tests run serially, without retries,
|
||||
and inherit the full suite's setup and forced-GC support. This makes existing
|
||||
regression coverage easy to run and attribute; it does not create new workload
|
||||
|
||||
@@ -0,0 +1,300 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { appendFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { availableParallelism, cpus, release, tmpdir, totalmem } from 'node:os'
|
||||
import { dirname, join, relative, resolve, sep } from 'node:path'
|
||||
import { performance } from 'node:perf_hooks'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
const root = resolve(import.meta.dirname, '../..')
|
||||
const digest = (value) => createHash('sha256').update(value).digest('hex')
|
||||
const args = process.argv.slice(2)
|
||||
|
||||
function argument(name, fallback) {
|
||||
const index = args.indexOf(name)
|
||||
return index === -1 ? fallback : args[index + 1]
|
||||
}
|
||||
|
||||
function positiveInteger(value, label) {
|
||||
const parsed = Number(value)
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
throw new Error(`${label} must be a positive integer`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
function memorySnapshot(phase, started) {
|
||||
globalThis.gc()
|
||||
globalThis.gc()
|
||||
return { phase, elapsedMs: performance.now() - started, ...process.memoryUsage() }
|
||||
}
|
||||
|
||||
async function runWorker() {
|
||||
assert.equal(typeof globalThis.gc, 'function', 'Run workers with --expose-gc')
|
||||
const parser = await import(pathToFileURL(argument('--bundle')).href)
|
||||
const count = positiveInteger(argument('--events'), 'events')
|
||||
const stride = positiveInteger(argument('--stride'), 'stride')
|
||||
const intervalMs = positiveInteger(argument('--sample-ms'), 'sample-ms')
|
||||
const arm = argument('--arm')
|
||||
let claimQueries = 0
|
||||
let committedKeys = 0
|
||||
const canClaim = () => claimQueries++ % stride === 0
|
||||
const options =
|
||||
arm === 'baseline'
|
||||
? { claimEventKey: canClaim }
|
||||
: { canClaimEventKey: canClaim, commitEventKey: () => committedKeys++ }
|
||||
const started = performance.now()
|
||||
const baseline = memorySnapshot('before', started)
|
||||
const samples = []
|
||||
const timer = setInterval(() => samples.push(memorySnapshot('during', started)), intervalMs)
|
||||
let result
|
||||
try {
|
||||
result = await parser.parseCodexUsageFile(argument('--fixture'), () => null, options)
|
||||
} finally {
|
||||
clearInterval(timer)
|
||||
}
|
||||
globalThis.__orcaMemoryResult = result
|
||||
const final = memorySnapshot('after', started)
|
||||
const duringSampleCount = samples.length
|
||||
samples.push(final)
|
||||
const expected = Math.ceil(count / stride)
|
||||
const correctness = {
|
||||
claimQueries,
|
||||
committedKeys: arm === 'baseline' ? null : committedKeys,
|
||||
ownedKeys: result.ownedEventKeys.length,
|
||||
countedEvents: result.sessions.reduce((sum, session) => sum + session.eventCount, 0),
|
||||
inputTokens: result.sessions.reduce((sum, session) => sum + session.totalInputTokens, 0),
|
||||
outputTokens: result.sessions.reduce((sum, session) => sum + session.totalOutputTokens, 0),
|
||||
totalTokens: result.sessions.reduce((sum, session) => sum + session.totalTokens, 0),
|
||||
hasDeferredClaims: result.hasDeferredClaims,
|
||||
projectionSha256: digest(
|
||||
JSON.stringify({
|
||||
sessions: result.sessions,
|
||||
dailyAggregates: result.dailyAggregates,
|
||||
ownedEventKeys: result.ownedEventKeys,
|
||||
hasDeferredClaims: result.hasDeferredClaims
|
||||
})
|
||||
)
|
||||
}
|
||||
assert.equal(correctness.claimQueries, count)
|
||||
assert.equal(correctness.ownedKeys, expected)
|
||||
assert.equal(correctness.countedEvents, expected)
|
||||
assert.equal(correctness.inputTokens, expected * 100)
|
||||
assert.equal(correctness.outputTokens, expected * 10)
|
||||
assert.equal(correctness.totalTokens, expected * 110)
|
||||
assert.equal(correctness.hasDeferredClaims, stride > 1 && count > 1)
|
||||
if (arm === 'current') {
|
||||
assert.equal(committedKeys, expected)
|
||||
}
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
arm,
|
||||
stride,
|
||||
duringSampleCount,
|
||||
finalOnlyMeasurement: duringSampleCount === 0,
|
||||
baseline,
|
||||
samples,
|
||||
sampledRetainedPeakDeltaBytes:
|
||||
Math.max(...samples.map((sample) => sample.heapUsed)) - baseline.heapUsed,
|
||||
finalRetainedDeltaBytes: final.heapUsed - baseline.heapUsed,
|
||||
correctness
|
||||
})}\n`
|
||||
)
|
||||
delete globalThis.__orcaMemoryResult
|
||||
}
|
||||
|
||||
function writeFixture(filePath, eventCount) {
|
||||
const cwd = join('benchmark', 'workspace')
|
||||
writeFileSync(
|
||||
filePath,
|
||||
`${JSON.stringify({ type: 'session_meta', payload: { id: 'session', cwd } })}\n`
|
||||
)
|
||||
let batch = ''
|
||||
for (let index = 0; index < eventCount; index++) {
|
||||
batch += `${JSON.stringify({
|
||||
timestamp: new Date(1791547200000 + index * 1000).toISOString(),
|
||||
type: 'event_msg',
|
||||
payload: {
|
||||
type: 'token_count',
|
||||
info: {
|
||||
model: 'gpt-5-codex',
|
||||
last_token_usage: { input_tokens: 100, output_tokens: 10, total_tokens: 110 }
|
||||
}
|
||||
}
|
||||
})}\n`
|
||||
if ((index + 1) % 5000 === 0) {
|
||||
appendFileSync(filePath, batch)
|
||||
batch = ''
|
||||
}
|
||||
}
|
||||
if (batch) {
|
||||
appendFileSync(filePath, batch)
|
||||
}
|
||||
}
|
||||
|
||||
async function runComparison() {
|
||||
const { build, version: esbuildVersion } = await import('esbuild')
|
||||
const { runProcessSync, describeProcessFailure } = await import('./script-child-process.mjs')
|
||||
const { buildCounterbalancedSchedule } = await import('./counterbalanced-benchmark-schedule.mjs')
|
||||
const eventCount = positiveInteger(argument('--events', '100000'), 'events')
|
||||
const pairCount = positiveInteger(argument('--pairs', '2'), 'pairs')
|
||||
const intervalMs = positiveInteger(argument('--sample-ms', '35'), 'sample-ms')
|
||||
const schedule = buildCounterbalancedSchedule(pairCount, 'baseline', 'current')
|
||||
const execute = (program, commandArgs) => {
|
||||
const result = runProcessSync({
|
||||
program,
|
||||
args: commandArgs,
|
||||
cwd: root,
|
||||
env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' },
|
||||
timeoutMs: 120000,
|
||||
maxOutputBytes: 16 * 1024 * 1024
|
||||
})
|
||||
if (result.code !== 0 || result.timedOut || result.outputTruncated) {
|
||||
throw new Error(describeProcessFailure(result))
|
||||
}
|
||||
return result.stdout
|
||||
}
|
||||
const baselineCommit = execute('git', [
|
||||
'rev-parse',
|
||||
'--verify',
|
||||
`${argument('--baseline-ref', 'HEAD')}^{commit}`
|
||||
]).trim()
|
||||
const temporary = mkdtempSync(join(tmpdir(), 'orca-codex-parser-memory-'))
|
||||
try {
|
||||
const sources = {}
|
||||
const bundles = {}
|
||||
for (const arm of ['baseline', 'current']) {
|
||||
const sourceHashes = new Map()
|
||||
const bundlePath = join(temporary, `${arm}.mjs`)
|
||||
await build({
|
||||
entryPoints: [join(root, 'src/main/codex-usage/codex-rollout-file-parse.ts')],
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
target: 'node20',
|
||||
format: 'esm',
|
||||
outfile: bundlePath,
|
||||
logLevel: 'silent',
|
||||
plugins: [
|
||||
{
|
||||
name: 'record-parser-source-graph',
|
||||
setup(builder) {
|
||||
builder.onLoad({ filter: /\.[cm]?[jt]sx?$/ }, ({ path }) => {
|
||||
const sourcePath = relative(root, path).split(sep).join('/')
|
||||
if (!sourcePath.startsWith('src/')) {
|
||||
return undefined
|
||||
}
|
||||
const contents =
|
||||
arm === 'baseline'
|
||||
? execute('git', ['show', `${baselineCommit}:${sourcePath}`])
|
||||
: readFileSync(path, 'utf8')
|
||||
sourceHashes.set(sourcePath, digest(contents))
|
||||
return {
|
||||
contents,
|
||||
resolveDir: dirname(path),
|
||||
loader: sourcePath.endsWith('.tsx') ? 'tsx' : 'ts'
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
]
|
||||
})
|
||||
bundles[arm] = bundlePath
|
||||
sources[arm] = {
|
||||
origin: arm === 'baseline' ? baselineCommit : 'worktree at bundle build',
|
||||
files: Object.fromEntries(
|
||||
[...sourceHashes].sort(([left], [right]) => left.localeCompare(right))
|
||||
),
|
||||
bundleSha256: digest(readFileSync(bundlePath))
|
||||
}
|
||||
}
|
||||
const fixturePath = join(temporary, 'rollout.jsonl')
|
||||
writeFixture(fixturePath, eventCount)
|
||||
const fixtureBytes = readFileSync(fixturePath)
|
||||
const measurements = []
|
||||
for (let round = 0; round < schedule.length; round++) {
|
||||
for (const stride of [1, 10]) {
|
||||
const paired = []
|
||||
for (const arm of schedule[round]) {
|
||||
const measurement = JSON.parse(
|
||||
execute(process.execPath, [
|
||||
'--expose-gc',
|
||||
import.meta.filename,
|
||||
'--worker',
|
||||
'--bundle',
|
||||
bundles[arm],
|
||||
'--fixture',
|
||||
fixturePath,
|
||||
'--events',
|
||||
String(eventCount),
|
||||
'--stride',
|
||||
String(stride),
|
||||
'--sample-ms',
|
||||
String(intervalMs),
|
||||
'--arm',
|
||||
arm
|
||||
])
|
||||
)
|
||||
paired.push(measurement)
|
||||
measurements.push({ round: round + 1, order: schedule[round], ...measurement })
|
||||
}
|
||||
assert.equal(paired[0].correctness.projectionSha256, paired[1].correctness.projectionSha256)
|
||||
}
|
||||
}
|
||||
const artifact = {
|
||||
schemaVersion: 1,
|
||||
recordedAt: new Date().toISOString(),
|
||||
command: [
|
||||
'node',
|
||||
relative(root, import.meta.filename)
|
||||
.split(sep)
|
||||
.join('/'),
|
||||
...args
|
||||
].join(' '),
|
||||
environment: {
|
||||
node: process.version,
|
||||
v8: process.versions.v8,
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
osRelease: release(),
|
||||
cpuModel: cpus()[0]?.model ?? null,
|
||||
availableParallelism: availableParallelism(),
|
||||
totalMemoryBytes: totalmem(),
|
||||
esbuild: esbuildVersion,
|
||||
timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone
|
||||
},
|
||||
instrumentation: {
|
||||
metric: 'sampled retained heapUsed after two forced full GCs; not absolute peak memory',
|
||||
sampleIntervalMs: intervalMs,
|
||||
isolation: 'one fresh Node process for each arm, scenario and round',
|
||||
finalResult: 'held by a global reference through the final sample',
|
||||
observerMemory: 'retained snapshot records contribute small duration-dependent overhead',
|
||||
baseline: 'all bundled project source modules read from the pinned baseline commit',
|
||||
current: 'all bundled project source modules read and hashed once before any worker runs',
|
||||
timing: 'elapsed values include forced GC and are not a speed benchmark'
|
||||
},
|
||||
instrumentationSha256: Object.fromEntries(
|
||||
[
|
||||
import.meta.filename,
|
||||
join(import.meta.dirname, 'script-child-process.mjs'),
|
||||
join(import.meta.dirname, 'counterbalanced-benchmark-schedule.mjs')
|
||||
].map((path) => [relative(root, path).split(sep).join('/'), digest(readFileSync(path))])
|
||||
),
|
||||
baselineCommit,
|
||||
sources,
|
||||
fixture: { events: eventCount, bytes: fixtureBytes.length, sha256: digest(fixtureBytes) },
|
||||
schedule,
|
||||
measurements
|
||||
}
|
||||
const output = `${JSON.stringify(artifact, null, 2)}\n`
|
||||
const outputPath = argument('--output')
|
||||
if (outputPath) {
|
||||
writeFileSync(resolve(root, outputPath), output)
|
||||
} else {
|
||||
process.stdout.write(output)
|
||||
}
|
||||
} finally {
|
||||
rmSync(temporary, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
await (args.includes('--worker') ? runWorker() : runComparison())
|
||||
@@ -7,6 +7,10 @@ import { nodeRuntimePool } from './scripts/vitest-node-runtime-pool'
|
||||
const contracts = [
|
||||
'src/main/sqlite/sync-database.test.ts',
|
||||
'src/main/runtime/orchestration/db/row-column-lists.test.ts',
|
||||
'src/main/codex-usage/codex-rollout-file-lifetime.test.ts',
|
||||
'src/main/codex-usage/scanner-incremental-append.test.ts',
|
||||
'src/main/usage/jsonl-file-reader.test.ts',
|
||||
'src/main/usage/jsonl-file-snapshot.test.ts',
|
||||
'src/relay/fs-path-metadata-symlink-concurrency.test.ts',
|
||||
'src/renderer/src/components/editor/rich-markdown-list-tokenizers.test.ts',
|
||||
'src/renderer/src/components/editor/rich-markdown-lowlight-cache.test.ts',
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"status": "historical exploratory parser-retention comparison, not the final pinned-source benchmark",
|
||||
"baselineCommit": "51e7181850b022bae2d41091fe1229accef4bd50",
|
||||
"sourceProvenance": {
|
||||
"baseline": "Only codex-rollout-file-parse.ts was loaded with git show from this commit; other bundled project dependencies came from the worktree.",
|
||||
"current": "Actual worktree parser and project dependencies at the time of the exploratory run.",
|
||||
"sourceHashes": null,
|
||||
"limitation": "Source hashes and individual heap snapshots were not saved during this run. Later hashes cannot establish the earlier source. Use the reproducible full-graph comparison for final evidence."
|
||||
},
|
||||
"environment": {
|
||||
"node": "v24.20.0",
|
||||
"v8": "13.6.233.17-node.53",
|
||||
"platform": "darwin",
|
||||
"architecture": "arm64",
|
||||
"timeZone": "America/Los_Angeles"
|
||||
},
|
||||
"instrumentation": {
|
||||
"fixture": "100,000 event_msg/token_count rows with distinct one-second timestamps, model gpt-5-codex, last usage input 100/output 10/total 110, and one session_meta row; written in batches of 5,000.",
|
||||
"clockOrigin": 1791547200000,
|
||||
"metric": "heapUsed after two forced full GCs before parse, every 35 ms during parse, and after parse",
|
||||
"isolation": "Each measurement had its own async function scope; result held on globalThis through the final sample, removed at function end, with setImmediate before the next baseline sample.",
|
||||
"ownership": "baseline claimEventKey and current canClaimEventKey accept queryIndex++ % stride === 0; current commitEventKey is a no-op",
|
||||
"interpretation": "Sampled retained memory, not absolute process peak. Timings are not a speed benchmark."
|
||||
},
|
||||
"measurements": [
|
||||
{
|
||||
"arm": "baseline",
|
||||
"stride": 1,
|
||||
"samples": 9,
|
||||
"claimedKeys": 100000,
|
||||
"countedEvents": 100000,
|
||||
"sampledRetainedPeakDeltaMiB": 45.45518493652344,
|
||||
"finalRetainedDeltaMiB": 6.44708251953125
|
||||
},
|
||||
{
|
||||
"arm": "current",
|
||||
"stride": 1,
|
||||
"samples": 10,
|
||||
"claimedKeys": 100000,
|
||||
"countedEvents": 100000,
|
||||
"sampledRetainedPeakDeltaMiB": 7.5893402099609375,
|
||||
"finalRetainedDeltaMiB": 6.328620910644531
|
||||
},
|
||||
{
|
||||
"arm": "baseline",
|
||||
"stride": 10,
|
||||
"samples": 4,
|
||||
"claimedKeys": 10000,
|
||||
"countedEvents": 10000,
|
||||
"sampledRetainedPeakDeltaMiB": 3.7055206298828125,
|
||||
"finalRetainedDeltaMiB": 0.6778564453125
|
||||
},
|
||||
{
|
||||
"arm": "current",
|
||||
"stride": 10,
|
||||
"samples": 5,
|
||||
"claimedKeys": 10000,
|
||||
"countedEvents": 10000,
|
||||
"sampledRetainedPeakDeltaMiB": 0.91009521484375,
|
||||
"finalRetainedDeltaMiB": 0.66552734375
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,978 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"recordedAt": "2026-10-10T00:27:00.815Z",
|
||||
"command": "node config/scripts/codex-usage-parser-memory-benchmark.mjs --baseline-ref 51e7181850b022bae2d41091fe1229accef4bd50 --events 100000 --pairs 2 --sample-ms 35 --output docs/bug-reproductions/usage-scan-append/codex-parser-memory-final.json",
|
||||
"environment": {
|
||||
"node": "v24.20.0",
|
||||
"v8": "13.6.233.17-node.53",
|
||||
"platform": "darwin",
|
||||
"architecture": "arm64",
|
||||
"osRelease": "25.3.0",
|
||||
"cpuModel": "Apple M4 Max",
|
||||
"availableParallelism": 16,
|
||||
"totalMemoryBytes": 137438953472,
|
||||
"esbuild": "0.28.2",
|
||||
"timeZone": "America/Los_Angeles"
|
||||
},
|
||||
"instrumentation": {
|
||||
"metric": "sampled retained heapUsed after two forced full GCs; not absolute peak memory",
|
||||
"sampleIntervalMs": 35,
|
||||
"isolation": "one fresh Node process for each arm, scenario and round",
|
||||
"finalResult": "held by a global reference through the final sample",
|
||||
"observerMemory": "retained snapshot records contribute small duration-dependent overhead",
|
||||
"baseline": "all bundled project source modules read from the pinned baseline commit",
|
||||
"current": "all bundled project source modules read and hashed once before any worker runs",
|
||||
"timing": "elapsed values include forced GC and are not a speed benchmark"
|
||||
},
|
||||
"instrumentationSha256": {
|
||||
"config/scripts/codex-usage-parser-memory-benchmark.mjs": "434e9818f8d8b132603912b104560a000e9e2635a2564cf65ae7e87cff483a1a",
|
||||
"config/scripts/script-child-process.mjs": "a7441428bf7b5c9a897b6a3c057c50f46acca24c4f218a4b849de5aec53c4173",
|
||||
"config/scripts/counterbalanced-benchmark-schedule.mjs": "bc35b40e954fa7ddec0225ed375bd3b113c2e05b8554bc50734d14f89987294d"
|
||||
},
|
||||
"baselineCommit": "51e7181850b022bae2d41091fe1229accef4bd50",
|
||||
"sources": {
|
||||
"baseline": {
|
||||
"origin": "51e7181850b022bae2d41091fe1229accef4bd50",
|
||||
"files": {
|
||||
"src/main/codex-usage/codex-model-pricing.ts": "05416b11916778dc2c69edfccfb277b0a2fe6ef7e88e0d8c9bdf030f2dc7ea1d",
|
||||
"src/main/codex-usage/codex-rollout-file-parse.ts": "9ef58c665491ea709166d9580913bb1791a1748435cb2af1a5bb41dfe47df792",
|
||||
"src/main/codex-usage/codex-rollout-resume-state.ts": "de254c8fbca043c311350adf5466d5e510c4894b9c0d9e1a086a204a8c949f6f",
|
||||
"src/main/codex-usage/codex-usage-aggregation.ts": "82774021e25bfcf413dbc2faad8d0cb92310a3ddeac87ba640abe4254f19d082",
|
||||
"src/main/codex-usage/codex-usage-event-attribution.ts": "1b3c37111bb5ff9a28fb9a97d3e40d0c76026b6b5be2d55a0c512b68d5944b02",
|
||||
"src/main/codex-usage/codex-usage-record-parser.ts": "b73ecd62b6ed20ce9cd7e21ca8e78e8a3f7d9a635311f2d8d56f9d64042a1661",
|
||||
"src/main/codex-usage/codex-usage-token-delta.ts": "fb3e64b17eef3d8e23fafbcb2ce4a1ebfe44cab1091faa979d8c0269f294d300",
|
||||
"src/main/usage/jsonl-line-offsets.ts": "ea7f58e248e8c332bfe6d18294261b1941816eb38f079e9a2c0c9b15c06820c5",
|
||||
"src/main/usage/usage-calendar-range.ts": "fe4c4eb09aaefcae6d2698e113e134663826c3ed9ca3ef960f3047a95686b1dd",
|
||||
"src/main/usage/usage-event-aggregation.ts": "68caf65824d32c55b706fd78ee90d583439f4b86445862cd75d10f3f4ef122d1",
|
||||
"src/main/usage/usage-event-attribution.ts": "e7a909437d836c34939e2709ede99794f81cc8746c6c55b1bbdd36bf352e7664",
|
||||
"src/main/usage/usage-path-comparison.ts": "07be51caa36e43962fac4cb66c1e0bfd4700f58aeb76eae32cc524fff88625d8",
|
||||
"src/main/usage/usage-record-coercion.ts": "1b7f9fd8a0c01fbe25f82171d9ceefcbc717e87df3e321fc9514c59c9cfbd7da",
|
||||
"src/main/usage/usage-rollup-merge.ts": "67cc062058439d659f77a8b47ce1b0ad3e2c323a83d2332aaa75e399de98b433",
|
||||
"src/main/usage/usage-rollup-records.ts": "eff570d174439c5786175081ffe3ac2aedd960ebfc7e317f9f3cfc4091f27dab",
|
||||
"src/main/usage/usage-session-breakdown-index.ts": "51725b3480ed078506beeec3105bd95402836e10478bd681cdff95fed080f81d"
|
||||
},
|
||||
"bundleSha256": "aaf0b5266d4511752229a998db9789c583ca3cd2c0b7bcc4afe03c465ef6e564"
|
||||
},
|
||||
"current": {
|
||||
"origin": "worktree at bundle build",
|
||||
"files": {
|
||||
"src/main/codex-usage/codex-model-pricing.ts": "05416b11916778dc2c69edfccfb277b0a2fe6ef7e88e0d8c9bdf030f2dc7ea1d",
|
||||
"src/main/codex-usage/codex-rollout-file-parse.ts": "f39a10c1dce47a71d46c0948d05df30aa6dcc044c81b777b6a013be6f23e9c7a",
|
||||
"src/main/codex-usage/codex-rollout-resume-state.ts": "aa028d2155b7744c0d961c5649537faa20c39bfe404dea8d9c4c1464d11a0e39",
|
||||
"src/main/codex-usage/codex-usage-aggregation.ts": "82774021e25bfcf413dbc2faad8d0cb92310a3ddeac87ba640abe4254f19d082",
|
||||
"src/main/codex-usage/codex-usage-event-attribution.ts": "1b3c37111bb5ff9a28fb9a97d3e40d0c76026b6b5be2d55a0c512b68d5944b02",
|
||||
"src/main/codex-usage/codex-usage-record-parser.ts": "b73ecd62b6ed20ce9cd7e21ca8e78e8a3f7d9a635311f2d8d56f9d64042a1661",
|
||||
"src/main/codex-usage/codex-usage-token-delta.ts": "fb3e64b17eef3d8e23fafbcb2ce4a1ebfe44cab1091faa979d8c0269f294d300",
|
||||
"src/main/usage/jsonl-file-checkpoint.ts": "b7f307564989dabaf1dfda095a26c7bd9f0683e0efcabe79a010c7b58fa1a2d4",
|
||||
"src/main/usage/jsonl-file-snapshot.ts": "b1b8869db92c08c762160acba7570aabff25879649cb6ffa3826912ee35deffe",
|
||||
"src/main/usage/jsonl-line-offsets.ts": "f3574adfad4549c6330e2ac52760c91be5a8cbb9bf8e1beb440e3226087074db",
|
||||
"src/main/usage/usage-calendar-range.ts": "fe4c4eb09aaefcae6d2698e113e134663826c3ed9ca3ef960f3047a95686b1dd",
|
||||
"src/main/usage/usage-event-aggregation.ts": "31adda2e8e0a32756416fb49b2c3e02c707703bfe76434aabc95c61a7efd180b",
|
||||
"src/main/usage/usage-event-attribution.ts": "e7a909437d836c34939e2709ede99794f81cc8746c6c55b1bbdd36bf352e7664",
|
||||
"src/main/usage/usage-path-comparison.ts": "07be51caa36e43962fac4cb66c1e0bfd4700f58aeb76eae32cc524fff88625d8",
|
||||
"src/main/usage/usage-record-coercion.ts": "1b7f9fd8a0c01fbe25f82171d9ceefcbc717e87df3e321fc9514c59c9cfbd7da",
|
||||
"src/main/usage/usage-rollup-merge.ts": "67cc062058439d659f77a8b47ce1b0ad3e2c323a83d2332aaa75e399de98b433",
|
||||
"src/main/usage/usage-rollup-records.ts": "eff570d174439c5786175081ffe3ac2aedd960ebfc7e317f9f3cfc4091f27dab",
|
||||
"src/main/usage/usage-session-breakdown-index.ts": "51725b3480ed078506beeec3105bd95402836e10478bd681cdff95fed080f81d"
|
||||
},
|
||||
"bundleSha256": "9f8b81ddae88e3d4fdd48175a53f68011d9606bfe169d6f5660238b3db74abaf"
|
||||
}
|
||||
},
|
||||
"fixture": {
|
||||
"events": 100000,
|
||||
"bytes": 20200079,
|
||||
"sha256": "fa36fd10e36653020c0997890c0c9d9e75f40ee4040adf3a8d79735c4fbe20d5"
|
||||
},
|
||||
"schedule": [
|
||||
["baseline", "current"],
|
||||
["current", "baseline"]
|
||||
],
|
||||
"measurements": [
|
||||
{
|
||||
"round": 1,
|
||||
"order": ["baseline", "current"],
|
||||
"arm": "baseline",
|
||||
"stride": 1,
|
||||
"duringSampleCount": 9,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 2.961416,
|
||||
"rss": 54771712,
|
||||
"heapTotal": 7667712,
|
||||
"heapUsed": 4923944,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 43.085458,
|
||||
"rss": 79740928,
|
||||
"heapTotal": 28196864,
|
||||
"heapUsed": 10333960,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 80.171208,
|
||||
"rss": 100352000,
|
||||
"heapTotal": 50823168,
|
||||
"heapUsed": 16436440,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 116.55525,
|
||||
"rss": 109264896,
|
||||
"heapTotal": 56590336,
|
||||
"heapUsed": 22257384,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 153.371208,
|
||||
"rss": 129040384,
|
||||
"heapTotal": 95125504,
|
||||
"heapUsed": 27179344,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 191.0125,
|
||||
"rss": 146718720,
|
||||
"heapTotal": 100581376,
|
||||
"heapUsed": 32420792,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 223.885208,
|
||||
"rss": 154877952,
|
||||
"heapTotal": 105824256,
|
||||
"heapUsed": 37958976,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 261.673083,
|
||||
"rss": 166674432,
|
||||
"heapTotal": 178733056,
|
||||
"heapUsed": 43593472,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 302.397916,
|
||||
"rss": 184074240,
|
||||
"heapTotal": 183451648,
|
||||
"heapUsed": 48113072,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 337.490916,
|
||||
"rss": 187842560,
|
||||
"heapTotal": 186597376,
|
||||
"heapUsed": 51628032,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 404.128916,
|
||||
"rss": 197738496,
|
||||
"heapTotal": 147144704,
|
||||
"heapUsed": 11730912,
|
||||
"external": 2455679,
|
||||
"arrayBuffers": 143819
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 46704088,
|
||||
"finalRetainedDeltaBytes": 6806968,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": null,
|
||||
"ownedKeys": 100000,
|
||||
"countedEvents": 100000,
|
||||
"inputTokens": 10000000,
|
||||
"outputTokens": 1000000,
|
||||
"totalTokens": 11000000,
|
||||
"hasDeferredClaims": false,
|
||||
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"round": 1,
|
||||
"order": ["baseline", "current"],
|
||||
"arm": "current",
|
||||
"stride": 1,
|
||||
"duringSampleCount": 9,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 2.704749999999997,
|
||||
"rss": 54919168,
|
||||
"heapTotal": 7929856,
|
||||
"heapUsed": 4938600,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 40.513166999999996,
|
||||
"rss": 68812800,
|
||||
"heapTotal": 11419648,
|
||||
"heapUsed": 6306616,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 75.017375,
|
||||
"rss": 75153408,
|
||||
"heapTotal": 16465920,
|
||||
"heapUsed": 7232808,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 109.88325,
|
||||
"rss": 75939840,
|
||||
"heapTotal": 16990208,
|
||||
"heapUsed": 7780688,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 145.465458,
|
||||
"rss": 77348864,
|
||||
"heapTotal": 18432000,
|
||||
"heapUsed": 9102536,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 180.921875,
|
||||
"rss": 78675968,
|
||||
"heapTotal": 27344896,
|
||||
"heapUsed": 9704632,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 215.932708,
|
||||
"rss": 85442560,
|
||||
"heapTotal": 27869184,
|
||||
"heapUsed": 10366048,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 250.763458,
|
||||
"rss": 88440832,
|
||||
"heapTotal": 29966336,
|
||||
"heapUsed": 12341424,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 286.477917,
|
||||
"rss": 89866240,
|
||||
"heapTotal": 30752768,
|
||||
"heapUsed": 12972312,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 321.256625,
|
||||
"rss": 91324416,
|
||||
"heapTotal": 31277056,
|
||||
"heapUsed": 13643896,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 336.48325,
|
||||
"rss": 92192768,
|
||||
"heapTotal": 29704192,
|
||||
"heapUsed": 11803368,
|
||||
"external": 2455471,
|
||||
"arrayBuffers": 143611
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 8705296,
|
||||
"finalRetainedDeltaBytes": 6864768,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": 100000,
|
||||
"ownedKeys": 100000,
|
||||
"countedEvents": 100000,
|
||||
"inputTokens": 10000000,
|
||||
"outputTokens": 1000000,
|
||||
"totalTokens": 11000000,
|
||||
"hasDeferredClaims": false,
|
||||
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"round": 1,
|
||||
"order": ["baseline", "current"],
|
||||
"arm": "baseline",
|
||||
"stride": 10,
|
||||
"duringSampleCount": 4,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 2.7109999999999985,
|
||||
"rss": 54607872,
|
||||
"heapTotal": 7929856,
|
||||
"heapUsed": 4924000,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 41.222042,
|
||||
"rss": 66764800,
|
||||
"heapTotal": 11337728,
|
||||
"heapUsed": 6303312,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 75.894208,
|
||||
"rss": 71761920,
|
||||
"heapTotal": 16498688,
|
||||
"heapUsed": 7411304,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 111.375667,
|
||||
"rss": 75366400,
|
||||
"heapTotal": 25935872,
|
||||
"heapUsed": 8445088,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 146.555292,
|
||||
"rss": 83542016,
|
||||
"heapTotal": 27410432,
|
||||
"heapUsed": 9730008,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 172.027042,
|
||||
"rss": 86786048,
|
||||
"heapTotal": 24182784,
|
||||
"heapUsed": 5946064,
|
||||
"external": 2455679,
|
||||
"arrayBuffers": 143819
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 4806008,
|
||||
"finalRetainedDeltaBytes": 1022064,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": null,
|
||||
"ownedKeys": 10000,
|
||||
"countedEvents": 10000,
|
||||
"inputTokens": 1000000,
|
||||
"outputTokens": 100000,
|
||||
"totalTokens": 1100000,
|
||||
"hasDeferredClaims": true,
|
||||
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
|
||||
}
|
||||
},
|
||||
{
|
||||
"round": 1,
|
||||
"order": ["baseline", "current"],
|
||||
"arm": "current",
|
||||
"stride": 10,
|
||||
"duringSampleCount": 4,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 3.0069579999999974,
|
||||
"rss": 54706176,
|
||||
"heapTotal": 7667712,
|
||||
"heapUsed": 4938616,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 40.785916,
|
||||
"rss": 65945600,
|
||||
"heapTotal": 10551296,
|
||||
"heapUsed": 5556160,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 75.849791,
|
||||
"rss": 66551808,
|
||||
"heapTotal": 10731520,
|
||||
"heapUsed": 5764056,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 110.944416,
|
||||
"rss": 66797568,
|
||||
"heapTotal": 10993664,
|
||||
"heapUsed": 5903120,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 145.28670799999998,
|
||||
"rss": 68075520,
|
||||
"heapTotal": 11419648,
|
||||
"heapUsed": 6229832,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 172.400125,
|
||||
"rss": 70189056,
|
||||
"heapTotal": 11075584,
|
||||
"heapUsed": 6002200,
|
||||
"external": 2455471,
|
||||
"arrayBuffers": 143611
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 1291216,
|
||||
"finalRetainedDeltaBytes": 1063584,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": 10000,
|
||||
"ownedKeys": 10000,
|
||||
"countedEvents": 10000,
|
||||
"inputTokens": 1000000,
|
||||
"outputTokens": 100000,
|
||||
"totalTokens": 1100000,
|
||||
"hasDeferredClaims": true,
|
||||
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
|
||||
}
|
||||
},
|
||||
{
|
||||
"round": 2,
|
||||
"order": ["current", "baseline"],
|
||||
"arm": "current",
|
||||
"stride": 1,
|
||||
"duringSampleCount": 10,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 2.3475420000000007,
|
||||
"rss": 54722560,
|
||||
"heapTotal": 7929856,
|
||||
"heapUsed": 4938624,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 40.420334,
|
||||
"rss": 68485120,
|
||||
"heapTotal": 11419648,
|
||||
"heapUsed": 6288496,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 76.145292,
|
||||
"rss": 74825728,
|
||||
"heapTotal": 16465920,
|
||||
"heapUsed": 7269912,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 110.49249999999999,
|
||||
"rss": 75481088,
|
||||
"heapTotal": 16990208,
|
||||
"heapUsed": 7924696,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 145.027584,
|
||||
"rss": 76890112,
|
||||
"heapTotal": 18432000,
|
||||
"heapUsed": 9178464,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 180.48204199999998,
|
||||
"rss": 78184448,
|
||||
"heapTotal": 27344896,
|
||||
"heapUsed": 9725784,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 217.70899999999997,
|
||||
"rss": 86163456,
|
||||
"heapTotal": 27869184,
|
||||
"heapUsed": 10424152,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 250.85633399999998,
|
||||
"rss": 89620480,
|
||||
"heapTotal": 29966336,
|
||||
"heapUsed": 12375976,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 286.69275,
|
||||
"rss": 91013120,
|
||||
"heapTotal": 30490624,
|
||||
"heapUsed": 12988984,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 354.68100000000004,
|
||||
"rss": 91783168,
|
||||
"heapTotal": 31014912,
|
||||
"heapUsed": 13393048,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 366.923584,
|
||||
"rss": 91799552,
|
||||
"heapTotal": 31014912,
|
||||
"heapUsed": 13405528,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 400.572084,
|
||||
"rss": 93257728,
|
||||
"heapTotal": 29442048,
|
||||
"heapUsed": 11784552,
|
||||
"external": 2455471,
|
||||
"arrayBuffers": 143611
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 8466904,
|
||||
"finalRetainedDeltaBytes": 6845928,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": 100000,
|
||||
"ownedKeys": 100000,
|
||||
"countedEvents": 100000,
|
||||
"inputTokens": 10000000,
|
||||
"outputTokens": 1000000,
|
||||
"totalTokens": 11000000,
|
||||
"hasDeferredClaims": false,
|
||||
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"round": 2,
|
||||
"order": ["current", "baseline"],
|
||||
"arm": "baseline",
|
||||
"stride": 1,
|
||||
"duringSampleCount": 17,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 3.085000000000008,
|
||||
"rss": 54870016,
|
||||
"heapTotal": 7929856,
|
||||
"heapUsed": 4923944,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 66.69608300000002,
|
||||
"rss": 70172672,
|
||||
"heapTotal": 16498688,
|
||||
"heapUsed": 7281848,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 85.53883300000001,
|
||||
"rss": 73515008,
|
||||
"heapTotal": 17809408,
|
||||
"heapUsed": 8570768,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 152.638541,
|
||||
"rss": 85196800,
|
||||
"heapTotal": 30031872,
|
||||
"heapUsed": 12156408,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 159.173375,
|
||||
"rss": 85639168,
|
||||
"heapTotal": 30228480,
|
||||
"heapUsed": 12357872,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 225.200083,
|
||||
"rss": 104693760,
|
||||
"heapTotal": 52133888,
|
||||
"heapUsed": 17698272,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 297.829375,
|
||||
"rss": 104857600,
|
||||
"heapTotal": 52396032,
|
||||
"heapUsed": 17837984,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 310.679916,
|
||||
"rss": 104988672,
|
||||
"heapTotal": 52396032,
|
||||
"heapUsed": 17986776,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 343.32525,
|
||||
"rss": 106856448,
|
||||
"heapTotal": 54493184,
|
||||
"heapUsed": 19932448,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 408.458833,
|
||||
"rss": 111263744,
|
||||
"heapTotal": 57901056,
|
||||
"heapUsed": 23407968,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 420.895625,
|
||||
"rss": 111493120,
|
||||
"heapTotal": 58163200,
|
||||
"heapUsed": 23541184,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 474.20687499999997,
|
||||
"rss": 120029184,
|
||||
"heapTotal": 94601216,
|
||||
"heapUsed": 26919320,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 494.13695800000005,
|
||||
"rss": 121274368,
|
||||
"heapTotal": 96174080,
|
||||
"heapUsed": 28168392,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 523.933083,
|
||||
"rss": 135741440,
|
||||
"heapTotal": 99794944,
|
||||
"heapUsed": 32038824,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 567.489416,
|
||||
"rss": 156827648,
|
||||
"heapTotal": 106348544,
|
||||
"heapUsed": 38424024,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 598.908,
|
||||
"rss": 160595968,
|
||||
"heapTotal": 110051328,
|
||||
"heapUsed": 42221320,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 634.888333,
|
||||
"rss": 170278912,
|
||||
"heapTotal": 181878784,
|
||||
"heapUsed": 46720192,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 670.023,
|
||||
"rss": 185548800,
|
||||
"heapTotal": 185810944,
|
||||
"heapUsed": 50929816,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 735.424125,
|
||||
"rss": 202817536,
|
||||
"heapTotal": 147144704,
|
||||
"heapUsed": 11727688,
|
||||
"external": 2455679,
|
||||
"arrayBuffers": 143819
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 46005872,
|
||||
"finalRetainedDeltaBytes": 6803744,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": null,
|
||||
"ownedKeys": 100000,
|
||||
"countedEvents": 100000,
|
||||
"inputTokens": 10000000,
|
||||
"outputTokens": 1000000,
|
||||
"totalTokens": 11000000,
|
||||
"hasDeferredClaims": false,
|
||||
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"round": 2,
|
||||
"order": ["current", "baseline"],
|
||||
"arm": "current",
|
||||
"stride": 10,
|
||||
"duringSampleCount": 5,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 2.7139999999999986,
|
||||
"rss": 54902784,
|
||||
"heapTotal": 7929856,
|
||||
"heapUsed": 4938792,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 39.88775,
|
||||
"rss": 66011136,
|
||||
"heapTotal": 10813440,
|
||||
"heapUsed": 5552696,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 75.154917,
|
||||
"rss": 66748416,
|
||||
"heapTotal": 10993664,
|
||||
"heapUsed": 5767816,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 110.337708,
|
||||
"rss": 67108864,
|
||||
"heapTotal": 11255808,
|
||||
"heapUsed": 5912424,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 147.088458,
|
||||
"rss": 68534272,
|
||||
"heapTotal": 11419648,
|
||||
"heapUsed": 6234112,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 179.94975,
|
||||
"rss": 68550656,
|
||||
"heapTotal": 11681792,
|
||||
"heapUsed": 6230488,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 205.10170799999997,
|
||||
"rss": 70860800,
|
||||
"heapTotal": 10813440,
|
||||
"heapUsed": 5992168,
|
||||
"external": 2455471,
|
||||
"arrayBuffers": 143611
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 1295320,
|
||||
"finalRetainedDeltaBytes": 1053376,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": 10000,
|
||||
"ownedKeys": 10000,
|
||||
"countedEvents": 10000,
|
||||
"inputTokens": 1000000,
|
||||
"outputTokens": 100000,
|
||||
"totalTokens": 1100000,
|
||||
"hasDeferredClaims": true,
|
||||
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
|
||||
}
|
||||
},
|
||||
{
|
||||
"round": 2,
|
||||
"order": ["current", "baseline"],
|
||||
"arm": "baseline",
|
||||
"stride": 10,
|
||||
"duringSampleCount": 4,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 3.1553749999999994,
|
||||
"rss": 54804480,
|
||||
"heapTotal": 7929856,
|
||||
"heapUsed": 4923976,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 41.432542,
|
||||
"rss": 66666496,
|
||||
"heapTotal": 11337728,
|
||||
"heapUsed": 6206048,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 75.186042,
|
||||
"rss": 71827456,
|
||||
"heapTotal": 16760832,
|
||||
"heapUsed": 7309048,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 111.10141699999998,
|
||||
"rss": 74629120,
|
||||
"heapTotal": 17809408,
|
||||
"heapUsed": 8403400,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 146.308042,
|
||||
"rss": 83525632,
|
||||
"heapTotal": 27148288,
|
||||
"heapUsed": 9674064,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 172.545917,
|
||||
"rss": 87998464,
|
||||
"heapTotal": 24182784,
|
||||
"heapUsed": 5945896,
|
||||
"external": 2455679,
|
||||
"arrayBuffers": 143819
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 4750088,
|
||||
"finalRetainedDeltaBytes": 1021920,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": null,
|
||||
"ownedKeys": 10000,
|
||||
"countedEvents": 10000,
|
||||
"inputTokens": 1000000,
|
||||
"outputTokens": 100000,
|
||||
"totalTokens": 1100000,
|
||||
"hasDeferredClaims": true,
|
||||
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,879 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"recordedAt": "2026-10-09T18:39:49.816Z",
|
||||
"command": "node config/scripts/codex-usage-parser-memory-benchmark.mjs --baseline-ref 51e7181850b022bae2d41091fe1229accef4bd50 --events 100000 --pairs 2 --sample-ms 35 --output docs/bug-reproductions/usage-scan-append/codex-parser-memory.json",
|
||||
"environment": {
|
||||
"node": "v24.20.0",
|
||||
"v8": "13.6.233.17-node.53",
|
||||
"platform": "darwin",
|
||||
"architecture": "arm64",
|
||||
"osRelease": "25.3.0",
|
||||
"cpuModel": "Apple M4 Max",
|
||||
"availableParallelism": 16,
|
||||
"totalMemoryBytes": 137438953472,
|
||||
"esbuild": "0.28.2",
|
||||
"timeZone": "America/Los_Angeles"
|
||||
},
|
||||
"instrumentation": {
|
||||
"metric": "sampled retained heapUsed after two forced full GCs; not absolute peak memory",
|
||||
"sampleIntervalMs": 35,
|
||||
"isolation": "one fresh Node process for each arm, scenario and round",
|
||||
"finalResult": "held by a global reference through the final sample",
|
||||
"observerMemory": "retained snapshot records contribute small duration-dependent overhead",
|
||||
"baseline": "all bundled project source modules read from the pinned baseline commit",
|
||||
"current": "all bundled project source modules read and hashed once before any worker runs",
|
||||
"timing": "elapsed values include forced GC and are not a speed benchmark"
|
||||
},
|
||||
"instrumentationSha256": {
|
||||
"config/scripts/codex-usage-parser-memory-benchmark.mjs": "434e9818f8d8b132603912b104560a000e9e2635a2564cf65ae7e87cff483a1a",
|
||||
"config/scripts/script-child-process.mjs": "a7441428bf7b5c9a897b6a3c057c50f46acca24c4f218a4b849de5aec53c4173",
|
||||
"config/scripts/counterbalanced-benchmark-schedule.mjs": "bc35b40e954fa7ddec0225ed375bd3b113c2e05b8554bc50734d14f89987294d"
|
||||
},
|
||||
"baselineCommit": "51e7181850b022bae2d41091fe1229accef4bd50",
|
||||
"sources": {
|
||||
"baseline": {
|
||||
"origin": "51e7181850b022bae2d41091fe1229accef4bd50",
|
||||
"files": {
|
||||
"src/main/codex-usage/codex-model-pricing.ts": "05416b11916778dc2c69edfccfb277b0a2fe6ef7e88e0d8c9bdf030f2dc7ea1d",
|
||||
"src/main/codex-usage/codex-rollout-file-parse.ts": "9ef58c665491ea709166d9580913bb1791a1748435cb2af1a5bb41dfe47df792",
|
||||
"src/main/codex-usage/codex-rollout-resume-state.ts": "de254c8fbca043c311350adf5466d5e510c4894b9c0d9e1a086a204a8c949f6f",
|
||||
"src/main/codex-usage/codex-usage-aggregation.ts": "82774021e25bfcf413dbc2faad8d0cb92310a3ddeac87ba640abe4254f19d082",
|
||||
"src/main/codex-usage/codex-usage-event-attribution.ts": "1b3c37111bb5ff9a28fb9a97d3e40d0c76026b6b5be2d55a0c512b68d5944b02",
|
||||
"src/main/codex-usage/codex-usage-record-parser.ts": "b73ecd62b6ed20ce9cd7e21ca8e78e8a3f7d9a635311f2d8d56f9d64042a1661",
|
||||
"src/main/codex-usage/codex-usage-token-delta.ts": "fb3e64b17eef3d8e23fafbcb2ce4a1ebfe44cab1091faa979d8c0269f294d300",
|
||||
"src/main/usage/jsonl-line-offsets.ts": "ea7f58e248e8c332bfe6d18294261b1941816eb38f079e9a2c0c9b15c06820c5",
|
||||
"src/main/usage/usage-calendar-range.ts": "fe4c4eb09aaefcae6d2698e113e134663826c3ed9ca3ef960f3047a95686b1dd",
|
||||
"src/main/usage/usage-event-aggregation.ts": "68caf65824d32c55b706fd78ee90d583439f4b86445862cd75d10f3f4ef122d1",
|
||||
"src/main/usage/usage-event-attribution.ts": "e7a909437d836c34939e2709ede99794f81cc8746c6c55b1bbdd36bf352e7664",
|
||||
"src/main/usage/usage-path-comparison.ts": "07be51caa36e43962fac4cb66c1e0bfd4700f58aeb76eae32cc524fff88625d8",
|
||||
"src/main/usage/usage-record-coercion.ts": "1b7f9fd8a0c01fbe25f82171d9ceefcbc717e87df3e321fc9514c59c9cfbd7da",
|
||||
"src/main/usage/usage-rollup-merge.ts": "67cc062058439d659f77a8b47ce1b0ad3e2c323a83d2332aaa75e399de98b433",
|
||||
"src/main/usage/usage-rollup-records.ts": "eff570d174439c5786175081ffe3ac2aedd960ebfc7e317f9f3cfc4091f27dab",
|
||||
"src/main/usage/usage-session-breakdown-index.ts": "51725b3480ed078506beeec3105bd95402836e10478bd681cdff95fed080f81d"
|
||||
},
|
||||
"bundleSha256": "aaf0b5266d4511752229a998db9789c583ca3cd2c0b7bcc4afe03c465ef6e564"
|
||||
},
|
||||
"current": {
|
||||
"origin": "worktree at bundle build",
|
||||
"files": {
|
||||
"src/main/codex-usage/codex-model-pricing.ts": "05416b11916778dc2c69edfccfb277b0a2fe6ef7e88e0d8c9bdf030f2dc7ea1d",
|
||||
"src/main/codex-usage/codex-rollout-file-parse.ts": "6e4937105fc9cf8457b79ac16bcfc71f706f22a87e036166bd461aeea8c07fdc",
|
||||
"src/main/codex-usage/codex-rollout-resume-state.ts": "aa028d2155b7744c0d961c5649537faa20c39bfe404dea8d9c4c1464d11a0e39",
|
||||
"src/main/codex-usage/codex-usage-aggregation.ts": "82774021e25bfcf413dbc2faad8d0cb92310a3ddeac87ba640abe4254f19d082",
|
||||
"src/main/codex-usage/codex-usage-event-attribution.ts": "1b3c37111bb5ff9a28fb9a97d3e40d0c76026b6b5be2d55a0c512b68d5944b02",
|
||||
"src/main/codex-usage/codex-usage-record-parser.ts": "b73ecd62b6ed20ce9cd7e21ca8e78e8a3f7d9a635311f2d8d56f9d64042a1661",
|
||||
"src/main/codex-usage/codex-usage-token-delta.ts": "fb3e64b17eef3d8e23fafbcb2ce4a1ebfe44cab1091faa979d8c0269f294d300",
|
||||
"src/main/usage/jsonl-file-checkpoint.ts": "8eb579f356e1bb23b36319f00b3d4603fff80a963222dcb8a22331319cab3a03",
|
||||
"src/main/usage/jsonl-file-snapshot.ts": "1bd98ac194f73097cd6c204238fe1db01e1e0d60a604e93c1d7841725971fe0d",
|
||||
"src/main/usage/jsonl-line-offsets.ts": "f119465fad3771acd72b128853e84eed9165f0f74b75743ad662aa5bab991273",
|
||||
"src/main/usage/usage-calendar-range.ts": "fe4c4eb09aaefcae6d2698e113e134663826c3ed9ca3ef960f3047a95686b1dd",
|
||||
"src/main/usage/usage-event-aggregation.ts": "31adda2e8e0a32756416fb49b2c3e02c707703bfe76434aabc95c61a7efd180b",
|
||||
"src/main/usage/usage-event-attribution.ts": "e7a909437d836c34939e2709ede99794f81cc8746c6c55b1bbdd36bf352e7664",
|
||||
"src/main/usage/usage-path-comparison.ts": "07be51caa36e43962fac4cb66c1e0bfd4700f58aeb76eae32cc524fff88625d8",
|
||||
"src/main/usage/usage-record-coercion.ts": "1b7f9fd8a0c01fbe25f82171d9ceefcbc717e87df3e321fc9514c59c9cfbd7da",
|
||||
"src/main/usage/usage-rollup-merge.ts": "67cc062058439d659f77a8b47ce1b0ad3e2c323a83d2332aaa75e399de98b433",
|
||||
"src/main/usage/usage-rollup-records.ts": "eff570d174439c5786175081ffe3ac2aedd960ebfc7e317f9f3cfc4091f27dab",
|
||||
"src/main/usage/usage-session-breakdown-index.ts": "51725b3480ed078506beeec3105bd95402836e10478bd681cdff95fed080f81d"
|
||||
},
|
||||
"bundleSha256": "9e56032701d379c5e1b5482c49c5734153fbc2735010d0cbc3bc82e44662fe25"
|
||||
}
|
||||
},
|
||||
"fixture": {
|
||||
"events": 100000,
|
||||
"bytes": 20200079,
|
||||
"sha256": "fa36fd10e36653020c0997890c0c9d9e75f40ee4040adf3a8d79735c4fbe20d5"
|
||||
},
|
||||
"schedule": [
|
||||
["baseline", "current"],
|
||||
["current", "baseline"]
|
||||
],
|
||||
"measurements": [
|
||||
{
|
||||
"round": 1,
|
||||
"order": ["baseline", "current"],
|
||||
"arm": "baseline",
|
||||
"stride": 1,
|
||||
"duringSampleCount": 12,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 2.196832999999998,
|
||||
"rss": 54837248,
|
||||
"heapTotal": 7929856,
|
||||
"heapUsed": 4923952,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 41.750583000000006,
|
||||
"rss": 85114880,
|
||||
"heapTotal": 29507584,
|
||||
"heapUsed": 11458104,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 77.58875,
|
||||
"rss": 100073472,
|
||||
"heapTotal": 52396032,
|
||||
"heapUsed": 17544544,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 115.21895800000001,
|
||||
"rss": 114868224,
|
||||
"heapTotal": 92504064,
|
||||
"heapUsed": 24514776,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 154.876542,
|
||||
"rss": 137347072,
|
||||
"heapTotal": 98222080,
|
||||
"heapUsed": 30184872,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 186.526458,
|
||||
"rss": 152600576,
|
||||
"heapTotal": 102416384,
|
||||
"heapUsed": 34527664,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 223.591625,
|
||||
"rss": 159465472,
|
||||
"heapTotal": 109232128,
|
||||
"heapUsed": 41191320,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 275.32012499999996,
|
||||
"rss": 171245568,
|
||||
"heapTotal": 181878784,
|
||||
"heapUsed": 46842960,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 329.858125,
|
||||
"rss": 171409408,
|
||||
"heapTotal": 182140928,
|
||||
"heapUsed": 46988472,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 372.46854199999996,
|
||||
"rss": 171507712,
|
||||
"heapTotal": 182403072,
|
||||
"heapUsed": 47139504,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 395.50424999999996,
|
||||
"rss": 171720704,
|
||||
"heapTotal": 182403072,
|
||||
"heapUsed": 47282504,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 430.291167,
|
||||
"rss": 174358528,
|
||||
"heapTotal": 185024512,
|
||||
"heapUsed": 49947720,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 460.099042,
|
||||
"rss": 176521216,
|
||||
"heapTotal": 186859520,
|
||||
"heapUsed": 51775528,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 521.9577919999999,
|
||||
"rss": 198000640,
|
||||
"heapTotal": 147144704,
|
||||
"heapUsed": 11731728,
|
||||
"external": 2455679,
|
||||
"arrayBuffers": 143819
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 46851576,
|
||||
"finalRetainedDeltaBytes": 6807776,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": null,
|
||||
"ownedKeys": 100000,
|
||||
"countedEvents": 100000,
|
||||
"inputTokens": 10000000,
|
||||
"outputTokens": 1000000,
|
||||
"totalTokens": 11000000,
|
||||
"hasDeferredClaims": false,
|
||||
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"round": 1,
|
||||
"order": ["baseline", "current"],
|
||||
"arm": "current",
|
||||
"stride": 1,
|
||||
"duringSampleCount": 9,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 2.388917000000003,
|
||||
"rss": 54624256,
|
||||
"heapTotal": 7929856,
|
||||
"heapUsed": 4935392,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 43.105542,
|
||||
"rss": 69894144,
|
||||
"heapTotal": 11681792,
|
||||
"heapUsed": 6351040,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 75.529417,
|
||||
"rss": 75300864,
|
||||
"heapTotal": 16728064,
|
||||
"heapUsed": 7330792,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 110.00737500000001,
|
||||
"rss": 76808192,
|
||||
"heapTotal": 18169856,
|
||||
"heapUsed": 8640192,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 147.38125000000002,
|
||||
"rss": 78299136,
|
||||
"heapTotal": 18432000,
|
||||
"heapUsed": 9275016,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 181.302042,
|
||||
"rss": 83001344,
|
||||
"heapTotal": 27344896,
|
||||
"heapUsed": 9940712,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 215.42712500000002,
|
||||
"rss": 88145920,
|
||||
"heapTotal": 29442048,
|
||||
"heapUsed": 11844616,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 250.383958,
|
||||
"rss": 89227264,
|
||||
"heapTotal": 29966336,
|
||||
"heapUsed": 12414208,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 285.865792,
|
||||
"rss": 92225536,
|
||||
"heapTotal": 30752768,
|
||||
"heapUsed": 13073640,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 339.514083,
|
||||
"rss": 95862784,
|
||||
"heapTotal": 32079872,
|
||||
"heapUsed": 14447032,
|
||||
"external": 2455471,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 349.316458,
|
||||
"rss": 93257728,
|
||||
"heapTotal": 29442048,
|
||||
"heapUsed": 11764560,
|
||||
"external": 2455471,
|
||||
"arrayBuffers": 143611
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 9511640,
|
||||
"finalRetainedDeltaBytes": 6829168,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": 100000,
|
||||
"ownedKeys": 100000,
|
||||
"countedEvents": 100000,
|
||||
"inputTokens": 10000000,
|
||||
"outputTokens": 1000000,
|
||||
"totalTokens": 11000000,
|
||||
"hasDeferredClaims": false,
|
||||
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"round": 1,
|
||||
"order": ["baseline", "current"],
|
||||
"arm": "baseline",
|
||||
"stride": 10,
|
||||
"duringSampleCount": 3,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 2.742457999999999,
|
||||
"rss": 54558720,
|
||||
"heapTotal": 7667712,
|
||||
"heapUsed": 4924000,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 40.256542,
|
||||
"rss": 67108864,
|
||||
"heapTotal": 15794176,
|
||||
"heapUsed": 6373000,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 75.06275,
|
||||
"rss": 72105984,
|
||||
"heapTotal": 17022976,
|
||||
"heapUsed": 7649648,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 111.00004200000001,
|
||||
"rss": 81936384,
|
||||
"heapTotal": 26198016,
|
||||
"heapUsed": 8795656,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 153.181625,
|
||||
"rss": 88260608,
|
||||
"heapTotal": 24444928,
|
||||
"heapUsed": 5941160,
|
||||
"external": 2455679,
|
||||
"arrayBuffers": 143819
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 3871656,
|
||||
"finalRetainedDeltaBytes": 1017160,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": null,
|
||||
"ownedKeys": 10000,
|
||||
"countedEvents": 10000,
|
||||
"inputTokens": 1000000,
|
||||
"outputTokens": 100000,
|
||||
"totalTokens": 1100000,
|
||||
"hasDeferredClaims": true,
|
||||
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
|
||||
}
|
||||
},
|
||||
{
|
||||
"round": 1,
|
||||
"order": ["baseline", "current"],
|
||||
"arm": "current",
|
||||
"stride": 10,
|
||||
"duringSampleCount": 4,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 2.448875000000001,
|
||||
"rss": 54788096,
|
||||
"heapTotal": 7929856,
|
||||
"heapUsed": 4935416,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 39.214,
|
||||
"rss": 66109440,
|
||||
"heapTotal": 10813440,
|
||||
"heapUsed": 5545328,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 74.328625,
|
||||
"rss": 66650112,
|
||||
"heapTotal": 10993664,
|
||||
"heapUsed": 5766000,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 109.42824999999999,
|
||||
"rss": 66830336,
|
||||
"heapTotal": 11255808,
|
||||
"heapUsed": 5927144,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 144.325792,
|
||||
"rss": 69926912,
|
||||
"heapTotal": 11419648,
|
||||
"heapUsed": 6228632,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 165.405959,
|
||||
"rss": 72056832,
|
||||
"heapTotal": 10551296,
|
||||
"heapUsed": 5981088,
|
||||
"external": 2455471,
|
||||
"arrayBuffers": 143611
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 1293216,
|
||||
"finalRetainedDeltaBytes": 1045672,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": 10000,
|
||||
"ownedKeys": 10000,
|
||||
"countedEvents": 10000,
|
||||
"inputTokens": 1000000,
|
||||
"outputTokens": 100000,
|
||||
"totalTokens": 1100000,
|
||||
"hasDeferredClaims": true,
|
||||
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
|
||||
}
|
||||
},
|
||||
{
|
||||
"round": 2,
|
||||
"order": ["current", "baseline"],
|
||||
"arm": "current",
|
||||
"stride": 1,
|
||||
"duringSampleCount": 8,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 2.7266249999999985,
|
||||
"rss": 54542336,
|
||||
"heapTotal": 7929856,
|
||||
"heapUsed": 4935464,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 41.696167,
|
||||
"rss": 69419008,
|
||||
"heapTotal": 11157504,
|
||||
"heapUsed": 6326664,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 76.023,
|
||||
"rss": 75857920,
|
||||
"heapTotal": 16465920,
|
||||
"heapUsed": 7366984,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 111.78200000000001,
|
||||
"rss": 77578240,
|
||||
"heapTotal": 17907712,
|
||||
"heapUsed": 8640264,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 146.21325000000002,
|
||||
"rss": 78397440,
|
||||
"heapTotal": 18169856,
|
||||
"heapUsed": 9204112,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 181.093667,
|
||||
"rss": 84312064,
|
||||
"heapTotal": 27344896,
|
||||
"heapUsed": 9868584,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 216.906709,
|
||||
"rss": 89128960,
|
||||
"heapTotal": 29442048,
|
||||
"heapUsed": 11800592,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 251.848084,
|
||||
"rss": 89964544,
|
||||
"heapTotal": 29966336,
|
||||
"heapUsed": 12519232,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 286.69679199999996,
|
||||
"rss": 92831744,
|
||||
"heapTotal": 30752768,
|
||||
"heapUsed": 13286328,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 315.54558399999996,
|
||||
"rss": 93945856,
|
||||
"heapTotal": 29179904,
|
||||
"heapUsed": 11758736,
|
||||
"external": 2455471,
|
||||
"arrayBuffers": 143611
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 8350864,
|
||||
"finalRetainedDeltaBytes": 6823272,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": 100000,
|
||||
"ownedKeys": 100000,
|
||||
"countedEvents": 100000,
|
||||
"inputTokens": 10000000,
|
||||
"outputTokens": 1000000,
|
||||
"totalTokens": 11000000,
|
||||
"hasDeferredClaims": false,
|
||||
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"round": 2,
|
||||
"order": ["current", "baseline"],
|
||||
"arm": "baseline",
|
||||
"stride": 1,
|
||||
"duringSampleCount": 8,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 2.759208000000001,
|
||||
"rss": 54788096,
|
||||
"heapTotal": 7929856,
|
||||
"heapUsed": 4923952,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 41.3715,
|
||||
"rss": 85540864,
|
||||
"heapTotal": 28983296,
|
||||
"heapUsed": 11037688,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 77.633541,
|
||||
"rss": 103530496,
|
||||
"heapTotal": 52133888,
|
||||
"heapUsed": 17544376,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 114.46741599999999,
|
||||
"rss": 115310592,
|
||||
"heapTotal": 92504064,
|
||||
"heapUsed": 24639432,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 152.397541,
|
||||
"rss": 140820480,
|
||||
"heapTotal": 98484224,
|
||||
"heapUsed": 30596424,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 191.927833,
|
||||
"rss": 154828800,
|
||||
"heapTotal": 105037824,
|
||||
"heapUsed": 36979976,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 222.867541,
|
||||
"rss": 160776192,
|
||||
"heapTotal": 110313472,
|
||||
"heapUsed": 42347640,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 266.556791,
|
||||
"rss": 172752896,
|
||||
"heapTotal": 183451648,
|
||||
"heapUsed": 48259976,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 297.627125,
|
||||
"rss": 183664640,
|
||||
"heapTotal": 187383808,
|
||||
"heapUsed": 52060344,
|
||||
"external": 2455471,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 354.97375,
|
||||
"rss": 195674112,
|
||||
"heapTotal": 147406848,
|
||||
"heapUsed": 11724288,
|
||||
"external": 2455679,
|
||||
"arrayBuffers": 143819
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 47136392,
|
||||
"finalRetainedDeltaBytes": 6800336,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": null,
|
||||
"ownedKeys": 100000,
|
||||
"countedEvents": 100000,
|
||||
"inputTokens": 10000000,
|
||||
"outputTokens": 1000000,
|
||||
"totalTokens": 11000000,
|
||||
"hasDeferredClaims": false,
|
||||
"projectionSha256": "e7d64b4651ff4f39da7c2f029ee338dfee0208d9d8c5730531c4a060d0e5564a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"round": 2,
|
||||
"order": ["current", "baseline"],
|
||||
"arm": "current",
|
||||
"stride": 10,
|
||||
"duringSampleCount": 4,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 2.1205,
|
||||
"rss": 54591488,
|
||||
"heapTotal": 7929856,
|
||||
"heapUsed": 4935416,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 38.77175,
|
||||
"rss": 65896448,
|
||||
"heapTotal": 10813440,
|
||||
"heapUsed": 5596672,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 74.407583,
|
||||
"rss": 66158592,
|
||||
"heapTotal": 10993664,
|
||||
"heapUsed": 5771488,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 109.03608299999999,
|
||||
"rss": 66912256,
|
||||
"heapTotal": 11255808,
|
||||
"heapUsed": 5941256,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 143.820708,
|
||||
"rss": 69861376,
|
||||
"heapTotal": 11419648,
|
||||
"heapUsed": 6247120,
|
||||
"external": 2586751,
|
||||
"arrayBuffers": 274891
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 161.987125,
|
||||
"rss": 71991296,
|
||||
"heapTotal": 10551296,
|
||||
"heapUsed": 5976440,
|
||||
"external": 2455471,
|
||||
"arrayBuffers": 143611
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 1311704,
|
||||
"finalRetainedDeltaBytes": 1041024,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": 10000,
|
||||
"ownedKeys": 10000,
|
||||
"countedEvents": 10000,
|
||||
"inputTokens": 1000000,
|
||||
"outputTokens": 100000,
|
||||
"totalTokens": 1100000,
|
||||
"hasDeferredClaims": true,
|
||||
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
|
||||
}
|
||||
},
|
||||
{
|
||||
"round": 2,
|
||||
"order": ["current", "baseline"],
|
||||
"arm": "baseline",
|
||||
"stride": 10,
|
||||
"duringSampleCount": 3,
|
||||
"finalOnlyMeasurement": false,
|
||||
"baseline": {
|
||||
"phase": "before",
|
||||
"elapsedMs": 2.370624999999997,
|
||||
"rss": 54919168,
|
||||
"heapTotal": 7929856,
|
||||
"heapUsed": 4924000,
|
||||
"external": 2455431,
|
||||
"arrayBuffers": 143611
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 40.106750000000005,
|
||||
"rss": 67092480,
|
||||
"heapTotal": 15532032,
|
||||
"heapUsed": 6373000,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 74.903292,
|
||||
"rss": 71827456,
|
||||
"heapTotal": 16760832,
|
||||
"heapUsed": 7649648,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "during",
|
||||
"elapsedMs": 110.34537499999999,
|
||||
"rss": 81395712,
|
||||
"heapTotal": 26722304,
|
||||
"heapUsed": 8851816,
|
||||
"external": 2586543,
|
||||
"arrayBuffers": 274683
|
||||
},
|
||||
{
|
||||
"phase": "after",
|
||||
"elapsedMs": 149.984917,
|
||||
"rss": 86966272,
|
||||
"heapTotal": 24444928,
|
||||
"heapUsed": 5941160,
|
||||
"external": 2455679,
|
||||
"arrayBuffers": 143819
|
||||
}
|
||||
],
|
||||
"sampledRetainedPeakDeltaBytes": 3927816,
|
||||
"finalRetainedDeltaBytes": 1017160,
|
||||
"correctness": {
|
||||
"claimQueries": 100000,
|
||||
"committedKeys": null,
|
||||
"ownedKeys": 10000,
|
||||
"countedEvents": 10000,
|
||||
"inputTokens": 1000000,
|
||||
"outputTokens": 100000,
|
||||
"totalTokens": 1100000,
|
||||
"hasDeferredClaims": true,
|
||||
"projectionSha256": "af92c14a857987f19b0ebfd9bd613a5c65f004c996ed8a60edfd67f810307aac"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
# Verified transcript reads and Codex usage memory
|
||||
|
||||
Codex could verify one file generation, reopen a replacement, and publish old
|
||||
totals under the replacement's metadata. A short parse could also publish a
|
||||
partial result as a complete file. Unchanged refreshes then reused those totals.
|
||||
|
||||
The shared JSONL reader now pins verification, parsing and checkpoint creation
|
||||
to one descriptor and the initial file size. It proves complete range consumption
|
||||
and validates generation and byte evidence before publishing. Stable short reads
|
||||
propagate errors; only evidence that the file changed permits a retry. Each failed
|
||||
attempt closes its handle. A successful attempt closes before ownership publication
|
||||
and aggregate work. Metadata-less caches reparse once.
|
||||
|
||||
Codex folds owned records into the existing session/day accumulator while reading.
|
||||
It commits ownership after validation, and lets unchanged forks reclaim keys that
|
||||
an owner actually loses. Deferred records still advance cumulative token context.
|
||||
This removes intermediate event arrays while preserving all history and results.
|
||||
|
||||
## Reproduce and verify
|
||||
|
||||
```sh
|
||||
ORCA_BACKGROUND_LAUNCH=1 pnpm test src/main/codex-usage src/main/usage
|
||||
ORCA_BACKGROUND_LAUNCH=1 node config/scripts/codex-usage-parser-memory-benchmark.mjs --baseline-ref 51e7181850b022bae2d41091fe1229accef4bd50 --events 100000 --pairs 2 --sample-ms 35 --output docs/bug-reproductions/usage-scan-append/codex-parser-memory.json
|
||||
```
|
||||
|
||||
The original [memory comparison](./codex-parser-memory.json) bundles the full parser
|
||||
dependency graph from the named baseline and current worktree. Two counterbalanced
|
||||
rounds use fresh Node workers and record source/tool hashes and raw forced-GC
|
||||
snapshots. Every arm preserves identical projections, owned keys and token totals.
|
||||
|
||||
| Owned records | Baseline sampled retained heap | Current sampled retained heap |
|
||||
| ------------------------- | -----------------------------: | ----------------------------: |
|
||||
| 100,000 | 44.7–45.0 MiB | 8.0–9.1 MiB |
|
||||
| 10,000, with 90% deferred | 3.69–3.75 MiB | 1.23–1.25 MiB |
|
||||
|
||||
Final result retention stays approximately 6.49 → 6.51 MiB for all owned records:
|
||||
the result still contains the full history. Timer samples miss synchronous peaks,
|
||||
and forced GC affects timing; these figures measure sampled retained heap rather
|
||||
than total process memory or speed.
|
||||
|
||||
Real-file regressions cover replacement, truncation, rewrite, suffix resume,
|
||||
partial tails, stable short reads, failed ownership publication, old-cache upgrade
|
||||
and handle cleanup. Local checks used macOS; CI covers Linux and Windows.
|
||||
The shared aggregation compatibility suites also passed for Muse and OpenCode.
|
||||
|
||||
Head and committed-boundary digests support the append-only contract. They cannot
|
||||
prove arbitrary middle-only rewrites during growth. Small changed files receive
|
||||
a full verification reread; unchanged files read no transcript bytes.
|
||||
|
||||
The companion Claude PR adopts this reader. The original integrated validation
|
||||
and benchmarks were recorded from a combined worktree; production source remains
|
||||
identical when split into PRs.
|
||||
|
||||
## Adversarial hardening
|
||||
|
||||
Tiny files now use bounded descriptor reads and exact byte comparison for both
|
||||
passes. This avoids creating a stream without reducing verification. Exact bigint
|
||||
device/inode values prevent large Windows file IDs from colliding after numeric
|
||||
rounding; unsafe persisted offsets fall back to a complete parse.
|
||||
|
||||
Discovery distinguishes a directory that disappeared from one that cannot be
|
||||
read. Missing children leave readable siblings available. Permission and I/O
|
||||
errors abort the refresh, retaining the previous verified totals and showing the
|
||||
existing error state rather than publishing incomplete totals.
|
||||
|
||||
A [counterbalanced ten-round comparison](./shared-reader-small-files.json) of the published reader and hardened
|
||||
reader over 1,000 tiny files measured median scanner cold time 85.024 → 80.529 ms
|
||||
and changed time 91.512 → 84.092 ms. Both arms read the same bytes twice; streams
|
||||
fell from 1,000 to zero. Unchanged scans read zero bytes in both arms and measured
|
||||
7.808 → 8.241 ms. These page-cache-warm macOS timings are observations, not CI
|
||||
thresholds or a claim about the original pre-verification reader.
|
||||
|
||||
Regression contracts exercise short reads, exact comparison, unsafe offsets,
|
||||
large file identities, fractional timestamps, and discovery failures. The
|
||||
original retained-heap artifact above predates this hardening; its source hashes
|
||||
describe that earlier measurement, rather than the final reader.
|
||||
|
||||
## Further risk reduction
|
||||
|
||||
The [refreshed memory comparison](./codex-parser-memory-final.json) captures the
|
||||
final parser and reader source. Its two counterbalanced rounds use eight fresh
|
||||
Node workers and preserve exact output parity with the original baseline.
|
||||
|
||||
| Owned records | Original sampled retained heap | Final sampled retained heap |
|
||||
| ------------------------- | -----------------------------: | --------------------------: |
|
||||
| 100,000 | 43.87–44.54 MiB | 8.07–8.30 MiB |
|
||||
| 10,000, with 90% deferred | 4.53–4.58 MiB | 1.231–1.235 MiB |
|
||||
|
||||
The median reductions are approximately 81.5% and 72.9%. Final output retention
|
||||
stays similar or slightly higher: 6.49 → 6.53–6.55 MiB for owned events, and
|
||||
0.975 → 1.005–1.014 MiB for deferred events. These are forced-GC samples, not
|
||||
absolute peaks, full-app memory, or speed measurements.
|
||||
|
||||
Real-file lifetime contracts cover both tiny and streamed transcripts. They
|
||||
prove the descriptor is closed before an ownership callback replaces the path,
|
||||
the verified old projection keeps its captured metadata, and the next parse
|
||||
observes the replacement. Failure cases check cleanup when validation fails or
|
||||
the first close attempt rejects before reaching the native close.
|
||||
|
||||
This shortens Windows replacement contention to reading and validation; parsing
|
||||
can still include asynchronous attribution while the descriptor is pinned.
|
||||
Ordinary append writes remain compatible. Tiny verification still reads the full
|
||||
file twice, and bounded append evidence retains its append-only assumption.
|
||||
@@ -0,0 +1,984 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"scenario": "1000-small-files",
|
||||
"scope": "Shared JSONL reader and file metadata hardening, exercised through the combined Claude scanner; the current Claude projection implementation is identical in both arms.",
|
||||
"capturedAtUtc": "2026-10-09",
|
||||
"environment": {
|
||||
"node": "v24.20.0",
|
||||
"platform": "darwin",
|
||||
"osRelease": "25.3.0",
|
||||
"arch": "arm64",
|
||||
"cpuModel": "Apple M4 Max"
|
||||
},
|
||||
"rounds": 10,
|
||||
"schedule": "Alternating before/after and after/before; five rounds in each order; tests and builds held during this final capture.",
|
||||
"baselineCommit": "03a2ecaa00138ee44ad0dbb9dbd957cfe1a5e3f8",
|
||||
"baselineModules": [
|
||||
"src/main/claude-usage/transcript-record-parser.ts",
|
||||
"src/main/usage/jsonl-file-checkpoint.ts",
|
||||
"src/main/usage/jsonl-file-snapshot.ts",
|
||||
"src/main/usage/jsonl-line-offsets.ts"
|
||||
],
|
||||
"baselineHashesVerifiedAgainstGit": true,
|
||||
"currentSourceHashesVerifiedAgainstFrozenWorktree": true,
|
||||
"sourceFingerprintLabels": {
|
||||
"HEAD": "The four pinned published baseline modules supplied as an overlay; this is the existing loader label, not the current Git HEAD.",
|
||||
"worktree": "Frozen combined-worktree modules, held identical in both arms except the four baseline overlays."
|
||||
},
|
||||
"methodology": {
|
||||
"cold": "1,000 freshly written tiny transcripts; empty scanner cache; filesystem page cache warm.",
|
||||
"changed": "Append a duplicate usage row with higher token maxima to each file, then scan from the unchanged cache.",
|
||||
"warm": "Reuse each arm's cold persisted cache without changing files; no transcript content reads.",
|
||||
"durationMs": "Production scan only, excluding fixture writes and cache serialization.",
|
||||
"totalDurationMs": "Production scan plus JSON cache serialization, excluding hydration and fixture writes.",
|
||||
"bytesRead": "All raw bytes reported by the existing benchmark loader, including verification reads.",
|
||||
"readCount": "Logical filesystem API read records, including streams; this does not claim an OS syscall count.",
|
||||
"parity": "Exact sessions, daily aggregates, and per-file projections in cold/changed phases across arms, excluding volatile stat values and additive generation/resume metadata.",
|
||||
"median": "Average of the middle two values for the ten samples; the temporary capture script's upper-middle summaries are retained separately.",
|
||||
"reproduction": [
|
||||
"Use loadClaudeUsageBenchmarkScanner and measureClaudeUsageBenchmarkScan from config/scripts/claude-usage-benchmark-scanner.mjs in the combined Claude/shared-reader worktree.",
|
||||
"Load only the four baselineModules from baselineCommit into the before source-overlay map; load every other module once into a shared worktree-source map used by both arms.",
|
||||
"Create an isolated agent home containing 1,000 JSONL files, one keyed assistant usage row per file, with input_tokens=100 and output_tokens=10.",
|
||||
"Measure a cold scan, an unchanged scan from its cache, then append the same keyed row with input_tokens=200 and measure the changed scan. Exclude batched fixture writes from timers.",
|
||||
"Alternate arm order for ten rounds, assert exact projected behavior parity for cold and changed phases, and assert that every unchanged scan reads zero transcript bytes.",
|
||||
"Compare byte counts and stream counts separately from duration, and record all raw samples and source fingerprints."
|
||||
]
|
||||
},
|
||||
"captureScriptSha256": "d4bac98e4bd15f0ba4760646b9c63cd69160b4f5df7d09ae7b22b101ea0fc9c5",
|
||||
"captureScriptStatus": "Local measurement adapter reused the existing loader; no separate benchmark framework or production tool was added.",
|
||||
"toolingSha256": {
|
||||
"config/scripts/claude-usage-benchmark-scanner.mjs": "ffb6e5defb921f2ab35726a716f5e731dec439ead53cc2f3a192c7bef4df45d3",
|
||||
"config/scripts/counterbalanced-benchmark-schedule.mjs": "bc35b40e954fa7ddec0225ed375bd3b113c2e05b8554bc50734d14f89987294d"
|
||||
},
|
||||
"historyBytes": 307780,
|
||||
"changedSnapshotBytes": 615560,
|
||||
"exactColdChangedProjectionParity": true,
|
||||
"medians": {
|
||||
"before": {
|
||||
"cold": {
|
||||
"durationMs": 85.02441700000008,
|
||||
"cacheSerializationMs": 1.705395499999895,
|
||||
"totalDurationMs": 86.68222949999995,
|
||||
"bytesRead": 615560.0,
|
||||
"directReadBytes": 307780.0,
|
||||
"readCount": 2000.0,
|
||||
"streamCount": 1000.0,
|
||||
"cachedStateBytes": 1508259.0
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 7.808062500000233,
|
||||
"cacheSerializationMs": 1.4102915000000849,
|
||||
"totalDurationMs": 9.195042000000285,
|
||||
"bytesRead": 0.0,
|
||||
"directReadBytes": 0.0,
|
||||
"readCount": 0.0,
|
||||
"streamCount": 0.0,
|
||||
"cachedStateBytes": 1508259.0
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 91.5124165000002,
|
||||
"cacheSerializationMs": 1.5766254999998637,
|
||||
"totalDurationMs": 93.02370800000017,
|
||||
"bytesRead": 1231120.0,
|
||||
"directReadBytes": 615560.0,
|
||||
"readCount": 2000.0,
|
||||
"streamCount": 1000.0,
|
||||
"cachedStateBytes": 1508232.0
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"cold": {
|
||||
"durationMs": 80.5293959999999,
|
||||
"cacheSerializationMs": 1.6749375000002829,
|
||||
"totalDurationMs": 82.21989599999984,
|
||||
"bytesRead": 615560.0,
|
||||
"directReadBytes": 615560.0,
|
||||
"readCount": 2000.0,
|
||||
"streamCount": 0.0,
|
||||
"cachedStateBytes": 1508252.0
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 8.240729500000043,
|
||||
"cacheSerializationMs": 1.4088955000003125,
|
||||
"totalDurationMs": 9.655500500000016,
|
||||
"bytesRead": 0.0,
|
||||
"directReadBytes": 0.0,
|
||||
"readCount": 0.0,
|
||||
"streamCount": 0.0,
|
||||
"cachedStateBytes": 1508252.0
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 84.09189549999974,
|
||||
"cacheSerializationMs": 1.587229000000093,
|
||||
"totalDurationMs": 85.75574999999981,
|
||||
"bytesRead": 1231120.0,
|
||||
"directReadBytes": 1231120.0,
|
||||
"readCount": 2000.0,
|
||||
"streamCount": 0.0,
|
||||
"cachedStateBytes": 1508243.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"durationChangePercent": {
|
||||
"cold": -5.286741336903466,
|
||||
"warm": 5.541285049905742,
|
||||
"changed": -8.108758662274473
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"arm": "before",
|
||||
"round": 0,
|
||||
"cold": {
|
||||
"durationMs": 392.049709,
|
||||
"cacheSerializationMs": 1.6989579999999478,
|
||||
"totalDurationMs": 393.74866699999995,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 307780,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508277
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 48.59562500000004,
|
||||
"cacheSerializationMs": 1.523916999999983,
|
||||
"totalDurationMs": 50.119542000000024,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508277
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 233.39462500000002,
|
||||
"cacheSerializationMs": 1.4007910000000265,
|
||||
"totalDurationMs": 234.79541600000005,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508233
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "after",
|
||||
"round": 0,
|
||||
"cold": {
|
||||
"durationMs": 132.50262499999985,
|
||||
"cacheSerializationMs": 1.6758330000000115,
|
||||
"totalDurationMs": 134.17845799999986,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508243
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 13.144000000000005,
|
||||
"cacheSerializationMs": 1.4084580000001097,
|
||||
"totalDurationMs": 14.552458000000115,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508243
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 111.28033299999993,
|
||||
"cacheSerializationMs": 1.5340830000000096,
|
||||
"totalDurationMs": 112.81441599999994,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 1231120,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508271
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "after",
|
||||
"round": 1,
|
||||
"cold": {
|
||||
"durationMs": 174.1105,
|
||||
"cacheSerializationMs": 1.588083000000097,
|
||||
"totalDurationMs": 175.6985830000001,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508241
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 12.503875000000107,
|
||||
"cacheSerializationMs": 1.357582999999977,
|
||||
"totalDurationMs": 13.861458000000084,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508241
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 127.85537499999987,
|
||||
"cacheSerializationMs": 1.5461250000000746,
|
||||
"totalDurationMs": 129.40149999999994,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 1231120,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508289
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "before",
|
||||
"round": 1,
|
||||
"cold": {
|
||||
"durationMs": 126.94487499999991,
|
||||
"cacheSerializationMs": 1.7684590000001208,
|
||||
"totalDurationMs": 128.71333400000003,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 307780,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508223
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 8.401833000000124,
|
||||
"cacheSerializationMs": 1.3886669999999413,
|
||||
"totalDurationMs": 9.790500000000065,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508223
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 114.26195800000005,
|
||||
"cacheSerializationMs": 1.8519999999998618,
|
||||
"totalDurationMs": 116.11395799999991,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508231
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "before",
|
||||
"round": 2,
|
||||
"cold": {
|
||||
"durationMs": 91.58558300000004,
|
||||
"cacheSerializationMs": 1.4894590000003518,
|
||||
"totalDurationMs": 93.0750420000004,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 307780,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508261
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 8.029250000000047,
|
||||
"cacheSerializationMs": 1.4045000000000982,
|
||||
"totalDurationMs": 9.433750000000146,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508261
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 95.63120799999979,
|
||||
"cacheSerializationMs": 1.6717079999998532,
|
||||
"totalDurationMs": 97.30291599999964,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508229
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "after",
|
||||
"round": 2,
|
||||
"cold": {
|
||||
"durationMs": 109.59825000000001,
|
||||
"cacheSerializationMs": 2.067250000000058,
|
||||
"totalDurationMs": 111.66550000000007,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508283
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 13.671583000000282,
|
||||
"cacheSerializationMs": 1.3706250000000182,
|
||||
"totalDurationMs": 15.0422080000003,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508283
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 108.32254200000034,
|
||||
"cacheSerializationMs": 1.7173749999997199,
|
||||
"totalDurationMs": 110.03991700000006,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 1231120,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508251
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "after",
|
||||
"round": 3,
|
||||
"cold": {
|
||||
"durationMs": 80.29837499999985,
|
||||
"cacheSerializationMs": 1.6784579999998641,
|
||||
"totalDurationMs": 81.97683299999971,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508213
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 8.142833000000337,
|
||||
"cacheSerializationMs": 1.4027909999999792,
|
||||
"totalDurationMs": 9.545624000000316,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508213
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 88.40525000000025,
|
||||
"cacheSerializationMs": 1.6283330000001115,
|
||||
"totalDurationMs": 90.03358300000036,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 1231120,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508237
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "before",
|
||||
"round": 3,
|
||||
"cold": {
|
||||
"durationMs": 75.91833399999996,
|
||||
"cacheSerializationMs": 1.7118329999998423,
|
||||
"totalDurationMs": 77.6301669999998,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 307780,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508261
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 7.40179100000023,
|
||||
"cacheSerializationMs": 1.40166599999975,
|
||||
"totalDurationMs": 8.80345699999998,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508261
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 86.82458300000008,
|
||||
"cacheSerializationMs": 1.5110839999997552,
|
||||
"totalDurationMs": 88.33566699999983,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508229
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "before",
|
||||
"round": 4,
|
||||
"cold": {
|
||||
"durationMs": 86.88720899999998,
|
||||
"cacheSerializationMs": 1.7541670000000522,
|
||||
"totalDurationMs": 88.64137600000004,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 307780,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508281
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 7.466167000000041,
|
||||
"cacheSerializationMs": 1.4322910000000775,
|
||||
"totalDurationMs": 8.898458000000119,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508281
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 91.41150000000016,
|
||||
"cacheSerializationMs": 1.5426669999997102,
|
||||
"totalDurationMs": 92.95416699999987,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508227
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "after",
|
||||
"round": 4,
|
||||
"cold": {
|
||||
"durationMs": 80.76041699999996,
|
||||
"cacheSerializationMs": 1.702541999999994,
|
||||
"totalDurationMs": 82.46295899999996,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508227
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 8.741708000000017,
|
||||
"cacheSerializationMs": 1.4854580000001079,
|
||||
"totalDurationMs": 10.227166000000125,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508227
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 77.78320800000029,
|
||||
"cacheSerializationMs": 1.4414170000000013,
|
||||
"totalDurationMs": 79.22462500000029,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 1231120,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508195
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "after",
|
||||
"round": 5,
|
||||
"cold": {
|
||||
"durationMs": 75.24266699999998,
|
||||
"cacheSerializationMs": 1.6534579999997732,
|
||||
"totalDurationMs": 76.89612499999976,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508281
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 8.298541999999998,
|
||||
"cacheSerializationMs": 1.406333999999788,
|
||||
"totalDurationMs": 9.704875999999786,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508281
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 76.96762499999977,
|
||||
"cacheSerializationMs": 1.446374999999989,
|
||||
"totalDurationMs": 78.41399999999976,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 1231120,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508247
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "before",
|
||||
"round": 5,
|
||||
"cold": {
|
||||
"durationMs": 83.16162500000019,
|
||||
"cacheSerializationMs": 1.561457999999675,
|
||||
"totalDurationMs": 84.72308299999986,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 307780,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508211
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 7.586875000000418,
|
||||
"cacheSerializationMs": 1.3694590000000062,
|
||||
"totalDurationMs": 8.956334000000425,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508211
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 81.44929200000024,
|
||||
"cacheSerializationMs": 1.6840000000001965,
|
||||
"totalDurationMs": 83.13329200000044,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508301
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "before",
|
||||
"round": 6,
|
||||
"cold": {
|
||||
"durationMs": 83.09058300000015,
|
||||
"cacheSerializationMs": 1.5731669999995574,
|
||||
"totalDurationMs": 84.66374999999971,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 307780,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508219
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 7.298624999999447,
|
||||
"cacheSerializationMs": 1.4347920000000158,
|
||||
"totalDurationMs": 8.733416999999463,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508219
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 84.96458299999995,
|
||||
"cacheSerializationMs": 1.6105840000000171,
|
||||
"totalDurationMs": 86.57516699999996,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508219
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "after",
|
||||
"round": 6,
|
||||
"cold": {
|
||||
"durationMs": 70.85174999999981,
|
||||
"cacheSerializationMs": 1.4307080000007772,
|
||||
"totalDurationMs": 72.28245800000059,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508265
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 7.037790999999743,
|
||||
"cacheSerializationMs": 1.4527090000001408,
|
||||
"totalDurationMs": 8.490499999999884,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508265
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 71.76158299999952,
|
||||
"cacheSerializationMs": 1.6893340000005992,
|
||||
"totalDurationMs": 73.45091700000012,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 1231120,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508197
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "after",
|
||||
"round": 7,
|
||||
"cold": {
|
||||
"durationMs": 71.874417,
|
||||
"cacheSerializationMs": 1.7616250000000946,
|
||||
"totalDurationMs": 73.63604200000009,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508241
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 7.877457999999933,
|
||||
"cacheSerializationMs": 1.4134170000006634,
|
||||
"totalDurationMs": 9.290875000000597,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508241
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 71.59512500000073,
|
||||
"cacheSerializationMs": 1.6312079999997877,
|
||||
"totalDurationMs": 73.22633300000052,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 1231120,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508171
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "before",
|
||||
"round": 7,
|
||||
"cold": {
|
||||
"durationMs": 76.68987500000003,
|
||||
"cacheSerializationMs": 1.752542000000176,
|
||||
"totalDurationMs": 78.4424170000002,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 307780,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508263
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 6.472415999999612,
|
||||
"cacheSerializationMs": 1.423917000000074,
|
||||
"totalDurationMs": 7.896332999999686,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508263
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 93.93520900000021,
|
||||
"cacheSerializationMs": 1.528958999999304,
|
||||
"totalDurationMs": 95.46416799999952,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508253
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "before",
|
||||
"round": 8,
|
||||
"cold": {
|
||||
"durationMs": 99.58229100000062,
|
||||
"cacheSerializationMs": 2.150957999999264,
|
||||
"totalDurationMs": 101.73324899999989,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 307780,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508241
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 8.437750000000051,
|
||||
"cacheSerializationMs": 1.4160830000000715,
|
||||
"totalDurationMs": 9.853833000000122,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508241
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 91.61333300000024,
|
||||
"cacheSerializationMs": 1.4799160000002303,
|
||||
"totalDurationMs": 93.09324900000047,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508269
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "after",
|
||||
"round": 8,
|
||||
"cold": {
|
||||
"durationMs": 118.9740839999995,
|
||||
"cacheSerializationMs": 1.6740420000005543,
|
||||
"totalDurationMs": 120.64812600000005,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508267
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 7.485000000000582,
|
||||
"cacheSerializationMs": 1.4093330000005153,
|
||||
"totalDurationMs": 8.894333000001097,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508267
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 83.58533299999999,
|
||||
"cacheSerializationMs": 1.4950419999995574,
|
||||
"totalDurationMs": 85.08037499999955,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 1231120,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508251
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "after",
|
||||
"round": 9,
|
||||
"cold": {
|
||||
"durationMs": 76.86124999999993,
|
||||
"cacheSerializationMs": 1.6329170000008162,
|
||||
"totalDurationMs": 78.49416700000074,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508261
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 8.182917000000089,
|
||||
"cacheSerializationMs": 1.4232080000001588,
|
||||
"totalDurationMs": 9.606125000000247,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508261
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 84.59845799999948,
|
||||
"cacheSerializationMs": 1.8326670000005834,
|
||||
"totalDurationMs": 86.43112500000007,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 1231120,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508239
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "before",
|
||||
"round": 9,
|
||||
"cold": {
|
||||
"durationMs": 79.30645899999945,
|
||||
"cacheSerializationMs": 1.5558329999994385,
|
||||
"totalDurationMs": 80.86229199999889,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 307780,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508257
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 8.131749999999556,
|
||||
"cacheSerializationMs": 1.381416999999601,
|
||||
"totalDurationMs": 9.513166999999157,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508257
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 90.45725000000039,
|
||||
"cacheSerializationMs": 1.6439999999993233,
|
||||
"totalDurationMs": 92.10124999999971,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508269
|
||||
}
|
||||
}
|
||||
],
|
||||
"capturedUpperMiddleSummaries": {
|
||||
"before": {
|
||||
"cold": {
|
||||
"durationMs": 86.88720899999998,
|
||||
"cacheSerializationMs": 1.7118329999998423,
|
||||
"totalDurationMs": 88.64137600000004,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 307780,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508261
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 8.029250000000047,
|
||||
"cacheSerializationMs": 1.4160830000000715,
|
||||
"totalDurationMs": 9.433750000000146,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508261
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 91.61333300000024,
|
||||
"cacheSerializationMs": 1.6105840000000171,
|
||||
"totalDurationMs": 93.09324900000047,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 1000,
|
||||
"cachedStateBytes": 1508233
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"cold": {
|
||||
"durationMs": 80.76041699999996,
|
||||
"cacheSerializationMs": 1.6758330000000115,
|
||||
"totalDurationMs": 82.46295899999996,
|
||||
"bytesRead": 615560,
|
||||
"directReadBytes": 615560,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508261
|
||||
},
|
||||
"warm": {
|
||||
"durationMs": 8.298541999999998,
|
||||
"cacheSerializationMs": 1.4093330000005153,
|
||||
"totalDurationMs": 9.704875999999786,
|
||||
"bytesRead": 0,
|
||||
"directReadBytes": 0,
|
||||
"readCount": 0,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508261
|
||||
},
|
||||
"changed": {
|
||||
"durationMs": 84.59845799999948,
|
||||
"cacheSerializationMs": 1.6283330000001115,
|
||||
"totalDurationMs": 86.43112500000007,
|
||||
"bytesRead": 1231120,
|
||||
"directReadBytes": 1231120,
|
||||
"readCount": 2000,
|
||||
"streamCount": 0,
|
||||
"cachedStateBytes": 1508247
|
||||
}
|
||||
}
|
||||
},
|
||||
"sourceFingerprints": {
|
||||
"before": {
|
||||
"src/main/claude-usage/scanner.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "ef42add20f63152cf2ebe106e0794119b9c388c763736c8450b559ae7c2418f5"
|
||||
},
|
||||
"src/main/claude-usage/transcript-file-discovery.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "54c84d0b99b45c0fa13e589a1bcec2b479cb01e8b17bcbddbb65a4bc85a03401"
|
||||
},
|
||||
"src/main/claude-usage/transcript-projection-integrity.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "f4efccd6f1be0603a90174f66ac961d07ea5bf1c1fbebe015b0537deee1c58f2"
|
||||
},
|
||||
"src/main/claude-usage/transcript-record-parser.ts": {
|
||||
"source": "HEAD",
|
||||
"sha256": "d4fab89434c1b6c60480a561310542246be34ef86bc13da0340c1dfdbb269854"
|
||||
},
|
||||
"src/main/claude-usage/transcript-resume-projection.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "b2ed0a589b60fdccb02c72fa234368853d32be080a84ee3bcc93f5834478a153"
|
||||
},
|
||||
"src/main/claude-usage/transcript-token-checkpoint.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "e96cc4aa9b899cad90aee462f053f483f5bb010dea99de1ee11a9f8fd8a2d0f2"
|
||||
},
|
||||
"src/main/claude-usage/transcript-usage-projection.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "56e2fe017f806670595dae766b6291e09b05b321b83e4b83e1ae5166bab2041c"
|
||||
},
|
||||
"src/main/claude-usage/usage-aggregation.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "be4dad6f41eaf720252137b16b9a96a2cfdd2e73b05eee097993653ce6c5e509"
|
||||
},
|
||||
"src/main/claude-usage/worktree-attribution.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "7450e5ca7d8dcb66d1f5003a3e445c29f05c1416c6cd63aeb3f5f74804bc6213"
|
||||
},
|
||||
"src/main/usage/jsonl-file-checkpoint.ts": {
|
||||
"source": "HEAD",
|
||||
"sha256": "8eb579f356e1bb23b36319f00b3d4603fff80a963222dcb8a22331319cab3a03"
|
||||
},
|
||||
"src/main/usage/jsonl-file-snapshot.ts": {
|
||||
"source": "HEAD",
|
||||
"sha256": "1bd98ac194f73097cd6c204238fe1db01e1e0d60a604e93c1d7841725971fe0d"
|
||||
},
|
||||
"src/main/usage/jsonl-line-offsets.ts": {
|
||||
"source": "HEAD",
|
||||
"sha256": "f119465fad3771acd72b128853e84eed9165f0f74b75743ad662aa5bab991273"
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"src/main/claude-usage/scanner.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "ef42add20f63152cf2ebe106e0794119b9c388c763736c8450b559ae7c2418f5"
|
||||
},
|
||||
"src/main/claude-usage/transcript-file-discovery.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "54c84d0b99b45c0fa13e589a1bcec2b479cb01e8b17bcbddbb65a4bc85a03401"
|
||||
},
|
||||
"src/main/claude-usage/transcript-projection-integrity.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "f4efccd6f1be0603a90174f66ac961d07ea5bf1c1fbebe015b0537deee1c58f2"
|
||||
},
|
||||
"src/main/claude-usage/transcript-record-parser.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "6598c02e38ea1776ef5ef926ca424dbc5880f27e7ff03b651cf8f60b791996f6"
|
||||
},
|
||||
"src/main/claude-usage/transcript-resume-projection.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "b2ed0a589b60fdccb02c72fa234368853d32be080a84ee3bcc93f5834478a153"
|
||||
},
|
||||
"src/main/claude-usage/transcript-token-checkpoint.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "e96cc4aa9b899cad90aee462f053f483f5bb010dea99de1ee11a9f8fd8a2d0f2"
|
||||
},
|
||||
"src/main/claude-usage/transcript-usage-projection.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "56e2fe017f806670595dae766b6291e09b05b321b83e4b83e1ae5166bab2041c"
|
||||
},
|
||||
"src/main/claude-usage/usage-aggregation.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "be4dad6f41eaf720252137b16b9a96a2cfdd2e73b05eee097993653ce6c5e509"
|
||||
},
|
||||
"src/main/claude-usage/worktree-attribution.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "7450e5ca7d8dcb66d1f5003a3e445c29f05c1416c6cd63aeb3f5f74804bc6213"
|
||||
},
|
||||
"src/main/usage/jsonl-file-checkpoint.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "b7f307564989dabaf1dfda095a26c7bd9f0683e0efcabe79a010c7b58fa1a2d4"
|
||||
},
|
||||
"src/main/usage/jsonl-file-snapshot.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "b1b8869db92c08c762160acba7570aabff25879649cb6ffa3826912ee35deffe"
|
||||
},
|
||||
"src/main/usage/jsonl-line-offsets.ts": {
|
||||
"source": "worktree",
|
||||
"sha256": "f3574adfad4549c6330e2ac52760c91be5a8cbb9bf8e1beb440e3226087074db"
|
||||
}
|
||||
}
|
||||
},
|
||||
"limits": [
|
||||
"One macOS host and page-cache-warm fixtures; durations are evidence for this scenario, not a platform-independent performance contract.",
|
||||
"This comparison is against the published incremental reader implementation, not the pre-PR full-scan implementation.",
|
||||
"Both arms still read each small cold/changed snapshot twice; unchanged scans read zero transcript bytes.",
|
||||
"Warm duration rose by about 0.43 ms over 1,000 files in this capture; exact bigint metadata adds CPU work without adding stat calls.",
|
||||
"Large-file append-only checkpoints retain their documented limitation for arbitrary middle rewrites during growth."
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { mkdtempSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
||||
import type * as NodeFsPromises from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { parseCodexUsageFile } from './codex-rollout-file-parse'
|
||||
|
||||
const { openedHandles, closedHandles, failNextClose } = vi.hoisted(() => ({
|
||||
openedHandles: new Array<NodeFsPromises.FileHandle>(),
|
||||
closedHandles: new Set<NodeFsPromises.FileHandle>(),
|
||||
failNextClose: { current: false }
|
||||
}))
|
||||
|
||||
vi.mock('node:fs/promises', async () => {
|
||||
const actual = await vi.importActual<typeof NodeFsPromises>('node:fs/promises')
|
||||
return {
|
||||
...actual,
|
||||
open: async (...args: Parameters<typeof actual.open>) => {
|
||||
const handle = await actual.open(...args)
|
||||
openedHandles.push(handle)
|
||||
const closeHandle = handle.close.bind(handle)
|
||||
vi.spyOn(handle, 'close').mockImplementation(async () => {
|
||||
if (failNextClose.current) {
|
||||
failNextClose.current = false
|
||||
throw new Error('descriptor close failed')
|
||||
}
|
||||
await closeHandle()
|
||||
closedHandles.add(handle)
|
||||
})
|
||||
return handle
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
function rollout(sessionId: string, recordCount: number): string {
|
||||
const records = [JSON.stringify({ type: 'session_meta', payload: { id: sessionId } })]
|
||||
for (let index = 0; index < recordCount; index++) {
|
||||
records.push(
|
||||
JSON.stringify({
|
||||
timestamp: new Date(Date.UTC(2026, 4, 26, 12, index)).toISOString(),
|
||||
type: 'event_msg',
|
||||
payload: {
|
||||
type: 'token_count',
|
||||
info: {
|
||||
model: 'gpt-5-codex',
|
||||
last_token_usage: { input_tokens: 1, total_tokens: 1 },
|
||||
total_token_usage: { input_tokens: index + 1, total_tokens: index + 1 }
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
return `${records.join('\n')}\n`
|
||||
}
|
||||
|
||||
let directory: string
|
||||
|
||||
beforeEach(() => {
|
||||
directory = mkdtempSync(join(tmpdir(), 'orca-codex-rollout-lifetime-'))
|
||||
openedHandles.length = 0
|
||||
closedHandles.clear()
|
||||
failNextClose.current = false
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(directory, { recursive: true, force: true })
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
describe('Codex rollout descriptor lifetime', () => {
|
||||
it.each([3, 100])(
|
||||
'closes the verified %i-record rollout before publishing ownership',
|
||||
async (recordCount) => {
|
||||
const rolloutPath = join(directory, 'rollout.jsonl')
|
||||
const replacementPath = join(directory, 'replacement.tmp')
|
||||
writeFileSync(rolloutPath, rollout('verified-session', recordCount))
|
||||
const originalStat = statSync(rolloutPath, { bigint: true })
|
||||
writeFileSync(replacementPath, rollout('replacement-session', 2))
|
||||
let committedKeys = 0
|
||||
|
||||
const verified = await parseCodexUsageFile(rolloutPath, () => null, {
|
||||
commitEventKey: () => {
|
||||
expect(closedHandles.size).toBe(openedHandles.length)
|
||||
expect(openedHandles.every((handle) => handle.fd === -1)).toBe(true)
|
||||
if (committedKeys++ === 0) {
|
||||
renameSync(replacementPath, rolloutPath)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
expect(committedKeys).toBe(recordCount)
|
||||
expect(verified.sessions.map(({ sessionId, eventCount }) => [sessionId, eventCount])).toEqual(
|
||||
[['verified-session', recordCount]]
|
||||
)
|
||||
expect(verified.size).toBe(Number(originalStat.size))
|
||||
expect(verified.physicalFileId).toBe(
|
||||
originalStat.ino === 0n ? null : `${originalStat.dev}:${originalStat.ino}`
|
||||
)
|
||||
expect(verified.parseResumeState !== null).toBe(recordCount === 100)
|
||||
const refreshed = await parseCodexUsageFile(
|
||||
rolloutPath,
|
||||
() => null,
|
||||
verified.parseResumeState
|
||||
? { resume: { state: verified.parseResumeState, previous: verified } }
|
||||
: {}
|
||||
)
|
||||
expect(
|
||||
refreshed.sessions.map(({ sessionId, eventCount }) => [sessionId, eventCount])
|
||||
).toEqual([['replacement-session', 2]])
|
||||
expect(closedHandles.size).toBe(openedHandles.length)
|
||||
}
|
||||
)
|
||||
|
||||
it('keeps the descriptor closed when publishing ownership fails', async () => {
|
||||
const rolloutPath = join(directory, 'rollout.jsonl')
|
||||
writeFileSync(rolloutPath, rollout('failed-claims', 3))
|
||||
|
||||
await expect(
|
||||
parseCodexUsageFile(rolloutPath, () => null, {
|
||||
commitEventKey: () => {
|
||||
expect(closedHandles.size).toBe(openedHandles.length)
|
||||
throw new Error('ownership publication failed')
|
||||
}
|
||||
})
|
||||
).rejects.toThrow('ownership publication failed')
|
||||
expect(openedHandles).toHaveLength(1)
|
||||
expect(closedHandles.size).toBe(1)
|
||||
expect(openedHandles[0]?.fd).toBe(-1)
|
||||
})
|
||||
|
||||
it('publishes no ownership if closing the verified descriptor fails', async () => {
|
||||
const rolloutPath = join(directory, 'rollout.jsonl')
|
||||
writeFileSync(rolloutPath, rollout('close-failure', 3))
|
||||
const commitEventKey = vi.fn()
|
||||
failNextClose.current = true
|
||||
|
||||
await expect(parseCodexUsageFile(rolloutPath, () => null, { commitEventKey })).rejects.toThrow(
|
||||
'descriptor close failed'
|
||||
)
|
||||
expect(commitEventKey).not.toHaveBeenCalled()
|
||||
expect(openedHandles).toHaveLength(1)
|
||||
expect(closedHandles.size).toBe(1)
|
||||
expect(openedHandles[0]?.fd).toBe(-1)
|
||||
})
|
||||
})
|
||||
@@ -1,6 +1,11 @@
|
||||
import { basename } from 'node:path'
|
||||
import { stat } from 'node:fs/promises'
|
||||
import { readJsonlFileSnapshot, type JsonlFileSnapshot } from '../usage/jsonl-file-snapshot'
|
||||
import { readJsonlLinesFromOffset } from '../usage/jsonl-line-offsets'
|
||||
import {
|
||||
openJsonlFileReader,
|
||||
validateJsonlFileReader,
|
||||
jsonlPhysicalFileId
|
||||
} from '../usage/jsonl-file-checkpoint'
|
||||
import { attributeCodexUsageEvent } from './codex-usage-event-attribution'
|
||||
import type { UsageWorktreeResolver } from '../usage/usage-worktree-resolver'
|
||||
import { parseCodexUsageRecord, type CodexUsageParseContext } from './codex-usage-record-parser'
|
||||
@@ -10,7 +15,6 @@ import {
|
||||
resolveCodexRolloutResume
|
||||
} from './codex-rollout-resume-state'
|
||||
import type {
|
||||
CodexUsageAttributedEvent,
|
||||
CodexUsageDailyAggregate,
|
||||
CodexUsageParseResumeState,
|
||||
CodexUsagePersistedFile,
|
||||
@@ -24,17 +28,24 @@ const { finalizeSessions, mergeSessions, mergeDailyAggregates, sortDailyAggregat
|
||||
export type CodexRolloutParseOptions = {
|
||||
/** Suffix-only parse for a diverged legacy copied-session bridge. */
|
||||
legacySourceSkipBytes?: number
|
||||
claimEventKey?: (eventKey: string) => boolean
|
||||
canClaimEventKey?: (eventKey: string) => boolean
|
||||
commitEventKey?: (eventKey: string) => void
|
||||
/** Resume point verified by the caller, with the cached projection to extend. */
|
||||
resume?: { state: CodexUsageParseResumeState; previous: CodexUsagePersistedFile }
|
||||
}
|
||||
|
||||
export async function getProcessedFileInfo(filePath: string): Promise<CodexUsageProcessedFile> {
|
||||
const fileStat = await stat(filePath)
|
||||
const fileStat = await readJsonlFileSnapshot(filePath)
|
||||
return processedFileInfo(filePath, fileStat)
|
||||
}
|
||||
|
||||
function processedFileInfo(filePath: string, fileStat: JsonlFileSnapshot): CodexUsageProcessedFile {
|
||||
return {
|
||||
path: filePath,
|
||||
mtimeMs: fileStat.mtimeMs,
|
||||
size: fileStat.size
|
||||
size: fileStat.size,
|
||||
ctimeMs: fileStat.ctimeMs,
|
||||
physicalFileId: jsonlPhysicalFileId(fileStat)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,106 +97,104 @@ export async function parseCodexUsageFile(
|
||||
resolveWorktree: UsageWorktreeResolver,
|
||||
options: CodexRolloutParseOptions = {}
|
||||
): Promise<CodexUsagePersistedFile> {
|
||||
// Why: the caller verified this resume point while walking the directory, and
|
||||
// every file discovered or parsed since then has run in between. Re-verify
|
||||
// here, against the file about to be read, or a rollout replaced in that gap
|
||||
// gets the cached session id, cwd, model and running totals stitched onto an
|
||||
// unrelated file's records — and `processedFile` below re-stats to the new
|
||||
// size, so the reuse path then freezes the corrupted projection.
|
||||
if (
|
||||
options.resume &&
|
||||
(await resolveCodexRolloutResume(filePath, options.resume.previous)) === null
|
||||
) {
|
||||
return parseCodexUsageFile(filePath, resolveWorktree, { ...options, resume: undefined })
|
||||
}
|
||||
let parseOptions = options
|
||||
for (;;) {
|
||||
const reader = await openJsonlFileReader(filePath)
|
||||
try {
|
||||
// Verification, parsing and checkpointing must all read the same opened file.
|
||||
if (
|
||||
parseOptions.resume &&
|
||||
(await resolveCodexRolloutResume(filePath, parseOptions.resume.previous, reader)) === null
|
||||
) {
|
||||
parseOptions = { ...parseOptions, resume: undefined }
|
||||
continue
|
||||
}
|
||||
|
||||
const processedFile = await getProcessedFileInfo(filePath)
|
||||
const legacySourceSkipBytes = options.legacySourceSkipBytes ?? 0
|
||||
const startOffset = options.resume?.state.parsedBytes ?? legacySourceSkipBytes
|
||||
const context = createParseContext(filePath, options)
|
||||
const legacySourceSkipBytes = parseOptions.legacySourceSkipBytes ?? 0
|
||||
const startOffset = parseOptions.resume?.state.parsedBytes ?? legacySourceSkipBytes
|
||||
const context = createParseContext(filePath, parseOptions)
|
||||
const accumulator = codexUsageAggregation.createAccumulator()
|
||||
const ownedEventKeys = new Set<string>()
|
||||
let hasDeferredClaims = false
|
||||
let parsedBytes = startOffset
|
||||
let resumeContext = context
|
||||
let partialTailProducedEvent = false
|
||||
|
||||
const events: CodexUsageAttributedEvent[] = []
|
||||
const ownedEventKeys = new Set<string>()
|
||||
let hasDeferredClaims = false
|
||||
let parsedBytes = startOffset
|
||||
// Points at the context as of `parsedBytes`, which excludes a partial tail.
|
||||
let resumeContext = context
|
||||
let partialTailProducedEvent = false
|
||||
|
||||
for await (const { line, endOffset, terminated } of readJsonlLinesFromOffset(
|
||||
filePath,
|
||||
startOffset
|
||||
)) {
|
||||
if (!terminated) {
|
||||
// Only the final fragment can be unterminated, and the next scan re-reads
|
||||
// it, so its context edits must not leak into the persisted resume point.
|
||||
resumeContext = { ...context }
|
||||
}
|
||||
const parsed = parseCodexUsageRecord(line, context)
|
||||
if (terminated) {
|
||||
parsedBytes = endOffset
|
||||
} else if (parsed) {
|
||||
partialTailProducedEvent = true
|
||||
}
|
||||
if (!parsed) {
|
||||
continue
|
||||
}
|
||||
// Why: fork/resume rollouts start with a copied prefix of the parent file.
|
||||
// Events another file already owns are dropped here, but the record still
|
||||
// advanced context.previousTotals above, so later deltas stay correct.
|
||||
if (options.claimEventKey && !options.claimEventKey(parsed.eventKey)) {
|
||||
hasDeferredClaims = true
|
||||
continue
|
||||
}
|
||||
ownedEventKeys.add(parsed.eventKey)
|
||||
const attributed = await attributeCodexUsageEvent(parsed, resolveWorktree)
|
||||
if (attributed) {
|
||||
events.push(attributed)
|
||||
}
|
||||
}
|
||||
|
||||
// A counted-but-unterminated tail would be counted again on resume, and a
|
||||
// legacy suffix offset is recomputed per scan, so neither may be resumed. A
|
||||
// prefix under the resumable floor is turned away by the builder itself.
|
||||
const resumeStateSuppressed = partialTailProducedEvent || legacySourceSkipBytes > 0
|
||||
const parseResumeState = resumeStateSuppressed
|
||||
? null
|
||||
: await buildCodexRolloutResumeState(
|
||||
for await (const { line, endOffset, terminated } of readJsonlLinesFromOffset(
|
||||
filePath,
|
||||
parsedBytes,
|
||||
resumeContext,
|
||||
// Already verified against the file at the top of this scan.
|
||||
options.resume?.state.headDigest ?? null
|
||||
)
|
||||
startOffset,
|
||||
reader
|
||||
)) {
|
||||
if (!terminated) {
|
||||
// The next scan rereads this tail, including its context changes.
|
||||
resumeContext = { ...context }
|
||||
}
|
||||
const parsed = parseCodexUsageRecord(line, context)
|
||||
if (terminated) {
|
||||
parsedBytes = endOffset
|
||||
} else if (parsed) {
|
||||
partialTailProducedEvent = true
|
||||
}
|
||||
if (!parsed) {
|
||||
continue
|
||||
}
|
||||
// Deferred records still advance cumulative totals without retaining their copies.
|
||||
if (parseOptions.canClaimEventKey && !parseOptions.canClaimEventKey(parsed.eventKey)) {
|
||||
hasDeferredClaims = true
|
||||
continue
|
||||
}
|
||||
ownedEventKeys.add(parsed.eventKey)
|
||||
const attributed = await attributeCodexUsageEvent(parsed, resolveWorktree)
|
||||
if (attributed) {
|
||||
accumulator.add(attributed)
|
||||
}
|
||||
}
|
||||
|
||||
// Why: a resume point only exists past the resumable floor and `parsedBytes`
|
||||
// only grows, so the builder's other null — a prefix too short to be worth
|
||||
// resuming — is unreachable here and this null means a short read: the file
|
||||
// shrank past the prefix this parse merged history for, after the
|
||||
// re-verification above and during the read. `processedFile` already re-stat'd
|
||||
// to the smaller size, so persisting that pair would let the next scan reuse a
|
||||
// pre-truncation total forever. An unterminated tail proves the file still
|
||||
// runs past the resume offset, so it cannot be this case.
|
||||
if (options.resume && !resumeStateSuppressed && parseResumeState === null) {
|
||||
return parseCodexUsageFile(filePath, resolveWorktree, { ...options, resume: undefined })
|
||||
}
|
||||
// A counted unterminated row would be counted again from the committed offset.
|
||||
const resumeStateSuppressed = partialTailProducedEvent || legacySourceSkipBytes > 0
|
||||
const parseResumeState = resumeStateSuppressed
|
||||
? null
|
||||
: await buildCodexRolloutResumeState(
|
||||
filePath,
|
||||
parsedBytes,
|
||||
resumeContext,
|
||||
parseOptions.resume?.state.headDigest ?? null,
|
||||
reader
|
||||
)
|
||||
|
||||
const appended = codexUsageAggregation.aggregate(events)
|
||||
const previous = options.resume?.previous
|
||||
if (!previous) {
|
||||
return {
|
||||
...processedFile,
|
||||
...appended,
|
||||
ownedEventKeys: [...ownedEventKeys],
|
||||
hasDeferredClaims,
|
||||
parseResumeState
|
||||
if (
|
||||
!(await validateJsonlFileReader(reader, parseOptions.resume?.state)) ||
|
||||
(parseOptions.resume && !resumeStateSuppressed && parseResumeState === null)
|
||||
) {
|
||||
parseOptions = { ...parseOptions, resume: undefined }
|
||||
continue
|
||||
}
|
||||
|
||||
await reader.handle.close()
|
||||
|
||||
// Discarded reads cannot claim keys that would hide records in another file.
|
||||
for (const eventKey of ownedEventKeys) {
|
||||
parseOptions.commitEventKey?.(eventKey)
|
||||
}
|
||||
|
||||
const processedFile = processedFileInfo(filePath, reader.stats)
|
||||
const appended = accumulator.finalize()
|
||||
const previous = parseOptions.resume?.previous
|
||||
const committedKeys = previous ? new Set(previous.ownedEventKeys) : ownedEventKeys
|
||||
if (previous) {
|
||||
for (const key of ownedEventKeys) {
|
||||
committedKeys.add(key)
|
||||
}
|
||||
}
|
||||
return {
|
||||
...processedFile,
|
||||
...(previous ? mergeRolloutProjections(previous, appended) : appended),
|
||||
ownedEventKeys: [...committedKeys],
|
||||
hasDeferredClaims: previous?.hasDeferredClaims || hasDeferredClaims,
|
||||
parseResumeState
|
||||
}
|
||||
} finally {
|
||||
await reader.handle.close()
|
||||
}
|
||||
}
|
||||
return {
|
||||
...processedFile,
|
||||
...mergeRolloutProjections(previous, appended),
|
||||
ownedEventKeys: [...new Set([...previous.ownedEventKeys, ...ownedEventKeys])],
|
||||
hasDeferredClaims: previous.hasDeferredClaims || hasDeferredClaims,
|
||||
parseResumeState
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,183 +1,46 @@
|
||||
/**
|
||||
* Decides whether a grown rollout can be parsed from where the last scan
|
||||
* stopped. A wrong answer here silently corrupts usage totals, so every check
|
||||
* fails closed: anything unproven falls back to a full reparse.
|
||||
*/
|
||||
import { createReadStream } from 'node:fs'
|
||||
import { stat } from 'node:fs/promises'
|
||||
import { createHash } from 'node:crypto'
|
||||
import type { CodexUsageParseContext } from './codex-usage-record-parser'
|
||||
import type { CodexUsageParseResumeState, CodexUsagePersistedFile } from './types'
|
||||
import {
|
||||
buildJsonlFileCheckpoint,
|
||||
resolveJsonlFileCheckpoint,
|
||||
type JsonlFileReader
|
||||
} from '../usage/jsonl-file-checkpoint'
|
||||
|
||||
/** Bytes hashed immediately before the resume offset. Large enough to span a
|
||||
* whole token_count record, small enough that verifying it is free next to
|
||||
* re-reading a multi-megabyte rollout. */
|
||||
const BOUNDARY_WINDOW_BYTES = 4096
|
||||
|
||||
/** Bytes hashed at the very start of the parsed prefix. The boundary window can
|
||||
* only speak for the bytes next to the resume offset, so without this a
|
||||
* same-length rewrite that leaves the tail intact resumes over changed
|
||||
* history. Every realistic rotation or rewrite of a rollout replaces the
|
||||
* leading session_meta line, which lands in this window. */
|
||||
const HEAD_WINDOW_BYTES = 4096
|
||||
|
||||
/**
|
||||
* Shortest prefix worth resuming over. A resumed scan verifies the prefix
|
||||
* twice — once when the scanner plans the resume, once against the file it is
|
||||
* about to read — and then records the moved boundary, so it pays five windows
|
||||
* whatever the file size. A cold reparse pays the prefix itself plus the two
|
||||
* windows it records. Below this length reading the whole file is cheaper, and
|
||||
* for a prefix short enough that the two windows overlap it is far cheaper,
|
||||
* because each verification then rehashes the entire prefix.
|
||||
*
|
||||
* Measured on the byte oracle: a 12,191 B prefix costs 21,234 B resumed against
|
||||
* 21,137 B cold; at 13,699 B it is 21,234 B against 22,645 B.
|
||||
*/
|
||||
export const MIN_RESUMABLE_PREFIX_BYTES = 3 * BOUNDARY_WINDOW_BYTES
|
||||
|
||||
/** Below the floor the two windows could also overlap, so every offset that
|
||||
* reaches the digest reads is guaranteed to give them a disjoint layout. */
|
||||
export function isResumablePrefixLength(parsedBytes: number): boolean {
|
||||
return parsedBytes > MIN_RESUMABLE_PREFIX_BYTES
|
||||
}
|
||||
|
||||
async function readWindowDigest(
|
||||
filePath: string,
|
||||
start: number,
|
||||
endExclusive: number
|
||||
): Promise<string | null> {
|
||||
const expectedBytes = endExclusive - start
|
||||
const hash = createHash('sha256')
|
||||
let readBytes = 0
|
||||
const stream = createReadStream(filePath, { start, end: endExclusive - 1 })
|
||||
for await (const chunk of stream) {
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
|
||||
hash.update(buffer)
|
||||
readBytes += buffer.length
|
||||
}
|
||||
// A short read means the file no longer reaches the offset we recorded.
|
||||
return readBytes === expectedBytes ? `${expectedBytes}:${hash.digest('hex')}` : null
|
||||
}
|
||||
|
||||
type PrefixDigests = { headDigest: string; boundaryDigest: string }
|
||||
|
||||
/**
|
||||
* Both window digests for a prefix of `parsedBytes`, or null if the file no
|
||||
* longer reaches that offset. The window layout is a pure function of
|
||||
* `parsedBytes`, so a later verification hashes exactly the same ranges. Only
|
||||
* called for a prefix past `MIN_RESUMABLE_PREFIX_BYTES`, so the two windows are
|
||||
* always disjoint and always their full size.
|
||||
*
|
||||
* `carriedHeadDigest` lets a caller that already verified the head window this
|
||||
* scan skip re-reading those bytes. Deferring to it is self-correcting: if the
|
||||
* head did change after that read, the next scan compares against the carried
|
||||
* value and falls back to a full reparse.
|
||||
*/
|
||||
async function readPrefixDigests(
|
||||
filePath: string,
|
||||
parsedBytes: number,
|
||||
carriedHeadDigest: string | null = null
|
||||
): Promise<PrefixDigests | null> {
|
||||
const headDigest = carriedHeadDigest?.startsWith(`${HEAD_WINDOW_BYTES}:`)
|
||||
? carriedHeadDigest
|
||||
: await readWindowDigest(filePath, 0, HEAD_WINDOW_BYTES)
|
||||
if (headDigest === null) {
|
||||
return null
|
||||
}
|
||||
const boundaryDigest = await readWindowDigest(
|
||||
filePath,
|
||||
parsedBytes - BOUNDARY_WINDOW_BYTES,
|
||||
parsedBytes
|
||||
)
|
||||
return boundaryDigest === null ? null : { headDigest, boundaryDigest }
|
||||
}
|
||||
|
||||
async function readPhysicalFileId(filePath: string): Promise<string | null> {
|
||||
try {
|
||||
const fileStat = await stat(filePath)
|
||||
return fileStat.ino === 0 ? null : `${fileStat.dev}:${fileStat.ino}`
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function isUsableResumeState(
|
||||
resume: CodexUsageParseResumeState | null | undefined
|
||||
): resume is CodexUsageParseResumeState {
|
||||
return (
|
||||
resume != null &&
|
||||
Number.isInteger(resume.parsedBytes) &&
|
||||
resume.parsedBytes >= 0 &&
|
||||
typeof resume.boundaryDigest === 'string' &&
|
||||
// State persisted before the head window existed cannot be verified.
|
||||
typeof resume.headDigest === 'string' &&
|
||||
// Caches written before the floor existed can hold a shorter prefix.
|
||||
isResumablePrefixLength(resume.parsedBytes) &&
|
||||
typeof resume.sessionId === 'string'
|
||||
)
|
||||
}
|
||||
export { MIN_RESUMABLE_PREFIX_BYTES, isResumablePrefixLength } from '../usage/jsonl-file-checkpoint'
|
||||
|
||||
export async function buildCodexRolloutResumeState(
|
||||
filePath: string,
|
||||
parsedBytes: number,
|
||||
context: CodexUsageParseContext,
|
||||
verifiedHeadDigest: string | null = null
|
||||
verifiedHeadDigest: string | null = null,
|
||||
reader?: JsonlFileReader
|
||||
): Promise<CodexUsageParseResumeState | null> {
|
||||
if (!isResumablePrefixLength(parsedBytes)) {
|
||||
return null
|
||||
}
|
||||
const digests = await readPrefixDigests(filePath, parsedBytes, verifiedHeadDigest)
|
||||
if (digests === null) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
const checkpoint = await buildJsonlFileCheckpoint(
|
||||
filePath,
|
||||
parsedBytes,
|
||||
boundaryDigest: digests.boundaryDigest,
|
||||
headDigest: digests.headDigest,
|
||||
physicalFileId: await readPhysicalFileId(filePath),
|
||||
sessionId: context.sessionId,
|
||||
sessionCwd: context.sessionCwd,
|
||||
currentCwd: context.currentCwd,
|
||||
currentModel: context.currentModel,
|
||||
previousTotals: context.previousTotals
|
||||
}
|
||||
verifiedHeadDigest,
|
||||
reader
|
||||
)
|
||||
return checkpoint === null
|
||||
? null
|
||||
: {
|
||||
...checkpoint,
|
||||
sessionId: context.sessionId,
|
||||
sessionCwd: context.sessionCwd,
|
||||
currentCwd: context.currentCwd,
|
||||
currentModel: context.currentModel,
|
||||
previousTotals: context.previousTotals
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the resume point only when the recorded prefix still looks like the
|
||||
* file's prefix. Deliberately does not consult any timestamp: filesystems vary
|
||||
* in clock granularity, so a rewrite can land under the mtime, ctime or
|
||||
* birthtime the cache already holds. `physicalFileId` is a cheap extra catch
|
||||
* rather than a rotation check — ext4 and overlayfs hand a recreated path the
|
||||
* inode the old file freed, so it only fires on filesystems that allocate a
|
||||
* fresh one. Truncation needs no separate check: a file that no longer reaches
|
||||
* the offset cannot produce the recorded digests.
|
||||
*
|
||||
* Bounded by design, so for a prefix larger than the two windows together this
|
||||
* cannot prove every byte between them is intact.
|
||||
*/
|
||||
export async function resolveCodexRolloutResume(
|
||||
filePath: string,
|
||||
previous: CodexUsagePersistedFile | undefined
|
||||
previous: CodexUsagePersistedFile | undefined,
|
||||
reader?: JsonlFileReader
|
||||
): Promise<CodexUsageParseResumeState | null> {
|
||||
const resume = previous?.parseResumeState
|
||||
if (!isUsableResumeState(resume)) {
|
||||
if (!resume || typeof resume.sessionId !== 'string') {
|
||||
return null
|
||||
}
|
||||
const physicalFileId = await readPhysicalFileId(filePath)
|
||||
if (
|
||||
resume.physicalFileId !== null &&
|
||||
physicalFileId !== null &&
|
||||
resume.physicalFileId !== physicalFileId
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const digests = await readPrefixDigests(filePath, resume.parsedBytes)
|
||||
if (digests === null) {
|
||||
return null
|
||||
}
|
||||
return digests.boundaryDigest === resume.boundaryDigest &&
|
||||
digests.headDigest === resume.headDigest
|
||||
? resume
|
||||
: null
|
||||
return (await resolveJsonlFileCheckpoint(filePath, resume, reader)) === null ? null : resume
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import { getOrcaManagedCodexHomePath, getSystemCodexHomePath } from '../codex/co
|
||||
import { getCodexAccountHomeSessionDirectories } from '../codex/codex-account-home-discovery'
|
||||
import { getLegacyCopiedCodexSessionBridgeScanPreference } from '../codex/codex-session-bridge'
|
||||
import { normalizeFsPath } from '../usage/usage-path-comparison'
|
||||
import { isDefinitiveAbsence } from '../../shared/definitive-filesystem-absence'
|
||||
|
||||
const YIELD_EVERY_DISCOVERY_ENTRIES = 100
|
||||
|
||||
@@ -25,7 +26,12 @@ async function walkJsonlFiles(
|
||||
dirPath: string,
|
||||
progress: { entriesVisited: number } = { entriesVisited: 0 }
|
||||
): Promise<string[]> {
|
||||
const entries = await readdir(dirPath, { withFileTypes: true })
|
||||
const entries = await readdir(dirPath, { withFileTypes: true }).catch((error: unknown) => {
|
||||
if (isDefinitiveAbsence(error)) {
|
||||
return []
|
||||
}
|
||||
throw error
|
||||
})
|
||||
const files: string[] = []
|
||||
|
||||
for (const entry of entries) {
|
||||
@@ -79,11 +85,7 @@ function hasLegacyCopiedSessionBridgeMarkers(): boolean {
|
||||
export async function listCodexSessionFiles(): Promise<string[]> {
|
||||
const files: string[] = []
|
||||
for (const dirPath of getCodexSessionDirectories()) {
|
||||
try {
|
||||
appendDiscoveredFiles(files, await walkJsonlFiles(dirPath))
|
||||
} catch {
|
||||
// Missing or unreadable history in one home should not hide the other.
|
||||
}
|
||||
appendDiscoveredFiles(files, await walkJsonlFiles(dirPath))
|
||||
}
|
||||
return dedupeCodexSessionFileAliases(files, hasLegacyCopiedSessionBridgeMarkers())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
appendFileSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync
|
||||
} from 'node:fs'
|
||||
import type * as NodeFsPromises from 'node:fs/promises'
|
||||
import type * as NodeOs from 'node:os'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join } from 'node:path'
|
||||
|
||||
const { homedirMock, getPathMock, parseReads, handles, mutation } = vi.hoisted(() => {
|
||||
const parseReads: { path: string; start: number; bytes: number }[] = []
|
||||
const mutation: {
|
||||
beforeParse: ((path: string) => void) | null
|
||||
duringParse: ((path: string) => void) | null
|
||||
endBeforeEOF: number | null
|
||||
zeroDescriptorRead: boolean
|
||||
} = {
|
||||
beforeParse: null,
|
||||
duringParse: null,
|
||||
endBeforeEOF: null,
|
||||
zeroDescriptorRead: false
|
||||
}
|
||||
return {
|
||||
homedirMock: vi.fn<() => string>(),
|
||||
getPathMock: vi.fn<(name: string) => string>(),
|
||||
parseReads,
|
||||
handles: { opened: 0, closed: 0 },
|
||||
mutation
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('electron', () => ({ app: { getPath: getPathMock } }))
|
||||
vi.mock('node:os', async () => ({
|
||||
...(await vi.importActual<typeof NodeOs>('node:os')),
|
||||
homedir: homedirMock
|
||||
}))
|
||||
|
||||
vi.mock('node:fs/promises', async () => {
|
||||
const actual = await vi.importActual<typeof NodeFsPromises>('node:fs/promises')
|
||||
return {
|
||||
...actual,
|
||||
open: async (...args: Parameters<typeof actual.open>) => {
|
||||
const handle = await actual.open(...args)
|
||||
handles.opened++
|
||||
const createStream = handle.createReadStream.bind(handle)
|
||||
const readDescriptor = handle.read.bind(handle)
|
||||
const close = handle.close.bind(handle)
|
||||
vi.spyOn(handle, 'read').mockImplementation(async (...readArgs) => {
|
||||
const result = await readDescriptor(...readArgs)
|
||||
if (mutation.zeroDescriptorRead) {
|
||||
mutation.zeroDescriptorRead = false
|
||||
return { ...result, bytesRead: 0 }
|
||||
}
|
||||
return result
|
||||
})
|
||||
let closed = false
|
||||
vi.spyOn(handle, 'close').mockImplementation(async () => {
|
||||
await close()
|
||||
if (!closed) {
|
||||
closed = true
|
||||
handles.closed++
|
||||
}
|
||||
})
|
||||
vi.spyOn(handle, 'createReadStream').mockImplementation((options) => {
|
||||
const start = options?.start ?? 0
|
||||
const end = options?.end
|
||||
const parseRead = end !== undefined && end - start + 1 !== 4096
|
||||
const filePath = String(args[0])
|
||||
if (parseRead) {
|
||||
mutation.beforeParse?.(filePath)
|
||||
}
|
||||
const forcedEnd = parseRead ? mutation.endBeforeEOF : null
|
||||
if (forcedEnd !== null) {
|
||||
mutation.endBeforeEOF = null
|
||||
}
|
||||
const stream = createStream(forcedEnd === null ? options : { ...options, end: forcedEnd })
|
||||
if (parseRead) {
|
||||
const read = { path: filePath, start, bytes: 0 }
|
||||
parseReads.push(read)
|
||||
stream.on('data', (chunk: string | Buffer) => {
|
||||
read.bytes += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length
|
||||
})
|
||||
stream.once('data', () => mutation.duringParse?.(filePath))
|
||||
}
|
||||
return stream
|
||||
})
|
||||
return handle
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
import { parseCodexUsageFile } from './codex-rollout-file-parse'
|
||||
import { scanCodexUsageFiles } from './scanner'
|
||||
|
||||
let root: string
|
||||
let rolloutPath: string
|
||||
|
||||
beforeEach(() => {
|
||||
root = mkdtempSync(join(tmpdir(), 'orca-codex-checkpoint-races-'))
|
||||
homedirMock.mockReturnValue(root)
|
||||
getPathMock.mockReturnValue(join(root, 'user-data'))
|
||||
vi.stubEnv('ORCA_USER_DATA_PATH', join(root, 'user-data'))
|
||||
vi.stubEnv('XDG_CONFIG_HOME', join(root, '.config'))
|
||||
vi.stubEnv('CODEX_HOME', join(root, '.codex'))
|
||||
const sessions = join(root, '.codex', 'sessions')
|
||||
mkdirSync(sessions, { recursive: true })
|
||||
rolloutPath = join(sessions, 'rollout.jsonl')
|
||||
parseReads.length = 0
|
||||
handles.opened = 0
|
||||
handles.closed = 0
|
||||
mutation.beforeParse = null
|
||||
mutation.duringParse = null
|
||||
mutation.endBeforeEOF = null
|
||||
mutation.zeroDescriptorRead = false
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
vi.unstubAllEnvs()
|
||||
expect(handles.closed).toBe(handles.opened)
|
||||
})
|
||||
|
||||
function record(index: number, baseline: number): string {
|
||||
return `${JSON.stringify({
|
||||
timestamp: `2026-05-26T12:${String(index % 60).padStart(2, '0')}:00.000Z`,
|
||||
type: 'event_msg',
|
||||
payload: {
|
||||
type: 'token_count',
|
||||
info: {
|
||||
model: 'gpt-5-codex',
|
||||
last_token_usage: { input_tokens: 1, total_tokens: 1 },
|
||||
total_token_usage: {
|
||||
input_tokens: baseline + index + 1,
|
||||
total_tokens: baseline + index + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
})}\n`
|
||||
}
|
||||
|
||||
function records(from: number, to: number, baseline = 1000): string {
|
||||
return Array.from({ length: to - from }, (_, index) => record(from + index, baseline)).join('')
|
||||
}
|
||||
|
||||
function transcript(sessionId: string, count: number, baseline = 1000): string {
|
||||
const metadata = `${JSON.stringify({ type: 'session_meta', payload: { id: sessionId, cwd: root } })}\n`
|
||||
const padding = `${JSON.stringify({ type: 'tool-result', content: 'x'.repeat(16_384) })}\n`
|
||||
return metadata + padding + records(0, count, baseline)
|
||||
}
|
||||
|
||||
function offsets(): number[] {
|
||||
return parseReads.filter((read) => read.path === rolloutPath).map((read) => read.start)
|
||||
}
|
||||
|
||||
describe('Codex checkpoint file races', () => {
|
||||
it.each(['rewrite', 'rotate'])(
|
||||
'discards a %s after verification before reading the suffix',
|
||||
async (kind) => {
|
||||
writeFileSync(rolloutPath, transcript('session-old', 40))
|
||||
const first = await scanCodexUsageFiles([], [])
|
||||
const offset = first.processedFiles[0]?.parseResumeState?.parsedBytes ?? 0
|
||||
expect(offset).toBeGreaterThan(12_288)
|
||||
appendFileSync(rolloutPath, records(40, 42))
|
||||
const replacement = transcript('session-new', 60, 2000)
|
||||
mutation.beforeParse = (filePath) => {
|
||||
mutation.beforeParse = null
|
||||
if (kind === 'rotate') {
|
||||
const replacementPath = join(root, 'replacement.jsonl')
|
||||
writeFileSync(replacementPath, replacement)
|
||||
renameSync(replacementPath, filePath)
|
||||
} else {
|
||||
writeFileSync(filePath, replacement)
|
||||
}
|
||||
}
|
||||
parseReads.length = 0
|
||||
|
||||
const second = await scanCodexUsageFiles([], first.processedFiles)
|
||||
expect(mutation.beforeParse).toBeNull()
|
||||
expect(offsets()).toEqual([offset, 0])
|
||||
expect(parseReads.every((read) => read.bytes > 0)).toBe(true)
|
||||
const cold = await scanCodexUsageFiles([], [])
|
||||
expect(second.sessions).toEqual(cold.sessions)
|
||||
expect(second.dailyAggregates).toEqual(cold.dailyAggregates)
|
||||
expect(second.sessions[0]?.sessionId).toBe('session-new')
|
||||
const third = await scanCodexUsageFiles([], second.processedFiles)
|
||||
expect(third.processedFiles[0]).toBe(second.processedFiles[0])
|
||||
expect(third.sessions).toEqual(cold.sessions)
|
||||
}
|
||||
)
|
||||
|
||||
it('discards cold-read events before claiming their keys when the file changes during parsing', async () => {
|
||||
writeFileSync(rolloutPath, transcript('session-old', 40))
|
||||
mutation.duringParse = (filePath) => {
|
||||
mutation.duringParse = null
|
||||
writeFileSync(filePath, transcript('session-new', 60, 2000))
|
||||
}
|
||||
const commitEventKey = vi.fn()
|
||||
|
||||
const parsed = await parseCodexUsageFile(rolloutPath, () => null, { commitEventKey })
|
||||
expect(offsets()).toEqual([0, 0])
|
||||
expect(parsed.sessions[0]?.sessionId).toBe('session-new')
|
||||
expect(parsed.sessions[0]?.eventCount).toBe(60)
|
||||
expect(commitEventKey).toHaveBeenCalledTimes(60)
|
||||
const cold = await parseCodexUsageFile(rolloutPath, () => null)
|
||||
expect(parsed.ownedEventKeys).toEqual(cold.ownedEventKeys)
|
||||
expect(parsed.sessions).toEqual(cold.sessions)
|
||||
})
|
||||
|
||||
it('leaves an append after the captured EOF for the next scan', async () => {
|
||||
writeFileSync(rolloutPath, transcript('session-stable', 40))
|
||||
const capturedSize = statSync(rolloutPath).size
|
||||
mutation.duringParse = (filePath) => {
|
||||
mutation.duringParse = null
|
||||
appendFileSync(filePath, records(40, 41))
|
||||
}
|
||||
|
||||
const first = await scanCodexUsageFiles([], [])
|
||||
expect(first.sessions[0]?.eventCount).toBe(40)
|
||||
expect(first.processedFiles[0]?.size).toBe(capturedSize)
|
||||
expect(first.processedFiles[0]?.parseResumeState?.parsedBytes).toBe(capturedSize)
|
||||
const second = await scanCodexUsageFiles([], first.processedFiles)
|
||||
expect(second.sessions[0]?.eventCount).toBe(41)
|
||||
const cold = await scanCodexUsageFiles([], [])
|
||||
expect(second.sessions).toEqual(cold.sessions)
|
||||
expect(second.dailyAggregates).toEqual(cold.dailyAggregates)
|
||||
})
|
||||
|
||||
it('releases old owner claims and lets an unchanged deferred fork reclaim them', async () => {
|
||||
const prefix = transcript('session-old', 40)
|
||||
const forkPath = join(dirname(rolloutPath), 'zzzz-fork.jsonl')
|
||||
writeFileSync(rolloutPath, prefix)
|
||||
writeFileSync(forkPath, prefix + records(40, 41))
|
||||
const first = await scanCodexUsageFiles([], [])
|
||||
const fork = first.processedFiles.find((file) => file.path === forkPath)
|
||||
expect(fork?.hasDeferredClaims).toBe(true)
|
||||
expect(fork?.ownedEventKeys).toHaveLength(1)
|
||||
appendFileSync(rolloutPath, records(40, 42))
|
||||
mutation.beforeParse = (filePath) => {
|
||||
if (filePath === rolloutPath) {
|
||||
mutation.beforeParse = null
|
||||
writeFileSync(filePath, transcript('session-new', 60, 2000))
|
||||
}
|
||||
}
|
||||
parseReads.length = 0
|
||||
|
||||
const second = await scanCodexUsageFiles([], first.processedFiles)
|
||||
expect(parseReads.filter((read) => read.path === forkPath).map((read) => read.start)).toEqual([
|
||||
0
|
||||
])
|
||||
expect(
|
||||
second.processedFiles.find((file) => file.path === forkPath)?.ownedEventKeys
|
||||
).toHaveLength(41)
|
||||
const cold = await scanCodexUsageFiles([], [])
|
||||
expect(second.sessions).toEqual(cold.sessions)
|
||||
expect(second.dailyAggregates).toEqual(cold.dailyAggregates)
|
||||
const third = await scanCodexUsageFiles([], second.processedFiles)
|
||||
expect(third.sessions).toEqual(cold.sessions)
|
||||
})
|
||||
|
||||
it('keeps an unchanged deferred fork cached when a short owner merely appends', async () => {
|
||||
const prefix = records(0, 3)
|
||||
const forkPath = join(dirname(rolloutPath), 'zzzz-fork.jsonl')
|
||||
writeFileSync(rolloutPath, prefix)
|
||||
writeFileSync(forkPath, prefix + records(3, 4))
|
||||
const first = await scanCodexUsageFiles([], [])
|
||||
const fork = first.processedFiles.find((file) => file.path === forkPath)
|
||||
expect(fork?.hasDeferredClaims).toBe(true)
|
||||
expect(first.processedFiles[0]?.parseResumeState).toBeNull()
|
||||
appendFileSync(rolloutPath, records(4, 5))
|
||||
parseReads.length = 0
|
||||
|
||||
const second = await scanCodexUsageFiles([], first.processedFiles)
|
||||
expect(second.processedFiles.find((file) => file.path === forkPath)).toBe(fork)
|
||||
expect(parseReads.some((read) => read.path === forkPath)).toBe(false)
|
||||
expect(second.dailyAggregates.reduce((sum, row) => sum + row.totalTokens, 0)).toBe(5)
|
||||
})
|
||||
|
||||
it('rejects an unchanged file stream that stops early before claiming partial history', async () => {
|
||||
writeFileSync(rolloutPath, transcript('session-stable', 40))
|
||||
mutation.endBeforeEOF = Buffer.byteLength(transcript('session-stable', 10)) - 1
|
||||
const commitEventKey = vi.fn()
|
||||
|
||||
await expect(parseCodexUsageFile(rolloutPath, () => null, { commitEventKey })).rejects.toThrow(
|
||||
'JSONL snapshot stream ended before the unchanged file boundary.'
|
||||
)
|
||||
expect(mutation.endBeforeEOF).toBeNull()
|
||||
expect(offsets()).toEqual([0])
|
||||
expect(commitEventKey).not.toHaveBeenCalled()
|
||||
const cold = await scanCodexUsageFiles([], [])
|
||||
expect(cold.sessions[0]?.eventCount).toBe(40)
|
||||
})
|
||||
|
||||
it('leaves the previous scan intact when an unchanged suffix stream stops early', async () => {
|
||||
const prefix = transcript('session-stable', 40)
|
||||
writeFileSync(rolloutPath, prefix)
|
||||
const previous = await scanCodexUsageFiles([], [])
|
||||
appendFileSync(rolloutPath, records(40, 42))
|
||||
mutation.endBeforeEOF = Buffer.byteLength(prefix + records(40, 41)) - 1
|
||||
parseReads.length = 0
|
||||
|
||||
await expect(scanCodexUsageFiles([], previous.processedFiles)).rejects.toThrow(
|
||||
'JSONL snapshot stream ended before the unchanged file boundary.'
|
||||
)
|
||||
expect(mutation.endBeforeEOF).toBeNull()
|
||||
expect(offsets()).toEqual([Buffer.byteLength(prefix)])
|
||||
expect(previous.sessions[0]?.eventCount).toBe(40)
|
||||
expect(previous.processedFiles[0]?.size).toBe(Buffer.byteLength(prefix))
|
||||
const recovered = await scanCodexUsageFiles([], previous.processedFiles)
|
||||
expect(recovered.sessions[0]?.eventCount).toBe(42)
|
||||
})
|
||||
|
||||
it('rejects a stable zero-byte read of the skipped prefix in a short legacy source', async () => {
|
||||
writeFileSync(rolloutPath, records(0, 3))
|
||||
mutation.zeroDescriptorRead = true
|
||||
const commitEventKey = vi.fn()
|
||||
const legacySourceSkipBytes = Buffer.byteLength(records(0, 1))
|
||||
|
||||
await expect(
|
||||
parseCodexUsageFile(rolloutPath, () => null, { legacySourceSkipBytes, commitEventKey })
|
||||
).rejects.toThrow('JSONL snapshot read ended before the unchanged file boundary.')
|
||||
expect(mutation.zeroDescriptorRead).toBe(false)
|
||||
expect(offsets()).toEqual([])
|
||||
expect(commitEventKey).not.toHaveBeenCalled()
|
||||
const recovered = await parseCodexUsageFile(rolloutPath, () => null, { legacySourceSkipBytes })
|
||||
expect(recovered.sessions[0]?.eventCount).toBe(2)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,192 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
utimesSync,
|
||||
writeFileSync
|
||||
} from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import type * as JsonlLineOffsets from '../usage/jsonl-line-offsets'
|
||||
|
||||
const fixture = vi.hoisted(() => {
|
||||
const parseOffsets: number[] = []
|
||||
return { root: '', parseOffsets }
|
||||
})
|
||||
|
||||
vi.mock('../codex/codex-home-paths', () => ({
|
||||
getOrcaManagedCodexHomePath: () => join(fixture.root, 'runtime'),
|
||||
getSystemCodexHomePath: () => join(fixture.root, 'system')
|
||||
}))
|
||||
|
||||
vi.mock('../codex/codex-account-home-discovery', () => ({
|
||||
getCodexAccountHomeSessionDirectories: () => []
|
||||
}))
|
||||
|
||||
vi.mock('../codex/codex-session-bridge', () => ({
|
||||
getLegacyCopiedCodexSessionBridgeScanPreference: () => null
|
||||
}))
|
||||
|
||||
vi.mock('../usage/jsonl-line-offsets', async () => {
|
||||
const actual = await vi.importActual<typeof JsonlLineOffsets>('../usage/jsonl-line-offsets')
|
||||
return {
|
||||
...actual,
|
||||
readJsonlLinesFromOffset: async function* (
|
||||
...args: Parameters<typeof actual.readJsonlLinesFromOffset>
|
||||
) {
|
||||
fixture.parseOffsets.push(args[1])
|
||||
yield* actual.readJsonlLinesFromOffset(...args)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
import { scanCodexUsageFiles } from './scanner'
|
||||
|
||||
const preservedMtime = new Date('2026-05-26T12:00:00.000Z')
|
||||
let rolloutPath: string
|
||||
|
||||
beforeEach(() => {
|
||||
fixture.root = mkdtempSync(join(tmpdir(), 'orca-codex-file-generation-'))
|
||||
const sessions = join(fixture.root, 'runtime', 'sessions')
|
||||
mkdirSync(sessions, { recursive: true })
|
||||
rolloutPath = join(sessions, 'rollout.jsonl')
|
||||
fixture.parseOffsets.length = 0
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(fixture.root, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
function transcript(sessionId: string, count: number, changedIndex = -1): string {
|
||||
const metadata = JSON.stringify({
|
||||
type: 'session_meta',
|
||||
payload: { id: sessionId, cwd: fixture.root }
|
||||
})
|
||||
const records = Array.from({ length: count }, (_, index) => {
|
||||
const tokens = index === changedIndex ? 20 : 10
|
||||
return JSON.stringify({
|
||||
timestamp: new Date(preservedMtime.getTime() + index * 60_000).toISOString(),
|
||||
type: 'event_msg',
|
||||
payload: {
|
||||
type: 'token_count',
|
||||
info: {
|
||||
model: 'gpt-5-codex',
|
||||
last_token_usage: { input_tokens: tokens, total_tokens: tokens }
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
return `${[metadata, ...records].join('\n')}\n`
|
||||
}
|
||||
|
||||
function writePreservingMtime(filePath: string, content: string): void {
|
||||
writeFileSync(filePath, content)
|
||||
utimesSync(filePath, preservedMtime, preservedMtime)
|
||||
}
|
||||
|
||||
function totalTokens(rows: { totalTokens: number }[]): number {
|
||||
return rows.reduce((sum, row) => sum + row.totalTokens, 0)
|
||||
}
|
||||
|
||||
describe('Codex cached rollout generation', () => {
|
||||
it.each([1, 80])(
|
||||
'reparses a same-size preserved-mtime atomic replacement containing %i events',
|
||||
async (count) => {
|
||||
const original = transcript('session-old', count)
|
||||
const replacement = transcript('session-new', count, Math.floor(count / 2))
|
||||
expect(Buffer.byteLength(replacement)).toBe(Buffer.byteLength(original))
|
||||
writePreservingMtime(rolloutPath, original)
|
||||
const before = statSync(rolloutPath, { bigint: true })
|
||||
const first = await scanCodexUsageFiles([], [])
|
||||
expect(totalTokens(first.dailyAggregates)).toBe(count * 10)
|
||||
expect(Boolean(first.processedFiles[0]?.parseResumeState)).toBe(count > 1)
|
||||
|
||||
const replacementPath = join(fixture.root, 'replacement.jsonl')
|
||||
writePreservingMtime(replacementPath, replacement)
|
||||
renameSync(replacementPath, rolloutPath)
|
||||
const after = statSync(rolloutPath, { bigint: true })
|
||||
expect(after.size).toBe(before.size)
|
||||
expect(after.mtimeNs).toBe(before.mtimeNs)
|
||||
if (before.ino !== 0n && after.ino !== 0n) {
|
||||
expect(`${after.dev}:${after.ino}`).not.toBe(`${before.dev}:${before.ino}`)
|
||||
}
|
||||
fixture.parseOffsets.length = 0
|
||||
|
||||
const second = await scanCodexUsageFiles([], first.processedFiles)
|
||||
expect(totalTokens(second.dailyAggregates)).toBe(count * 10 + 10)
|
||||
expect(second.sessions[0]?.sessionId).toBe('session-new')
|
||||
expect(fixture.parseOffsets).toEqual([0])
|
||||
const cold = await scanCodexUsageFiles([], [])
|
||||
expect(second.sessions).toEqual(cold.sessions)
|
||||
expect(second.dailyAggregates).toEqual(cold.dailyAggregates)
|
||||
fixture.parseOffsets.length = 0
|
||||
const third = await scanCodexUsageFiles([], second.processedFiles)
|
||||
expect(third.processedFiles[0]).toBe(second.processedFiles[0])
|
||||
expect(fixture.parseOffsets).toEqual([])
|
||||
}
|
||||
)
|
||||
|
||||
it('cold-reparses a same-inode middle rewrite when size and mtime are preserved', async () => {
|
||||
const original = transcript('session-stable', 80)
|
||||
const replacement = transcript('session-stable', 80, 40)
|
||||
expect(Buffer.byteLength(replacement)).toBe(Buffer.byteLength(original))
|
||||
expect(replacement.slice(0, 4096)).toBe(original.slice(0, 4096))
|
||||
expect(replacement.slice(-4096)).toBe(original.slice(-4096))
|
||||
writePreservingMtime(rolloutPath, original)
|
||||
const before = statSync(rolloutPath)
|
||||
const first = await scanCodexUsageFiles([], [])
|
||||
expect(totalTokens(first.dailyAggregates)).toBe(800)
|
||||
expect(first.processedFiles[0]?.parseResumeState?.parsedBytes).toBe(before.size)
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 20))
|
||||
writePreservingMtime(rolloutPath, replacement)
|
||||
const after = statSync(rolloutPath)
|
||||
expect(after.ino).toBe(before.ino)
|
||||
expect(after.dev).toBe(before.dev)
|
||||
expect(after.size).toBe(before.size)
|
||||
expect(after.mtimeMs).toBe(before.mtimeMs)
|
||||
expect(after.ctimeMs).not.toBe(before.ctimeMs)
|
||||
expect(readFileSync(rolloutPath, 'utf8')).toBe(replacement)
|
||||
fixture.parseOffsets.length = 0
|
||||
|
||||
const second = await scanCodexUsageFiles([], first.processedFiles)
|
||||
expect(totalTokens(second.dailyAggregates)).toBe(810)
|
||||
expect(fixture.parseOffsets).toEqual([0])
|
||||
expect(second.processedFiles[0]?.ctimeMs).toBe(after.ctimeMs)
|
||||
const cold = await scanCodexUsageFiles([], [])
|
||||
expect(second.sessions).toEqual(cold.sessions)
|
||||
expect(second.dailyAggregates).toEqual(cold.dailyAggregates)
|
||||
})
|
||||
|
||||
it('cold-reparses legacy metadata once even when the old checkpoint remains valid', async () => {
|
||||
writePreservingMtime(rolloutPath, transcript('session-legacy', 80))
|
||||
const first = await scanCodexUsageFiles([], [])
|
||||
expect(first.processedFiles[0]?.parseResumeState?.parsedBytes).toBeGreaterThan(12_288)
|
||||
const legacy = first.processedFiles.map((file) => {
|
||||
const oldFile = { ...file }
|
||||
delete oldFile.physicalFileId
|
||||
delete oldFile.ctimeMs
|
||||
return oldFile
|
||||
})
|
||||
fixture.parseOffsets.length = 0
|
||||
|
||||
const second = await scanCodexUsageFiles([], legacy)
|
||||
expect(fixture.parseOffsets).toEqual([0])
|
||||
expect(second.sessions).toEqual(first.sessions)
|
||||
expect(second.dailyAggregates).toEqual(first.dailyAggregates)
|
||||
const stats = statSync(rolloutPath)
|
||||
expect(second.processedFiles[0]?.physicalFileId).toBe(
|
||||
stats.ino === 0 ? null : `${stats.dev}:${stats.ino}`
|
||||
)
|
||||
expect(second.processedFiles[0]?.ctimeMs).toBe(stats.ctimeMs)
|
||||
fixture.parseOffsets.length = 0
|
||||
|
||||
const third = await scanCodexUsageFiles([], second.processedFiles)
|
||||
expect(third.processedFiles[0]).toBe(second.processedFiles[0])
|
||||
expect(fixture.parseOffsets).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -31,6 +31,22 @@ vi.mock('node:fs/promises', async () => {
|
||||
? Object.assign(stats, { dev: Number(identity.dev), ino: Number(identity.ino) })
|
||||
: stats
|
||||
},
|
||||
open: async (...args: Parameters<typeof actual.open>) => {
|
||||
const handle = await actual.open(...args)
|
||||
const identity = fixture.identities.get(String(args[0]))
|
||||
const stat = handle.stat.bind(handle)
|
||||
vi.spyOn(handle, 'stat').mockImplementation(async (options) => {
|
||||
if (options?.bigint) {
|
||||
const stats = await stat({ bigint: true })
|
||||
return identity ? Object.assign(stats, identity) : stats
|
||||
}
|
||||
const stats = await stat()
|
||||
return identity
|
||||
? Object.assign(stats, { dev: Number(identity.dev), ino: Number(identity.ino) })
|
||||
: stats
|
||||
})
|
||||
return handle
|
||||
},
|
||||
realpath: async (filePath: PathLike) => {
|
||||
if (fixture.realpathErrors.has(String(filePath))) {
|
||||
throw new Error('Fixture realpath unavailable')
|
||||
|
||||
@@ -2,19 +2,22 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { tmpdir } from 'node:os'
|
||||
import type * as NodeOs from 'node:os'
|
||||
import type * as NodeFs from 'node:fs'
|
||||
import type * as NodeFsPromises from 'node:fs/promises'
|
||||
import type * as JsonlLineOffsets from '../usage/jsonl-line-offsets'
|
||||
import { join } from 'node:path'
|
||||
|
||||
const { getPathMock, homedirMock, streamReads, onStreamOpen } = vi.hoisted(() => {
|
||||
const streamReads: { path: string; bytes: number; start: number; bounded: boolean }[] = []
|
||||
const { getPathMock, homedirMock, streamReads, onStreamOpen, parsingPaths } = vi.hoisted(() => {
|
||||
const streamReads: { path: string; bytes: number; start: number; digest: boolean }[] = []
|
||||
// Seam for mutating the tree mid-scan, between two files' parse reads.
|
||||
const onStreamOpen: { current: ((path: string, bounded: boolean) => void) | null } = {
|
||||
const onStreamOpen: { current: ((path: string, digest: boolean) => void) | null } = {
|
||||
current: null
|
||||
}
|
||||
return {
|
||||
getPathMock: vi.fn<(name: string) => string>(),
|
||||
homedirMock: vi.fn<() => string>(),
|
||||
streamReads,
|
||||
onStreamOpen
|
||||
onStreamOpen,
|
||||
parsingPaths: new Set<string>()
|
||||
}
|
||||
})
|
||||
|
||||
@@ -44,16 +47,84 @@ vi.mock('node:fs', async () => {
|
||||
const filePath = String(path)
|
||||
const range = typeof options === 'object' && options !== null ? options : {}
|
||||
const start = range.start ?? 0
|
||||
const size = actual.statSync(filePath).size
|
||||
const stop = range.end === undefined ? size : Math.min(size, range.end + 1)
|
||||
streamReads.push({
|
||||
path: filePath,
|
||||
bytes: Math.max(0, stop - start),
|
||||
start,
|
||||
bounded: range.end !== undefined
|
||||
const read = { path: filePath, bytes: 0, start, digest: !parsingPaths.has(filePath) }
|
||||
streamReads.push(read)
|
||||
onStreamOpen.current?.(filePath, read.digest)
|
||||
const stream = actual.createReadStream(path, options)
|
||||
stream.on('data', (chunk: string | Buffer) => {
|
||||
read.bytes += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length
|
||||
})
|
||||
onStreamOpen.current?.(filePath, range.end !== undefined)
|
||||
return actual.createReadStream(path, options)
|
||||
return stream
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('node:fs/promises', async () => {
|
||||
const actual = await vi.importActual<typeof NodeFsPromises>('node:fs/promises')
|
||||
return {
|
||||
...actual,
|
||||
open: async (...args: Parameters<typeof actual.open>) => {
|
||||
const handle = await actual.open(...args)
|
||||
const createStream = handle.createReadStream.bind(handle)
|
||||
const readWindow = handle.read.bind(handle)
|
||||
let streamActive = false
|
||||
vi.spyOn(handle, 'read').mockImplementation(async (...readArgs) => {
|
||||
if (streamActive) {
|
||||
return readWindow(...readArgs)
|
||||
}
|
||||
const filePath = String(args[0])
|
||||
const digest = !parsingPaths.has(filePath)
|
||||
onStreamOpen.current?.(filePath, digest)
|
||||
const result = await readWindow(...readArgs)
|
||||
const position = readArgs.at(-1)
|
||||
streamReads.push({
|
||||
path: filePath,
|
||||
bytes: result.bytesRead,
|
||||
start: typeof position === 'number' ? position : 0,
|
||||
digest
|
||||
})
|
||||
return result
|
||||
})
|
||||
vi.spyOn(handle, 'createReadStream').mockImplementation((options) => {
|
||||
const filePath = String(args[0])
|
||||
const read = {
|
||||
path: filePath,
|
||||
bytes: 0,
|
||||
start: options?.start ?? 0,
|
||||
digest: !parsingPaths.has(filePath)
|
||||
}
|
||||
streamReads.push(read)
|
||||
onStreamOpen.current?.(filePath, read.digest)
|
||||
streamActive = true
|
||||
const stream = createStream(options)
|
||||
const finish = (): void => {
|
||||
streamActive = false
|
||||
}
|
||||
stream.once('end', finish)
|
||||
stream.once('close', finish)
|
||||
stream.on('data', (chunk: string | Buffer) => {
|
||||
read.bytes += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length
|
||||
})
|
||||
return stream
|
||||
})
|
||||
return handle
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('../usage/jsonl-line-offsets', async () => {
|
||||
const actual = await vi.importActual<typeof JsonlLineOffsets>('../usage/jsonl-line-offsets')
|
||||
return {
|
||||
...actual,
|
||||
readJsonlLinesFromOffset: async function* (
|
||||
...args: Parameters<typeof actual.readJsonlLinesFromOffset>
|
||||
) {
|
||||
parsingPaths.add(args[0])
|
||||
try {
|
||||
yield* actual.readJsonlLinesFromOffset(...args)
|
||||
} finally {
|
||||
parsingPaths.delete(args[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -62,7 +133,7 @@ import { mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync, appendFileSync
|
||||
import { scanCodexUsageFiles } from './scanner'
|
||||
import type { CodexUsagePersistedFile } from './types'
|
||||
|
||||
/** Mirrors BOUNDARY_WINDOW_BYTES in codex-rollout-resume-state.ts. */
|
||||
/** Mirrors the checkpoint window in jsonl-file-checkpoint.ts. */
|
||||
const BOUNDARY_WINDOW_BYTES = 4096
|
||||
|
||||
/** Enough records (~377 B each) to put a prefix past MIN_RESUMABLE_PREFIX_BYTES.
|
||||
@@ -148,12 +219,10 @@ function bytesReadFor(filePath: string): number {
|
||||
.reduce((total, entry) => total + entry.bytes, 0)
|
||||
}
|
||||
|
||||
/** Offsets at which this file's parse reads opened, in order. Bounded reads are
|
||||
* digest windows; an unbounded one at 0 is a full reparse and at the recorded
|
||||
* offset is a resume, so this says exactly which path a scan took. */
|
||||
/** Digest and parse streams both have end bounds; the iterator marks parse reads. */
|
||||
function parseReadOffsets(filePath: string): number[] {
|
||||
return streamReads
|
||||
.filter((entry) => entry.path === filePath && !entry.bounded)
|
||||
.filter((entry) => entry.path === filePath && !entry.digest)
|
||||
.map((entry) => entry.start)
|
||||
}
|
||||
|
||||
@@ -195,6 +264,7 @@ beforeEach(() => {
|
||||
mkdirSync(sessionsDir, { recursive: true })
|
||||
streamReads.length = 0
|
||||
onStreamOpen.current = null
|
||||
parsingPaths.clear()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
@@ -231,14 +301,13 @@ describe('scanCodexUsageFiles incremental append', () => {
|
||||
expect(totalTokens(second.dailyAggregates)).toBe(202)
|
||||
expect(second.sessions[0]?.eventCount).toBe(202)
|
||||
|
||||
// The defect: the scanner restarts at byte 0 and re-reads the whole file.
|
||||
// The fix reads the appended bytes plus five bounded windows: a head and a
|
||||
// boundary window to plan the resume, both again at the point of use so a
|
||||
// rollout replaced in between cannot be stitched onto, then the moved
|
||||
// boundary to record the new resume point.
|
||||
// Eleven windows prove the planned prefix, opened snapshot and final checkpoint.
|
||||
expect(reparsedFromStart(rolloutPath)).toBe(false)
|
||||
expect(bytesReadFor(rolloutPath)).toBeLessThan(sizeBeforeAppend)
|
||||
expect(bytesReadFor(rolloutPath)).toBeLessThanOrEqual(appendedBytes + 5 * BOUNDARY_WINDOW_BYTES)
|
||||
expect(bytesReadFor(rolloutPath)).toBeGreaterThan(appendedBytes)
|
||||
expect(bytesReadFor(rolloutPath)).toBeLessThanOrEqual(
|
||||
appendedBytes + 11 * BOUNDARY_WINDOW_BYTES
|
||||
)
|
||||
})
|
||||
|
||||
it('carries cumulative token totals across the resume boundary', async () => {
|
||||
@@ -379,10 +448,8 @@ describe('scanCodexUsageFiles incremental append', () => {
|
||||
|
||||
appendFileSync(rolloutPath, usageRecordRange(RESUMABLE_RECORDS, RESUMABLE_RECORDS + 2), 'utf-8')
|
||||
const truncated = `${sessionMeta('session-shrinker')}${usageRecordRange(0, 5)}`
|
||||
onStreamOpen.current = (path, bounded) => {
|
||||
// Bounded reads are the digest windows; the unbounded one is the parse
|
||||
// read, which opens after every check this scan is going to make.
|
||||
if (path === rolloutPath && !bounded) {
|
||||
onStreamOpen.current = (path, digest) => {
|
||||
if (path === rolloutPath && !digest) {
|
||||
onStreamOpen.current = null
|
||||
writeFileSync(path, truncated, 'utf-8')
|
||||
}
|
||||
@@ -429,10 +496,10 @@ describe('scanCodexUsageFiles incremental append', () => {
|
||||
const replacement = `${sessionMeta('session-other')}${usageRecordRange(200, 260)}`
|
||||
// Past the recorded offset, so every digest window still reads its full size.
|
||||
expect(Buffer.byteLength(replacement)).toBeGreaterThan(resumeOffset)
|
||||
onStreamOpen.current = (path, bounded) => {
|
||||
onStreamOpen.current = (path, digest) => {
|
||||
// Lands while the earlier-sorted rollout is being parsed, which is after
|
||||
// the scanner's discovery loop verified every file's prefix.
|
||||
if (path === driverPath && !bounded) {
|
||||
if (path === driverPath && !digest) {
|
||||
onStreamOpen.current = null
|
||||
writeFileSync(targetPath, replacement, 'utf-8')
|
||||
}
|
||||
|
||||
@@ -61,12 +61,19 @@ export async function scanCodexUsageFiles(
|
||||
const previous = previousByPath.get(filePath)
|
||||
// When an owner disappears, only deferred-claim files need reparse.
|
||||
const mustReclaimDeferred = lostOwnerPath && previous?.hasDeferredClaims !== false
|
||||
const hasGenerationMetadata =
|
||||
previous &&
|
||||
typeof previous.ctimeMs === 'number' &&
|
||||
(typeof previous.physicalFileId === 'string' || previous.physicalFileId === null)
|
||||
const canReuse =
|
||||
!mustReclaimDeferred &&
|
||||
legacySourceSkipBytes === 0 &&
|
||||
previous &&
|
||||
hasGenerationMetadata &&
|
||||
previous.mtimeMs === fileInfo.mtimeMs &&
|
||||
previous.size === fileInfo.size &&
|
||||
previous.ctimeMs === fileInfo.ctimeMs &&
|
||||
previous.physicalFileId === fileInfo.physicalFileId &&
|
||||
Array.isArray(previous.ownedEventKeys) &&
|
||||
typeof previous.hasDeferredClaims === 'boolean'
|
||||
if (canReuse) {
|
||||
@@ -75,7 +82,13 @@ export async function scanCodexUsageFiles(
|
||||
// Why: rollouts are append-only and grow all day, so re-reading each one
|
||||
// from byte 0 dominated scans (#20940). A reclaim or a legacy suffix
|
||||
// offset still needs the whole file, so neither may resume.
|
||||
if (!mustReclaimDeferred && legacySourceSkipBytes === 0 && previous) {
|
||||
if (
|
||||
!mustReclaimDeferred &&
|
||||
legacySourceSkipBytes === 0 &&
|
||||
previous &&
|
||||
hasGenerationMetadata &&
|
||||
fileInfo.size > previous.size
|
||||
) {
|
||||
const state = await resolveCodexRolloutResume(filePath, previous)
|
||||
if (state) {
|
||||
resumeByPath.set(filePath, { state, previous })
|
||||
@@ -97,7 +110,7 @@ export async function scanCodexUsageFiles(
|
||||
// stay deterministic.
|
||||
const eventOwnerByKey = new Map<string, string>()
|
||||
for (const filePath of files) {
|
||||
const retained = reusedByPath.get(filePath) ?? resumeByPath.get(filePath)?.previous
|
||||
const retained = previousByPath.get(filePath)
|
||||
for (const eventKey of retained?.ownedEventKeys ?? []) {
|
||||
// First retained claim wins so conflicting projections stay deterministic.
|
||||
if (!eventOwnerByKey.has(eventKey)) {
|
||||
@@ -107,20 +120,37 @@ export async function scanCodexUsageFiles(
|
||||
}
|
||||
|
||||
const parsedByPath = new Map<string, CodexUsagePersistedFile>()
|
||||
for (const [index, filePath] of pathsToParse.entries()) {
|
||||
let lostOwnerKeys = lostOwnerPath
|
||||
const parseAndPublish = async (
|
||||
filePath: string,
|
||||
resume?: CodexRolloutResumePlan
|
||||
): Promise<void> => {
|
||||
const previous =
|
||||
parsedByPath.get(filePath) ?? reusedByPath.get(filePath) ?? previousByPath.get(filePath)
|
||||
const processed = await parseCodexUsageFile(filePath, resolveWorktree, {
|
||||
legacySourceSkipBytes: legacySourceSkipBytesByPath.get(filePath) ?? 0,
|
||||
resume: resumeByPath.get(filePath),
|
||||
claimEventKey: (eventKey) => {
|
||||
resume,
|
||||
canClaimEventKey: (eventKey) => {
|
||||
const owner = eventOwnerByKey.get(eventKey)
|
||||
if (owner !== undefined && owner !== filePath) {
|
||||
return false
|
||||
}
|
||||
return owner === undefined || owner === filePath
|
||||
},
|
||||
commitEventKey: (eventKey) => {
|
||||
eventOwnerByKey.set(eventKey, filePath)
|
||||
return true
|
||||
}
|
||||
})
|
||||
const currentKeys = new Set(processed.ownedEventKeys)
|
||||
for (const key of previous?.ownedEventKeys ?? []) {
|
||||
if (eventOwnerByKey.get(key) === filePath && !currentKeys.has(key)) {
|
||||
eventOwnerByKey.delete(key)
|
||||
lostOwnerKeys = true
|
||||
}
|
||||
}
|
||||
reusedByPath.delete(filePath)
|
||||
parsedByPath.set(filePath, processed)
|
||||
}
|
||||
|
||||
for (const [index, filePath] of pathsToParse.entries()) {
|
||||
await parseAndPublish(filePath, resumeByPath.get(filePath))
|
||||
|
||||
// Why: Codex session history can grow large, and scans run on the Electron
|
||||
// main process. Yield regularly so opening Settings does not stall while
|
||||
@@ -131,6 +161,21 @@ export async function scanCodexUsageFiles(
|
||||
}
|
||||
}
|
||||
|
||||
// A replaced owner can release records that an unchanged fork previously deferred.
|
||||
while (lostOwnerKeys) {
|
||||
lostOwnerKeys = false
|
||||
for (const [index, filePath] of files.entries()) {
|
||||
const current = parsedByPath.get(filePath) ?? reusedByPath.get(filePath)
|
||||
if (current?.hasDeferredClaims) {
|
||||
await parseAndPublish(filePath)
|
||||
onFilesScanned?.(1)
|
||||
}
|
||||
if ((index + 1) % YIELD_EVERY_FILES === 0) {
|
||||
await yieldToEventLoop()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const processedFiles: CodexUsagePersistedFile[] = []
|
||||
const sessionsById = new Map<string, CodexUsageSession>()
|
||||
const dailyByKey = new Map<string, CodexUsageDailyAggregate>()
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import type * as FsPromises from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
|
||||
const state = vi.hoisted(() => ({ managed: '', system: '', blocked: '', code: '' }))
|
||||
|
||||
vi.mock('../codex/codex-home-paths', () => ({
|
||||
getOrcaManagedCodexHomePath: () => state.managed,
|
||||
getSystemCodexHomePath: () => state.system
|
||||
}))
|
||||
|
||||
vi.mock('../codex/codex-account-home-discovery', () => ({
|
||||
getCodexAccountHomeSessionDirectories: () => []
|
||||
}))
|
||||
|
||||
vi.mock('../codex/codex-session-bridge', () => ({
|
||||
getLegacyCopiedCodexSessionBridgeScanPreference: () => null
|
||||
}))
|
||||
|
||||
vi.mock('node:fs/promises', async () => {
|
||||
const actual = await vi.importActual<typeof FsPromises>('node:fs/promises')
|
||||
return {
|
||||
...actual,
|
||||
readdir: async (...args: Parameters<typeof actual.readdir>) => {
|
||||
if (String(args[0]) === state.blocked) {
|
||||
throw Object.assign(new Error(`Cannot discover ${state.blocked}`), { code: state.code })
|
||||
}
|
||||
return actual.readdir(...args)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
import { listCodexSessionFiles } from './codex-session-file-discovery'
|
||||
|
||||
let root: string
|
||||
let sessions: string
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), 'orca-codex-discovery-error-'))
|
||||
state.managed = join(root, 'managed')
|
||||
state.system = join(root, 'system')
|
||||
state.blocked = ''
|
||||
state.code = ''
|
||||
sessions = join(state.managed, 'sessions')
|
||||
await mkdir(sessions, { recursive: true })
|
||||
await mkdir(join(state.system, 'sessions'), { recursive: true })
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
describe('Codex history discovery failures', () => {
|
||||
it.each(['EACCES', 'EPERM', 'EIO', 'UNKNOWN'])(
|
||||
'rejects an inaccessible root (%s)',
|
||||
async (code) => {
|
||||
state.blocked = sessions
|
||||
state.code = code
|
||||
await expect(listCodexSessionFiles()).rejects.toMatchObject({ code })
|
||||
}
|
||||
)
|
||||
|
||||
it('rejects an unreadable child instead of publishing an incomplete history list', async () => {
|
||||
await writeFile(join(sessions, 'kept.jsonl'), '')
|
||||
state.blocked = join(sessions, 'inaccessible')
|
||||
await mkdir(state.blocked)
|
||||
state.code = 'EACCES'
|
||||
await expect(listCodexSessionFiles()).rejects.toMatchObject({ code: 'EACCES' })
|
||||
})
|
||||
|
||||
it.each(['ENOENT', 'ENOTDIR'])(
|
||||
'keeps sibling files when a child disappears (%s)',
|
||||
async (code) => {
|
||||
const sibling = join(sessions, 'kept.jsonl')
|
||||
await writeFile(sibling, '')
|
||||
state.blocked = join(sessions, 'removed')
|
||||
await mkdir(state.blocked)
|
||||
state.code = code
|
||||
await expect(listCodexSessionFiles()).resolves.toEqual([sibling])
|
||||
}
|
||||
)
|
||||
|
||||
it('accepts absent homes while preserving the available history', async () => {
|
||||
const sibling = join(sessions, 'kept.jsonl')
|
||||
await writeFile(sibling, '')
|
||||
await rm(state.system, { recursive: true })
|
||||
await expect(listCodexSessionFiles()).resolves.toEqual([sibling])
|
||||
})
|
||||
})
|
||||
@@ -12,6 +12,9 @@ export type CodexUsageProcessedFile = {
|
||||
path: string
|
||||
mtimeMs: number
|
||||
size: number
|
||||
/** Absent in older caches; identifies the file that produced this projection. */
|
||||
physicalFileId?: string | null
|
||||
ctimeMs?: number
|
||||
}
|
||||
|
||||
/** Everything needed to resume parsing a grown rollout where the last scan
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
import { createReadStream } from 'node:fs'
|
||||
import { open, type FileHandle } from 'node:fs/promises'
|
||||
import { createHash } from 'node:crypto'
|
||||
import {
|
||||
readJsonlFileSnapshot,
|
||||
readJsonlHandleSnapshot,
|
||||
readJsonlSnapshotRange,
|
||||
releaseJsonlReadStream,
|
||||
sameJsonlFileSnapshot,
|
||||
verifyJsonlSnapshotContents,
|
||||
type JsonlFileSnapshot
|
||||
} from './jsonl-file-snapshot'
|
||||
|
||||
const CHECKPOINT_WINDOW_BYTES = 4096
|
||||
|
||||
// Below this measured crossover, verifying the prefix costs more than rereading it.
|
||||
export const MIN_RESUMABLE_PREFIX_BYTES = 3 * CHECKPOINT_WINDOW_BYTES
|
||||
|
||||
export type JsonlFileCheckpoint = {
|
||||
parsedBytes: number
|
||||
boundaryDigest: string
|
||||
headDigest: string
|
||||
physicalFileId: string | null
|
||||
}
|
||||
|
||||
export type JsonlFileReader = {
|
||||
path: string
|
||||
handle: FileHandle
|
||||
stats: JsonlFileSnapshot
|
||||
snapshotCheckpoint: JsonlFileCheckpoint | null
|
||||
snapshotChunks: Buffer[] | null
|
||||
snapshotBytes: number
|
||||
streamStartOffset: number
|
||||
streamBytes: number
|
||||
onChunk: (chunk: Buffer) => void
|
||||
prepareRead: (startOffset: number) => Promise<void>
|
||||
}
|
||||
|
||||
export function isResumablePrefixLength(parsedBytes: number): boolean {
|
||||
return parsedBytes > MIN_RESUMABLE_PREFIX_BYTES
|
||||
}
|
||||
|
||||
async function readWindowDigest(
|
||||
filePath: string,
|
||||
start: number,
|
||||
endExclusive: number,
|
||||
reader?: JsonlFileReader
|
||||
): Promise<string | null> {
|
||||
const expectedBytes = endExclusive - start
|
||||
const hash = createHash('sha256')
|
||||
let readBytes = 0
|
||||
if (expectedBytes === 0) {
|
||||
return `0:${hash.digest('hex')}`
|
||||
}
|
||||
if (reader) {
|
||||
const buffer = await readJsonlSnapshotRange(
|
||||
filePath,
|
||||
reader.handle,
|
||||
reader.stats,
|
||||
start,
|
||||
endExclusive
|
||||
)
|
||||
if (!buffer) {
|
||||
return null
|
||||
}
|
||||
hash.update(buffer)
|
||||
return `${buffer.length}:${hash.digest('hex')}`
|
||||
}
|
||||
const range = { start, end: endExclusive - 1 }
|
||||
const stream = createReadStream(filePath, range)
|
||||
try {
|
||||
for await (const chunk of stream) {
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
|
||||
hash.update(buffer)
|
||||
readBytes += buffer.length
|
||||
}
|
||||
} finally {
|
||||
await releaseJsonlReadStream(stream)
|
||||
}
|
||||
return readBytes === expectedBytes ? `${expectedBytes}:${hash.digest('hex')}` : null
|
||||
}
|
||||
|
||||
async function readPrefixDigests(
|
||||
filePath: string,
|
||||
parsedBytes: number,
|
||||
carriedHeadDigest: string | null = null,
|
||||
reader?: JsonlFileReader
|
||||
): Promise<{ headDigest: string; boundaryDigest: string } | null> {
|
||||
const headBytes = Math.min(CHECKPOINT_WINDOW_BYTES, parsedBytes)
|
||||
const headDigest = carriedHeadDigest?.startsWith(`${headBytes}:`)
|
||||
? carriedHeadDigest
|
||||
: await readWindowDigest(filePath, 0, headBytes, reader)
|
||||
if (headDigest === null) {
|
||||
return null
|
||||
}
|
||||
const boundaryDigest = await readWindowDigest(
|
||||
filePath,
|
||||
Math.max(0, parsedBytes - CHECKPOINT_WINDOW_BYTES),
|
||||
parsedBytes,
|
||||
reader
|
||||
)
|
||||
return boundaryDigest === null ? null : { headDigest, boundaryDigest }
|
||||
}
|
||||
|
||||
export function jsonlPhysicalFileId(stats: {
|
||||
dev: number | bigint
|
||||
ino: number | bigint
|
||||
}): string | null {
|
||||
return stats.ino === 0 || stats.ino === 0n ? null : `${stats.dev}:${stats.ino}`
|
||||
}
|
||||
|
||||
export async function openJsonlFileReader(filePath: string): Promise<JsonlFileReader> {
|
||||
while (true) {
|
||||
const handle = await open(filePath, 'r')
|
||||
let retained = false
|
||||
try {
|
||||
const stats = await readJsonlHandleSnapshot(handle)
|
||||
const reader: JsonlFileReader = {
|
||||
path: filePath,
|
||||
handle,
|
||||
stats,
|
||||
snapshotCheckpoint: null,
|
||||
snapshotChunks: isResumablePrefixLength(stats.size) ? null : [],
|
||||
snapshotBytes: 0,
|
||||
streamStartOffset: 0,
|
||||
streamBytes: 0,
|
||||
onChunk: (chunk) => {
|
||||
reader.streamBytes += chunk.length
|
||||
if (reader.snapshotChunks) {
|
||||
reader.snapshotChunks.push(chunk)
|
||||
reader.snapshotBytes += chunk.length
|
||||
}
|
||||
},
|
||||
prepareRead: async (startOffset) => {
|
||||
reader.streamStartOffset = Math.min(startOffset, stats.size)
|
||||
if (!reader.snapshotChunks || reader.snapshotBytes !== 0 || startOffset <= 0) {
|
||||
return
|
||||
}
|
||||
const skipped = await readJsonlSnapshotRange(
|
||||
filePath,
|
||||
handle,
|
||||
stats,
|
||||
0,
|
||||
Math.min(startOffset, stats.size)
|
||||
)
|
||||
if (!skipped) {
|
||||
return
|
||||
}
|
||||
reader.snapshotChunks.push(skipped)
|
||||
reader.snapshotBytes += skipped.length
|
||||
}
|
||||
}
|
||||
if (!isResumablePrefixLength(stats.size)) {
|
||||
retained = true
|
||||
return reader
|
||||
}
|
||||
const digests = await readPrefixDigests(filePath, stats.size, null, reader)
|
||||
if (!digests) {
|
||||
continue
|
||||
}
|
||||
reader.snapshotCheckpoint = {
|
||||
parsedBytes: stats.size,
|
||||
...digests,
|
||||
physicalFileId: jsonlPhysicalFileId(stats)
|
||||
}
|
||||
retained = true
|
||||
return reader
|
||||
} finally {
|
||||
if (!retained) {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function readPhysicalFileId(filePath: string): Promise<string | null> {
|
||||
try {
|
||||
const fileStat = await readJsonlFileSnapshot(filePath)
|
||||
return jsonlPhysicalFileId(fileStat)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export async function buildJsonlFileCheckpoint(
|
||||
filePath: string,
|
||||
parsedBytes: number,
|
||||
verifiedHeadDigest: string | null = null,
|
||||
reader?: JsonlFileReader
|
||||
): Promise<JsonlFileCheckpoint | null> {
|
||||
if (!isResumablePrefixLength(parsedBytes)) {
|
||||
return null
|
||||
}
|
||||
if (reader?.snapshotCheckpoint?.parsedBytes === parsedBytes) {
|
||||
return reader.snapshotCheckpoint
|
||||
}
|
||||
const digests = await readPrefixDigests(
|
||||
filePath,
|
||||
parsedBytes,
|
||||
verifiedHeadDigest ?? reader?.snapshotCheckpoint?.headDigest,
|
||||
reader
|
||||
)
|
||||
return digests === null
|
||||
? null
|
||||
: {
|
||||
parsedBytes,
|
||||
...digests,
|
||||
physicalFileId: reader
|
||||
? jsonlPhysicalFileId(reader.stats)
|
||||
: await readPhysicalFileId(filePath)
|
||||
}
|
||||
}
|
||||
|
||||
// Append-only evidence catches rotation, truncation and head/tail rewrites; it cannot prove the middle.
|
||||
export async function resolveJsonlFileCheckpoint(
|
||||
filePath: string,
|
||||
checkpoint: JsonlFileCheckpoint | null | undefined,
|
||||
reader?: JsonlFileReader
|
||||
): Promise<JsonlFileCheckpoint | null> {
|
||||
if (
|
||||
!checkpoint ||
|
||||
!Number.isSafeInteger(checkpoint.parsedBytes) ||
|
||||
!isResumablePrefixLength(checkpoint.parsedBytes) ||
|
||||
(reader && checkpoint.parsedBytes > reader.stats.size) ||
|
||||
typeof checkpoint.boundaryDigest !== 'string' ||
|
||||
typeof checkpoint.headDigest !== 'string'
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const currentFileId = reader
|
||||
? jsonlPhysicalFileId(reader.stats)
|
||||
: await readPhysicalFileId(filePath)
|
||||
if (
|
||||
checkpoint.physicalFileId !== null &&
|
||||
currentFileId !== null &&
|
||||
checkpoint.physicalFileId !== currentFileId
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const digests = await readPrefixDigests(
|
||||
filePath,
|
||||
checkpoint.parsedBytes,
|
||||
reader?.snapshotCheckpoint?.headDigest,
|
||||
reader
|
||||
)
|
||||
return digests?.boundaryDigest === checkpoint.boundaryDigest &&
|
||||
digests?.headDigest === checkpoint.headDigest
|
||||
? checkpoint
|
||||
: null
|
||||
}
|
||||
|
||||
export async function validateJsonlFileReader(
|
||||
reader: JsonlFileReader,
|
||||
originalCheckpoint?: JsonlFileCheckpoint | null
|
||||
): Promise<boolean> {
|
||||
const [current, pathStats] = await Promise.all([
|
||||
readJsonlHandleSnapshot(reader.handle),
|
||||
readJsonlFileSnapshot(reader.path)
|
||||
])
|
||||
if (reader.streamBytes !== reader.stats.size - reader.streamStartOffset) {
|
||||
if (
|
||||
sameJsonlFileSnapshot(reader.stats, current) &&
|
||||
sameJsonlFileSnapshot(reader.stats, pathStats)
|
||||
) {
|
||||
throw new Error('JSONL snapshot stream ended before the unchanged file boundary.')
|
||||
}
|
||||
return false
|
||||
}
|
||||
if (
|
||||
current.dev !== reader.stats.dev ||
|
||||
current.ino !== reader.stats.ino ||
|
||||
pathStats.dev !== current.dev ||
|
||||
pathStats.ino !== current.ino ||
|
||||
current.size < reader.stats.size ||
|
||||
(current.size === reader.stats.size &&
|
||||
(current.mtimeMs !== reader.stats.mtimeMs || current.ctimeMs !== reader.stats.ctimeMs))
|
||||
) {
|
||||
return false
|
||||
}
|
||||
const snapshot = reader.snapshotCheckpoint
|
||||
if (!snapshot) {
|
||||
if (!reader.snapshotChunks || reader.snapshotBytes !== reader.stats.size) {
|
||||
return false
|
||||
}
|
||||
const chunks = reader.snapshotChunks
|
||||
reader.snapshotChunks = null
|
||||
return verifyJsonlSnapshotContents(reader.path, reader.handle, reader.stats, chunks)
|
||||
}
|
||||
const snapshotDigests = await readPrefixDigests(reader.path, snapshot.parsedBytes, null, reader)
|
||||
if (
|
||||
snapshotDigests?.headDigest !== snapshot.headDigest ||
|
||||
snapshotDigests?.boundaryDigest !== snapshot.boundaryDigest
|
||||
) {
|
||||
return false
|
||||
}
|
||||
if (!originalCheckpoint || originalCheckpoint.parsedBytes === snapshot.parsedBytes) {
|
||||
return true
|
||||
}
|
||||
const originalBoundary = await readWindowDigest(
|
||||
reader.path,
|
||||
Math.max(0, originalCheckpoint.parsedBytes - CHECKPOINT_WINDOW_BYTES),
|
||||
originalCheckpoint.parsedBytes,
|
||||
reader
|
||||
)
|
||||
return (
|
||||
originalCheckpoint.headDigest === snapshotDigests.headDigest &&
|
||||
originalCheckpoint.boundaryDigest === originalBoundary
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
import { appendFile, mkdtemp, rename, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
MIN_RESUMABLE_PREFIX_BYTES,
|
||||
jsonlPhysicalFileId,
|
||||
openJsonlFileReader,
|
||||
resolveJsonlFileCheckpoint,
|
||||
validateJsonlFileReader,
|
||||
type JsonlFileReader
|
||||
} from './jsonl-file-checkpoint'
|
||||
import { readJsonlLinesFromOffset, type JsonlLineAtOffset } from './jsonl-line-offsets'
|
||||
|
||||
let directory: string
|
||||
let filePath: string
|
||||
let reader: JsonlFileReader | undefined
|
||||
|
||||
beforeEach(async () => {
|
||||
directory = await mkdtemp(join(tmpdir(), 'orca-jsonl-reader-'))
|
||||
filePath = join(directory, 'transcript.jsonl')
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await reader?.handle.close()
|
||||
reader = undefined
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
async function openFixture(content: string): Promise<JsonlFileReader> {
|
||||
await writeFile(filePath, content)
|
||||
reader = await openJsonlFileReader(filePath)
|
||||
return reader
|
||||
}
|
||||
|
||||
async function collect(active: JsonlFileReader, start = 0): Promise<JsonlLineAtOffset[]> {
|
||||
const lines: JsonlLineAtOffset[] = []
|
||||
for await (const line of readJsonlLinesFromOffset(filePath, start, active)) {
|
||||
lines.push(line)
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
function positionalReadArgs(args: readonly unknown[]): [Buffer, number, number, number] {
|
||||
const [buffer, offset, length, position] = args
|
||||
if (
|
||||
!Buffer.isBuffer(buffer) ||
|
||||
typeof offset !== 'number' ||
|
||||
typeof length !== 'number' ||
|
||||
typeof position !== 'number'
|
||||
) {
|
||||
throw new Error('Expected positional descriptor read arguments.')
|
||||
}
|
||||
return [buffer, offset, length, position]
|
||||
}
|
||||
|
||||
describe('pinned small JSONL reader', () => {
|
||||
it('encodes large physical identifiers exactly and preserves numeric caller compatibility', () => {
|
||||
expect(jsonlPhysicalFileId({ dev: 1n, ino: 9_007_199_254_740_992n })).toBe('1:9007199254740992')
|
||||
expect(jsonlPhysicalFileId({ dev: 1n, ino: 9_007_199_254_740_993n })).toBe('1:9007199254740993')
|
||||
expect(jsonlPhysicalFileId({ dev: 3, ino: 4 })).toBe('3:4')
|
||||
expect(jsonlPhysicalFileId({ dev: 3, ino: 0 })).toBeNull()
|
||||
expect(jsonlPhysicalFileId({ dev: 3n, ino: 0n })).toBeNull()
|
||||
})
|
||||
|
||||
it.each([Number.NaN, Infinity, -1, 1.5, Number.MAX_SAFE_INTEGER + 1])(
|
||||
'rejects an invalid persisted offset %s before attempting a range read',
|
||||
async (parsedBytes) => {
|
||||
await expect(
|
||||
resolveJsonlFileCheckpoint(filePath, {
|
||||
parsedBytes,
|
||||
headDigest: '4096:invalid',
|
||||
boundaryDigest: '4096:invalid',
|
||||
physicalFileId: null
|
||||
})
|
||||
).resolves.toBeNull()
|
||||
}
|
||||
)
|
||||
|
||||
it('verifies an empty snapshot without creating a stream or issuing a read', async () => {
|
||||
const active = await openFixture('')
|
||||
const read = vi.spyOn(active.handle, 'read')
|
||||
const stream = vi.spyOn(active.handle, 'createReadStream')
|
||||
|
||||
expect(await collect(active)).toEqual([])
|
||||
expect(await validateJsonlFileReader(active)).toBe(true)
|
||||
expect(stream).not.toHaveBeenCalled()
|
||||
expect(read).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reads and verifies all bytes twice without creating a stream', async () => {
|
||||
const contents = '会🙂\r\n\r\nnext\n尾🙂'
|
||||
const active = await openFixture(contents)
|
||||
const read = vi.spyOn(active.handle, 'read')
|
||||
const stream = vi.spyOn(active.handle, 'createReadStream')
|
||||
|
||||
expect(await collect(active)).toEqual([
|
||||
{ line: '会🙂', endOffset: Buffer.byteLength('会🙂\r\n'), terminated: true },
|
||||
{ line: '', endOffset: Buffer.byteLength('会🙂\r\n\r\n'), terminated: true },
|
||||
{ line: 'next', endOffset: Buffer.byteLength('会🙂\r\n\r\nnext\n'), terminated: true },
|
||||
{ line: '尾🙂', endOffset: Buffer.byteLength(contents), terminated: false }
|
||||
])
|
||||
expect(await validateJsonlFileReader(active)).toBe(true)
|
||||
expect(stream).not.toHaveBeenCalled()
|
||||
expect(read).toHaveBeenCalledTimes(2)
|
||||
expect(
|
||||
(await Promise.all(read.mock.results.map((result) => result.value))).map(
|
||||
(result) => result.bytesRead
|
||||
)
|
||||
).toEqual([Buffer.byteLength(contents), Buffer.byteLength(contents)])
|
||||
})
|
||||
|
||||
it('accounts for a skipped legacy prefix in the full snapshot proof', async () => {
|
||||
const contents = 'legacy会\r\nnew🙂\r\n'
|
||||
const active = await openFixture(contents)
|
||||
const offset = Buffer.byteLength('legacy会\r\n')
|
||||
const read = vi.spyOn(active.handle, 'read')
|
||||
|
||||
expect(await collect(active, offset)).toEqual([
|
||||
{ line: 'new🙂', endOffset: Buffer.byteLength(contents), terminated: true }
|
||||
])
|
||||
expect(await validateJsonlFileReader(active)).toBe(true)
|
||||
expect(active.streamBytes).toBe(Buffer.byteLength(contents) - offset)
|
||||
expect(active.snapshotBytes).toBe(Buffer.byteLength(contents))
|
||||
expect(read).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
|
||||
it('accumulates short positional reads without exposing gaps or duplicate bytes', async () => {
|
||||
const contents = '会🙂\r\nnext\n終🙂'
|
||||
const active = await openFixture(contents)
|
||||
const read = active.handle.read.bind(active.handle)
|
||||
vi.spyOn(active.handle, 'read').mockImplementation(async (...args) => {
|
||||
const [buffer, offset, length, position] = positionalReadArgs(args)
|
||||
return read(buffer, offset, Math.min(length, 3), position)
|
||||
})
|
||||
|
||||
expect(await collect(active)).toEqual([
|
||||
{ line: '会🙂', endOffset: Buffer.byteLength('会🙂\r\n'), terminated: true },
|
||||
{ line: 'next', endOffset: Buffer.byteLength('会🙂\r\nnext\n'), terminated: true },
|
||||
{ line: '終🙂', endOffset: Buffer.byteLength(contents), terminated: false }
|
||||
])
|
||||
expect(await validateJsonlFileReader(active)).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects a stable premature EOF instead of publishing partial history', async () => {
|
||||
const active = await openFixture('one\ntwo\n')
|
||||
const read = active.handle.read.bind(active.handle)
|
||||
vi.spyOn(active.handle, 'read').mockImplementationOnce(async (...args) => {
|
||||
const result = await read(...args)
|
||||
return { ...result, bytesRead: 0 }
|
||||
})
|
||||
|
||||
await expect(collect(active)).rejects.toThrow(/snapshot read.*unchanged file boundary/)
|
||||
expect(active.streamBytes).toBe(0)
|
||||
expect(active.snapshotBytes).toBe(0)
|
||||
})
|
||||
|
||||
it('rejects an unchanged EOF after a positive short read without yielding a partial line', async () => {
|
||||
const active = await openFixture('one\ntwo\n')
|
||||
const read = active.handle.read.bind(active.handle)
|
||||
vi.spyOn(active.handle, 'read')
|
||||
.mockImplementationOnce(async (...args) => {
|
||||
const [buffer, offset, length, position] = positionalReadArgs(args)
|
||||
return read(buffer, offset, Math.min(length, 2), position)
|
||||
})
|
||||
.mockImplementationOnce(async (...args) => {
|
||||
const result = await read(...args)
|
||||
return { ...result, bytesRead: 0 }
|
||||
})
|
||||
|
||||
await expect(collect(active)).rejects.toThrow(/snapshot read.*unchanged file boundary/)
|
||||
expect(active.streamBytes).toBe(0)
|
||||
expect(active.snapshotBytes).toBe(0)
|
||||
})
|
||||
|
||||
it('rejects a zero-byte verification read even after parsing every line', async () => {
|
||||
const active = await openFixture('one\ntwo\n')
|
||||
await collect(active)
|
||||
const read = active.handle.read.bind(active.handle)
|
||||
vi.spyOn(active.handle, 'read').mockImplementationOnce(async (...args) => {
|
||||
const result = await read(...args)
|
||||
return { ...result, bytesRead: 0 }
|
||||
})
|
||||
|
||||
await expect(validateJsonlFileReader(active)).rejects.toThrow(
|
||||
/snapshot read.*unchanged file boundary/
|
||||
)
|
||||
})
|
||||
|
||||
it('reads a skipped legacy prefix again when its proof changes during parsing', async () => {
|
||||
const active = await openFixture('old\nnext\n')
|
||||
await collect(active, 4)
|
||||
await writeFile(filePath, 'NEW\nnext\nappend\n')
|
||||
|
||||
expect(await validateJsonlFileReader(active)).toBe(false)
|
||||
})
|
||||
|
||||
it('leaves an append after the captured boundary for the next scan', async () => {
|
||||
const original = 'one会🙂\r\n'
|
||||
const active = await openFixture(original)
|
||||
await appendFile(filePath, 'two🙂\n')
|
||||
|
||||
expect(await collect(active)).toEqual([
|
||||
{ line: 'one会🙂', endOffset: Buffer.byteLength(original), terminated: true }
|
||||
])
|
||||
expect(await validateJsonlFileReader(active)).toBe(true)
|
||||
expect(active.streamBytes).toBe(Buffer.byteLength(original))
|
||||
})
|
||||
|
||||
it('rejects a full small-file middle rewrite even when the source also grows', async () => {
|
||||
const original = 'head\nold\ntail\n'
|
||||
const active = await openFixture(original)
|
||||
await collect(active)
|
||||
await writeFile(filePath, 'head\nNEW\ntail\nappend\n')
|
||||
|
||||
expect(await validateJsonlFileReader(active)).toBe(false)
|
||||
})
|
||||
|
||||
it('keeps the pinned handle but rejects path rotation', async () => {
|
||||
const original = 'one\n'
|
||||
const active = await openFixture(original)
|
||||
const replacement = join(directory, 'replacement.jsonl')
|
||||
await writeFile(replacement, 'two\n')
|
||||
await rename(filePath, join(directory, 'rotated.jsonl'))
|
||||
await rename(replacement, filePath)
|
||||
|
||||
expect(await collect(active)).toEqual([{ line: 'one', endOffset: 4, terminated: true }])
|
||||
expect(await validateJsonlFileReader(active)).toBe(false)
|
||||
})
|
||||
|
||||
it('invalidates a truncation encountered by the descriptor loop', async () => {
|
||||
const active = await openFixture('one\ntwo\n')
|
||||
await writeFile(filePath, 'one\n')
|
||||
|
||||
expect(await collect(active)).toEqual([])
|
||||
expect(await validateJsonlFileReader(active)).toBe(false)
|
||||
})
|
||||
|
||||
it('uses the bounded buffer at the crossover and streams larger snapshots', async () => {
|
||||
const active = await openFixture(`${'x'.repeat(MIN_RESUMABLE_PREFIX_BYTES - 1)}\n`)
|
||||
const stream = vi.spyOn(active.handle, 'createReadStream')
|
||||
await collect(active)
|
||||
expect(await validateJsonlFileReader(active)).toBe(true)
|
||||
expect(stream).not.toHaveBeenCalled()
|
||||
await active.handle.close()
|
||||
reader = undefined
|
||||
const large = await openFixture(`${'x'.repeat(MIN_RESUMABLE_PREFIX_BYTES)}\n`)
|
||||
const largeStream = vi.spyOn(large.handle, 'createReadStream')
|
||||
await collect(large)
|
||||
expect(await validateJsonlFileReader(large)).toBe(true)
|
||||
expect(largeStream).toHaveBeenCalledOnce()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,92 @@
|
||||
import { mkdtemp, open, rm, stat, utimes, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
normalizeJsonlFileSnapshot,
|
||||
readJsonlFileSnapshot,
|
||||
readJsonlHandleSnapshot,
|
||||
sameJsonlFileSnapshot
|
||||
} from './jsonl-file-snapshot'
|
||||
|
||||
let directory: string
|
||||
let filePath: string
|
||||
|
||||
beforeEach(async () => {
|
||||
directory = await mkdtemp(join(tmpdir(), 'orca-jsonl-snapshot-'))
|
||||
filePath = join(directory, 'transcript.jsonl')
|
||||
await writeFile(filePath, 'one\n')
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
describe('JSONL file snapshot normalization', () => {
|
||||
it.each(['dev', 'ino'] as const)(
|
||||
'distinguishes exact %s identifiers that collide as JavaScript numbers',
|
||||
async (field) => {
|
||||
const stats = await stat(filePath, { bigint: true })
|
||||
const lower = { ...stats, [field]: 9_007_199_254_740_992n }
|
||||
const upper = { ...stats, [field]: 9_007_199_254_740_993n }
|
||||
|
||||
expect(Number(lower[field])).toBe(Number(upper[field]))
|
||||
expect(normalizeJsonlFileSnapshot(lower)[field]).not.toBe(
|
||||
normalizeJsonlFileSnapshot(upper)[field]
|
||||
)
|
||||
expect(
|
||||
sameJsonlFileSnapshot(normalizeJsonlFileSnapshot(lower), normalizeJsonlFileSnapshot(upper))
|
||||
).toBe(false)
|
||||
}
|
||||
)
|
||||
|
||||
it.each([0.123456789, 1.999999999, 1_784_029_260.125, 2_147_483_647.875])(
|
||||
'preserves numeric stat timestamps at fractional epoch seconds %s',
|
||||
async (seconds) => {
|
||||
await utimes(filePath, seconds, seconds)
|
||||
const numeric = await stat(filePath)
|
||||
const snapshot = await readJsonlFileSnapshot(filePath)
|
||||
const handle = await open(filePath, 'r')
|
||||
try {
|
||||
expect(snapshot.mtimeMs).toBe(numeric.mtimeMs)
|
||||
expect(snapshot.ctimeMs).toBe(numeric.ctimeMs)
|
||||
expect(snapshot.size).toBe(numeric.size)
|
||||
expect(await readJsonlHandleSnapshot(handle)).toEqual(snapshot)
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
it('retains fractional nanoseconds instead of truncating bigint milliseconds', async () => {
|
||||
const stats = await stat(filePath, { bigint: true })
|
||||
const snapshot = normalizeJsonlFileSnapshot({
|
||||
...stats,
|
||||
mtimeNs: 1_784_029_260_123_456_789n,
|
||||
ctimeNs: 1_784_029_260_987_654_321n
|
||||
})
|
||||
|
||||
expect(snapshot.mtimeMs).toBe(1_784_029_260_000 + 123.456789)
|
||||
expect(snapshot.ctimeMs).toBe(1_784_029_260_000 + 987.654321)
|
||||
expect(snapshot.mtimeMs).not.toBe(1_784_029_260_123)
|
||||
expect(snapshot.ctimeMs).not.toBe(1_784_029_260_987)
|
||||
})
|
||||
|
||||
it('accepts zero and the largest safe byte offset', async () => {
|
||||
const stats = await stat(filePath, { bigint: true })
|
||||
|
||||
expect(normalizeJsonlFileSnapshot({ ...stats, size: 0n }).size).toBe(0)
|
||||
expect(
|
||||
normalizeJsonlFileSnapshot({ ...stats, size: BigInt(Number.MAX_SAFE_INTEGER) }).size
|
||||
).toBe(Number.MAX_SAFE_INTEGER)
|
||||
})
|
||||
|
||||
it.each([-1n, BigInt(Number.MAX_SAFE_INTEGER) + 1n, 1n << 1024n])(
|
||||
'rejects a file size that is not a safe byte offset: %s',
|
||||
async (size) => {
|
||||
const stats = await stat(filePath, { bigint: true })
|
||||
|
||||
expect(() => normalizeJsonlFileSnapshot({ ...stats, size })).toThrow(/safe byte offset/)
|
||||
}
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,113 @@
|
||||
import type { BigIntStats, ReadStream } from 'node:fs'
|
||||
import { stat, type FileHandle } from 'node:fs/promises'
|
||||
|
||||
const SNAPSHOT_FIELDS = ['dev', 'ino', 'size', 'mtimeMs', 'ctimeMs'] as const
|
||||
|
||||
export type JsonlFileSnapshot = {
|
||||
dev: bigint
|
||||
ino: bigint
|
||||
size: number
|
||||
mtimeMs: number
|
||||
ctimeMs: number
|
||||
}
|
||||
|
||||
export async function releaseJsonlReadStream(stream: ReadStream): Promise<void> {
|
||||
if (!stream.closed) {
|
||||
await new Promise<void>((resolve) => {
|
||||
stream.once('close', resolve)
|
||||
stream.destroy()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function nanosecondsToMilliseconds(value: bigint): number {
|
||||
return Number(value / 1_000_000_000n) * 1000 + Number(value % 1_000_000_000n) / 1_000_000
|
||||
}
|
||||
|
||||
export function normalizeJsonlFileSnapshot(
|
||||
stats: Pick<BigIntStats, 'dev' | 'ino' | 'size' | 'mtimeNs' | 'ctimeNs'>
|
||||
): JsonlFileSnapshot {
|
||||
const size = Number(stats.size)
|
||||
if (!Number.isSafeInteger(size) || size < 0) {
|
||||
throw new RangeError('JSONL snapshot size cannot be represented as a safe byte offset.')
|
||||
}
|
||||
return {
|
||||
dev: stats.dev,
|
||||
ino: stats.ino,
|
||||
size,
|
||||
mtimeMs: nanosecondsToMilliseconds(stats.mtimeNs),
|
||||
ctimeMs: nanosecondsToMilliseconds(stats.ctimeNs)
|
||||
}
|
||||
}
|
||||
|
||||
export async function readJsonlFileSnapshot(filePath: string): Promise<JsonlFileSnapshot> {
|
||||
return normalizeJsonlFileSnapshot(await stat(filePath, { bigint: true }))
|
||||
}
|
||||
|
||||
export async function readJsonlHandleSnapshot(handle: FileHandle): Promise<JsonlFileSnapshot> {
|
||||
return normalizeJsonlFileSnapshot(await handle.stat({ bigint: true }))
|
||||
}
|
||||
|
||||
export function sameJsonlFileSnapshot(left: JsonlFileSnapshot, right: JsonlFileSnapshot): boolean {
|
||||
return SNAPSHOT_FIELDS.every((field) => left[field] === right[field])
|
||||
}
|
||||
|
||||
export async function rejectUnchangedJsonlFileRead(
|
||||
filePath: string,
|
||||
handle: FileHandle,
|
||||
initial: JsonlFileSnapshot
|
||||
): Promise<void> {
|
||||
const [current, pathStats] = await Promise.all([
|
||||
readJsonlHandleSnapshot(handle),
|
||||
readJsonlFileSnapshot(filePath)
|
||||
])
|
||||
if (sameJsonlFileSnapshot(initial, current) && sameJsonlFileSnapshot(initial, pathStats)) {
|
||||
throw new Error('JSONL snapshot read ended before the unchanged file boundary.')
|
||||
}
|
||||
}
|
||||
|
||||
export async function readJsonlSnapshotRange(
|
||||
filePath: string,
|
||||
handle: FileHandle,
|
||||
initial: JsonlFileSnapshot,
|
||||
start: number,
|
||||
endExclusive: number
|
||||
): Promise<Buffer | null> {
|
||||
const buffer = Buffer.allocUnsafe(endExclusive - start)
|
||||
let consumed = 0
|
||||
while (consumed < buffer.length) {
|
||||
const { bytesRead } = await handle.read(
|
||||
buffer,
|
||||
consumed,
|
||||
buffer.length - consumed,
|
||||
start + consumed
|
||||
)
|
||||
if (bytesRead === 0) {
|
||||
await rejectUnchangedJsonlFileRead(filePath, handle, initial)
|
||||
return null
|
||||
}
|
||||
consumed += bytesRead
|
||||
}
|
||||
return buffer
|
||||
}
|
||||
|
||||
export async function verifyJsonlSnapshotContents(
|
||||
filePath: string,
|
||||
handle: FileHandle,
|
||||
initial: JsonlFileSnapshot,
|
||||
pieces: readonly Buffer[]
|
||||
): Promise<boolean> {
|
||||
const current = await readJsonlSnapshotRange(filePath, handle, initial, 0, initial.size)
|
||||
if (!current) {
|
||||
return false
|
||||
}
|
||||
let consumed = 0
|
||||
for (const piece of pieces) {
|
||||
const end = consumed + piece.length
|
||||
if (end > current.length || piece.compare(current, consumed, end) !== 0) {
|
||||
return false
|
||||
}
|
||||
consumed = end
|
||||
}
|
||||
return consumed === current.length
|
||||
}
|
||||
@@ -5,6 +5,8 @@
|
||||
* CRLF transcript would otherwise drift one byte per line.
|
||||
*/
|
||||
import { createReadStream } from 'node:fs'
|
||||
import { isResumablePrefixLength, type JsonlFileReader } from './jsonl-file-checkpoint'
|
||||
import { readJsonlSnapshotRange, releaseJsonlReadStream } from './jsonl-file-snapshot'
|
||||
|
||||
const LINE_FEED = 0x0a
|
||||
const CARRIAGE_RETURN = 0x0d
|
||||
@@ -23,17 +25,61 @@ function decodeLine(pieces: Buffer[]): string {
|
||||
return raw.toString('utf-8', 0, end)
|
||||
}
|
||||
|
||||
async function* readJsonlChunks(
|
||||
filePath: string,
|
||||
startOffset: number,
|
||||
reader?: Pick<JsonlFileReader, 'handle' | 'stats'>
|
||||
): AsyncGenerator<Buffer> {
|
||||
if (reader && !isResumablePrefixLength(reader.stats.size)) {
|
||||
// Tiny snapshots avoid stream setup while keeping the same pinned-range proof.
|
||||
const chunk = await readJsonlSnapshotRange(
|
||||
filePath,
|
||||
reader.handle,
|
||||
reader.stats,
|
||||
startOffset,
|
||||
reader.stats.size
|
||||
)
|
||||
if (chunk) {
|
||||
yield chunk
|
||||
}
|
||||
return
|
||||
}
|
||||
const stream = reader
|
||||
? reader.handle.createReadStream({
|
||||
start: startOffset,
|
||||
end: reader.stats.size - 1,
|
||||
autoClose: false
|
||||
})
|
||||
: createReadStream(filePath, { start: startOffset })
|
||||
try {
|
||||
for await (const chunk of stream) {
|
||||
yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
|
||||
}
|
||||
} finally {
|
||||
if (!reader) {
|
||||
await releaseJsonlReadStream(stream)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function* readJsonlLinesFromOffset(
|
||||
filePath: string,
|
||||
startOffset: number
|
||||
startOffset: number,
|
||||
reader?: Pick<JsonlFileReader, 'handle' | 'stats'> & {
|
||||
onChunk?: JsonlFileReader['onChunk']
|
||||
prepareRead?: JsonlFileReader['prepareRead']
|
||||
}
|
||||
): AsyncGenerator<JsonlLineAtOffset> {
|
||||
const stream = createReadStream(filePath, { start: startOffset })
|
||||
await reader?.prepareRead?.(startOffset)
|
||||
if (reader && reader.stats.size <= startOffset) {
|
||||
return
|
||||
}
|
||||
const pending: Buffer[] = []
|
||||
let pendingBytes = 0
|
||||
let endOffset = startOffset
|
||||
|
||||
for await (const rawChunk of stream) {
|
||||
const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk)
|
||||
for await (const chunk of readJsonlChunks(filePath, startOffset, reader)) {
|
||||
reader?.onChunk?.(chunk)
|
||||
let searchFrom = 0
|
||||
for (;;) {
|
||||
const lineFeedIndex = chunk.indexOf(LINE_FEED, searchFrom)
|
||||
|
||||
@@ -55,6 +55,145 @@ it('folds events without rescanning accumulated location breakdowns', () => {
|
||||
expect(result.dailyAggregates).toHaveLength(1000)
|
||||
})
|
||||
|
||||
it('folds streamed events immediately with the batch counters, metrics and metadata', () => {
|
||||
const [seed] = events(1)
|
||||
const first: Event = {
|
||||
...seed,
|
||||
sessionId: 'mixed',
|
||||
timestamp: '2026-09-09T00:00:00Z',
|
||||
day: '2026-09-09',
|
||||
projectKey: 'z',
|
||||
projectLabel: 'Z',
|
||||
repoId: 'repo-z',
|
||||
worktreeId: 'tree-z',
|
||||
model: 'large',
|
||||
inputTokens: 3,
|
||||
cachedInputTokens: 1,
|
||||
outputTokens: 5,
|
||||
reasoningOutputTokens: 2,
|
||||
totalTokens: 8,
|
||||
cost: 2
|
||||
}
|
||||
const input: Event[] = [
|
||||
first,
|
||||
{
|
||||
...seed,
|
||||
sessionId: 'unknown',
|
||||
timestamp: '2026-09-10T00:00:00Z',
|
||||
day: '2026-09-10',
|
||||
model: null
|
||||
},
|
||||
{
|
||||
...first,
|
||||
timestamp: '2026-09-07T00:00:00Z',
|
||||
day: '2026-09-07',
|
||||
projectKey: 'a',
|
||||
projectLabel: 'A',
|
||||
repoId: 'repo-a',
|
||||
worktreeId: 'tree-a',
|
||||
model: 'small',
|
||||
inputTokens: 7,
|
||||
cachedInputTokens: 3,
|
||||
outputTokens: 11,
|
||||
reasoningOutputTokens: 5,
|
||||
totalTokens: 18,
|
||||
cost: 5
|
||||
},
|
||||
{
|
||||
...first,
|
||||
timestamp: '2026-09-09T12:00:00Z',
|
||||
projectLabel: 'Changed label',
|
||||
repoId: 'changed-repo',
|
||||
worktreeId: 'changed-tree',
|
||||
inputTokens: 13,
|
||||
cachedInputTokens: 7,
|
||||
outputTokens: 17,
|
||||
reasoningOutputTokens: 11,
|
||||
totalTokens: 30,
|
||||
cost: 7
|
||||
}
|
||||
]
|
||||
const expected = aggregation.aggregate(input)
|
||||
const accumulator = aggregation.createAccumulator()
|
||||
for (const event of input) {
|
||||
accumulator.add(event)
|
||||
event.totalTokens = 999
|
||||
event.cost = 999
|
||||
event.model = 'changed'
|
||||
event.projectLabel = 'Changed after adding'
|
||||
}
|
||||
const result = accumulator.finalize()
|
||||
expect(result).toEqual(expected)
|
||||
expect(result.sessions.map((session) => session.sessionId)).toEqual(['unknown', 'mixed'])
|
||||
expect(result.sessions[0].primaryModel).toBe('Unknown model')
|
||||
const mixed = result.sessions[1]
|
||||
expect(mixed).toMatchObject({
|
||||
firstTimestamp: '2026-09-07T00:00:00Z',
|
||||
lastTimestamp: '2026-09-09T12:00:00Z',
|
||||
primaryModel: 'Mixed models',
|
||||
hasMixedModels: true,
|
||||
primaryProjectLabel: 'Multiple locations',
|
||||
hasMixedLocations: true,
|
||||
primaryRepoId: 'repo-z',
|
||||
primaryWorktreeId: 'tree-z',
|
||||
eventCount: 3,
|
||||
totalInputTokens: 23,
|
||||
totalCachedInputTokens: 11,
|
||||
totalOutputTokens: 33,
|
||||
totalReasoningOutputTokens: 18,
|
||||
totalTokens: 56,
|
||||
cost: 14
|
||||
})
|
||||
expect(mixed.locationBreakdown).toMatchObject([
|
||||
{ locationKey: 'z', projectLabel: 'Z', totalTokens: 38, cost: 9 },
|
||||
{ locationKey: 'a', projectLabel: 'A', totalTokens: 18, cost: 5 }
|
||||
])
|
||||
expect(mixed.modelBreakdown).toMatchObject([
|
||||
{ modelKey: 'large', totalTokens: 38, cost: 9 },
|
||||
{ modelKey: 'small', totalTokens: 18, cost: 5 }
|
||||
])
|
||||
expect(mixed.locationModelBreakdown).toMatchObject([
|
||||
{ locationKey: 'z', modelKey: 'large', totalTokens: 38, cost: 9 },
|
||||
{ locationKey: 'a', modelKey: 'small', totalTokens: 18, cost: 5 }
|
||||
])
|
||||
expect(result.dailyAggregates).toMatchObject([
|
||||
{ day: '2026-09-07', projectKey: 'a', totalTokens: 18, cost: 5 },
|
||||
{ day: '2026-09-09', projectKey: 'z', projectLabel: 'Z', totalTokens: 38, cost: 9 },
|
||||
{ day: '2026-09-10', projectKey: 'path-0', totalTokens: 3, cost: 0.5 }
|
||||
])
|
||||
})
|
||||
|
||||
it('keeps first encounter order when streamed rollup sort keys tie', () => {
|
||||
const accumulator = aggregation.createAccumulator()
|
||||
for (const [index, event] of events(4).entries()) {
|
||||
accumulator.add({ ...event, sessionId: `session-${index % 2}`, projectLabel: 'Same label' })
|
||||
}
|
||||
const result = accumulator.finalize()
|
||||
expect(result.sessions.map((session) => session.sessionId)).toEqual(['session-0', 'session-1'])
|
||||
expect(result.sessions[0].locationBreakdown.map((entry) => entry.locationKey)).toEqual([
|
||||
'path-0',
|
||||
'path-2'
|
||||
])
|
||||
expect(result.sessions[0].modelBreakdown.map((entry) => entry.modelKey)).toEqual([
|
||||
'model-0',
|
||||
'model-2'
|
||||
])
|
||||
expect(result.dailyAggregates.map((entry) => entry.projectKey)).toEqual([
|
||||
'path-0',
|
||||
'path-1',
|
||||
'path-2',
|
||||
'path-3'
|
||||
])
|
||||
})
|
||||
|
||||
it('finalizes an empty stream once and rejects subsequent additions', () => {
|
||||
const accumulator = aggregation.createAccumulator()
|
||||
const result = accumulator.finalize()
|
||||
expect(result).toEqual({ sessions: [], dailyAggregates: [] })
|
||||
expect(accumulator.finalize()).toBe(result)
|
||||
expect(() => accumulator.add(events(1)[0])).toThrow('Cannot add usage events after finalization')
|
||||
})
|
||||
|
||||
it('merges rollups using first-match indexes without mutating source breakdowns', () => {
|
||||
const source = aggregation.aggregate(events(1000)).sessions[0]
|
||||
const existing = structuredClone(source)
|
||||
|
||||
@@ -25,6 +25,11 @@ export type UsageEventAggregationOptions<TEvent, TMetric> = {
|
||||
cloneSessionForMerge(session: UsageSession<TMetric>): UsageSession<TMetric>
|
||||
}
|
||||
|
||||
export type UsageEventAggregationResult<TMetric> = {
|
||||
sessions: UsageSession<TMetric>[]
|
||||
dailyAggregates: UsageDailyAggregate<TMetric>[]
|
||||
}
|
||||
|
||||
export function createUsageEventAggregation<
|
||||
TEvent extends UsageAttributedEventFields,
|
||||
TMetric extends object
|
||||
@@ -214,15 +219,17 @@ export function createUsageEventAggregation<
|
||||
)
|
||||
}
|
||||
|
||||
function aggregate(events: TEvent[]): {
|
||||
sessions: UsageSession<TMetric>[]
|
||||
dailyAggregates: UsageDailyAggregate<TMetric>[]
|
||||
} {
|
||||
/** Finalization closes the stream and releases its lookup indexes. */
|
||||
function createAccumulator() {
|
||||
const sessionsById = new Map<string, UsageSession<TMetric>>()
|
||||
const dailyByKey = new Map<string, UsageDailyAggregate<TMetric>>()
|
||||
const breakdownsBySession = new Map<string, UsageSessionBreakdownIndex<TMetric>>()
|
||||
let result: UsageEventAggregationResult<TMetric> | null = null
|
||||
|
||||
for (const event of events) {
|
||||
function add(event: TEvent): void {
|
||||
if (result) {
|
||||
throw new Error('Cannot add usage events after finalization')
|
||||
}
|
||||
const eventMetric = metric.fromEvent(event)
|
||||
const session = sessionsById.get(event.sessionId) ?? createEmptySession(event)
|
||||
if (!sessionsById.has(event.sessionId)) {
|
||||
@@ -269,14 +276,34 @@ export function createUsageEventAggregation<
|
||||
metric.fold(daily, eventMetric)
|
||||
}
|
||||
|
||||
return {
|
||||
sessions: finalizeSessions(sessionsById),
|
||||
dailyAggregates: sortDailyAggregates(dailyByKey)
|
||||
function finalize(): UsageEventAggregationResult<TMetric> {
|
||||
if (result) {
|
||||
return result
|
||||
}
|
||||
result = {
|
||||
sessions: finalizeSessions(sessionsById),
|
||||
dailyAggregates: sortDailyAggregates(dailyByKey)
|
||||
}
|
||||
sessionsById.clear()
|
||||
dailyByKey.clear()
|
||||
breakdownsBySession.clear()
|
||||
return result
|
||||
}
|
||||
|
||||
return { add, finalize }
|
||||
}
|
||||
|
||||
function aggregate(events: TEvent[]) {
|
||||
const accumulator = createAccumulator()
|
||||
for (const event of events) {
|
||||
accumulator.add(event)
|
||||
}
|
||||
return accumulator.finalize()
|
||||
}
|
||||
|
||||
return {
|
||||
aggregate,
|
||||
createAccumulator,
|
||||
finalizeSessions,
|
||||
sortDailyAggregates,
|
||||
mergeSessions: (
|
||||
|
||||
Reference in New Issue
Block a user