mirror of
https://github.com/stablyai/orca.git
synced 2026-11-18 11:54:46 +00:00
fix(usage): rebuild corrupt per-source usage cache records instead of failing every scan (#42418)
A null or non-object entry in a per-source usage sidecar made the Codex, Muse and OpenCode scans throw, and since a failed scan never rewrites the sidecar the failure repeated on every refresh. Records missing their sessions/dailyAggregates projections were either reused as zero usage (Muse/OpenCode) or crashed the Codex reuse and incremental-resume paths. - readUsageSourceCache drops entries that are not plain objects. - Muse and OpenCode normalizers drop records lacking projection arrays so the source is reparsed. - Codex requires both projection arrays before reusing or resuming a cached rollout record. Fixes #42413 Co-authored-by: Neil <neil@stably.ai>
This commit is contained in:
@@ -142,4 +142,25 @@ describe('scanCodexUsageFiles incremental append', () => {
|
||||
const second = await scanCodexUsageFiles([], first.processedFiles)
|
||||
expect(totalTokens(second.dailyAggregates)).toBe(5)
|
||||
})
|
||||
|
||||
it('reparses a grown rollout whose cached record lost its sessions instead of failing the scan', async () => {
|
||||
const rolloutPath = join(sessionsDir, 'rollout-damaged.jsonl')
|
||||
writeFileSync(
|
||||
rolloutPath,
|
||||
`${sessionMeta('session-damaged')}${usageRecordRange(0, RESUMABLE_RECORDS)}`
|
||||
)
|
||||
const first = await scanCodexUsageFiles([], [])
|
||||
expect(recordedResumeOffset(first.processedFiles, rolloutPath)).toBeGreaterThan(0)
|
||||
const damaged = first.processedFiles.map((file) => ({ ...file }))
|
||||
for (const file of damaged) {
|
||||
Reflect.deleteProperty(file, 'sessions')
|
||||
}
|
||||
|
||||
writeFileSync(
|
||||
rolloutPath,
|
||||
`${sessionMeta('session-damaged')}${usageRecordRange(0, RESUMABLE_RECORDS + 5)}`
|
||||
)
|
||||
const second = await scanCodexUsageFiles([], damaged)
|
||||
expect(totalTokens(second.dailyAggregates)).toBe(RESUMABLE_RECORDS + 5)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -57,15 +57,18 @@ export async function scanCodexUsageFiles(
|
||||
const fileInfo = await getProcessedFileInfo(filePath)
|
||||
const previous = previousByPath.get(filePath)
|
||||
const mustReclaimDeferred = lostOwnerPath && previous?.hasDeferredClaims !== false
|
||||
const hasGenerationMetadata =
|
||||
// Reuse and resume both build on the cached projections, so a record missing them is reparsed.
|
||||
const hasReusableRecord =
|
||||
previous &&
|
||||
typeof previous.ctimeMs === 'number' &&
|
||||
(typeof previous.physicalFileId === 'string' || previous.physicalFileId === null)
|
||||
(typeof previous.physicalFileId === 'string' || previous.physicalFileId === null) &&
|
||||
Array.isArray(previous.sessions) &&
|
||||
Array.isArray(previous.dailyAggregates)
|
||||
const canReuse =
|
||||
!mustReclaimDeferred &&
|
||||
legacySourceSkipBytes === 0 &&
|
||||
previous &&
|
||||
hasGenerationMetadata &&
|
||||
hasReusableRecord &&
|
||||
previous.mtimeMs === fileInfo.mtimeMs &&
|
||||
previous.size === fileInfo.size &&
|
||||
previous.ctimeMs === fileInfo.ctimeMs &&
|
||||
@@ -80,7 +83,7 @@ export async function scanCodexUsageFiles(
|
||||
!mustReclaimDeferred &&
|
||||
legacySourceSkipBytes === 0 &&
|
||||
previous &&
|
||||
hasGenerationMetadata &&
|
||||
hasReusableRecord &&
|
||||
fileInfo.size > previous.size
|
||||
) {
|
||||
const state = await resolveCodexRolloutResume(filePath, previous)
|
||||
|
||||
@@ -4,13 +4,18 @@ import type { MuseUsagePersistedFile } from './types'
|
||||
export function normalizeMuseUsagePersistedFiles(
|
||||
files: MuseUsagePersistedFile[] | undefined
|
||||
): MuseUsagePersistedFile[] {
|
||||
return (files ?? []).map((file) => ({
|
||||
...file,
|
||||
sessions: file.sessions ?? [],
|
||||
dailyAggregates: file.dailyAggregates ?? [],
|
||||
ownedEventKeys: file.ownedEventKeys ?? [],
|
||||
hasDeferredClaims: file.hasDeferredClaims ?? true,
|
||||
sessionCwd: file.sessionCwd ?? null,
|
||||
inheritedCwd: file.inheritedCwd ?? null
|
||||
}))
|
||||
// A record missing a projection would be reused as zero usage; drop it so the scanner rereads the file.
|
||||
return (files ?? []).flatMap((file) =>
|
||||
Array.isArray(file.sessions) && Array.isArray(file.dailyAggregates)
|
||||
? [
|
||||
{
|
||||
...file,
|
||||
ownedEventKeys: file.ownedEventKeys ?? [],
|
||||
hasDeferredClaims: file.hasDeferredClaims ?? true,
|
||||
sessionCwd: file.sessionCwd ?? null,
|
||||
inheritedCwd: file.inheritedCwd ?? null
|
||||
}
|
||||
]
|
||||
: []
|
||||
)
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
vi.mock('electron', () => ({ app: { getPath: vi.fn(() => tmpdir()) } }))
|
||||
|
||||
import { parseMuseUsageLine, type MuseUsageParseContext } from './muse-usage-record-parser'
|
||||
import { normalizeMuseUsagePersistedFiles } from './persisted-file-normalization'
|
||||
import { scanMuseUsageFiles } from './scanner'
|
||||
|
||||
const MICROS = 1_790_052_419_142_353
|
||||
@@ -130,6 +131,26 @@ describe('scanMuseUsageFiles', () => {
|
||||
displayName: string
|
||||
}[] => [{ repoId: 'repo-1', worktreeId: 'wt-1', path: worktreePath, displayName: 'repo/main' }]
|
||||
|
||||
it('rereads a log whose cached record lost its sessions instead of reporting zero usage', async () => {
|
||||
writeSession('2026-09-22', 'a', [
|
||||
metadata(worktreePath),
|
||||
modelCompleted(MICROS, { input_tokens: 100, output_tokens: 10 }, 'm')
|
||||
])
|
||||
const first = await scanMuseUsageFiles(worktrees(), [], undefined, sessionsDir)
|
||||
const damaged = first.processedFiles.map((file) => ({ ...file }))
|
||||
for (const file of damaged) {
|
||||
Reflect.deleteProperty(file, 'sessions')
|
||||
}
|
||||
|
||||
const second = await scanMuseUsageFiles(
|
||||
worktrees(),
|
||||
normalizeMuseUsagePersistedFiles(damaged),
|
||||
undefined,
|
||||
sessionsDir
|
||||
)
|
||||
expect(second.sessions.reduce((sum, session) => sum + session.totalTokens, 0)).toBe(110)
|
||||
})
|
||||
|
||||
it('counts a usage record copied into another session log once', async () => {
|
||||
const shared = modelCompleted(MICROS, { input_tokens: 100, output_tokens: 10 }, 'm')
|
||||
writeSession('2026-09-22', 'a-parent', [metadata(worktreePath), shared])
|
||||
|
||||
@@ -8,11 +8,18 @@ import type {
|
||||
export function normalizeOpenCodeUsagePersistedDatabases(
|
||||
databases: OpenCodeUsagePersistedDatabase[] | undefined
|
||||
): OpenCodeUsagePersistedDatabase[] {
|
||||
return (databases ?? []).map((database) => ({
|
||||
...database,
|
||||
sessions: (database.sessions ?? []).map(normalizeSessionCost),
|
||||
dailyAggregates: (database.dailyAggregates ?? []).map(normalizeDailyAggregateCost)
|
||||
}))
|
||||
// A record missing a projection would be reused as zero usage; drop it so the scanner rereads the database.
|
||||
return (databases ?? []).flatMap((database) =>
|
||||
Array.isArray(database.sessions) && Array.isArray(database.dailyAggregates)
|
||||
? [
|
||||
{
|
||||
...database,
|
||||
sessions: database.sessions.map(normalizeSessionCost),
|
||||
dailyAggregates: database.dailyAggregates.map(normalizeDailyAggregateCost)
|
||||
}
|
||||
]
|
||||
: []
|
||||
)
|
||||
}
|
||||
|
||||
export function normalizeDailyAggregateCost(
|
||||
|
||||
@@ -102,6 +102,20 @@ describe('usage source cache file', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('drops entries that are not records so one bad entry cannot fail every later scan', async () => {
|
||||
const ref = {
|
||||
path: join(directory, 'sources.json'),
|
||||
schemaVersion: 7,
|
||||
worktreeFingerprint: 'w',
|
||||
reuse: true
|
||||
}
|
||||
writeFileSync(
|
||||
ref.path,
|
||||
'{"schemaVersion":7,"worktreeFingerprint":"w","sources":[null,7,["a.jsonl"],{"path":"a.jsonl"}]}'
|
||||
)
|
||||
expect(await readUsageSourceCache(ref)).toEqual([{ path: 'a.jsonl' }])
|
||||
})
|
||||
|
||||
it('keeps the prior cache and removes the partial temp when encoding fails after writing', async () => {
|
||||
const ref = {
|
||||
path: join(directory, 'sources.json'),
|
||||
|
||||
+6
-1
@@ -46,13 +46,18 @@ export async function readUsageSourceCache<T>(ref: UsageSourceCacheRef): Promise
|
||||
return parsed?.schemaVersion === ref.schemaVersion &&
|
||||
parsed.worktreeFingerprint === ref.worktreeFingerprint &&
|
||||
Array.isArray(parsed.sources)
|
||||
? parsed.sources
|
||||
? parsed.sources.filter(isSourceRecord)
|
||||
: []
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
// One unreadable entry would otherwise fail every later scan, since a failed scan never rewrites the sidecar.
|
||||
function isSourceRecord(source: unknown): boolean {
|
||||
return typeof source === 'object' && source !== null && !Array.isArray(source)
|
||||
}
|
||||
|
||||
export async function writeUsageSourceCache(
|
||||
ref: UsageSourceCacheRef,
|
||||
sources: readonly unknown[]
|
||||
|
||||
Reference in New Issue
Block a user