orca/config/scripts/claude-usage-benchmark-scanner.mjs

import assert from 'node:assert/strict'
import { createHash } from 'node:crypto'
import { readFile, writeFile } from 'node:fs/promises'
import { dirname, extname, join, relative, sep } from 'node:path'
import { performance } from 'node:perf_hooks'
import { fileURLToPath } from 'node:url'
import { build, version } from 'esbuild'

const MIB = 1024 * 1024
const ROOT = fileURLToPath(new URL('../..', import.meta.url))
const BASELINE_MODULES = [
  'src/main/claude-usage/scanner.ts',
  'src/main/claude-usage/transcript-record-parser.ts',
  'src/main/claude-usage/usage-aggregation.ts',
  'src/main/claude-usage/worktree-attribution.ts',
  'src/main/claude-usage/transcript-file-discovery.ts'
]

export async function describeClaudeUsageBenchmarkTooling() {
  const paths = [
    'config/scripts/claude-usage-append-benchmark.mjs',
    'config/scripts/claude-usage-benchmark-scanner.mjs',
    'config/scripts/counterbalanced-benchmark-schedule.mjs',
    'src/packages/process-host/src/run-process.ts',
    'src/packages/process-host/dist/run-process.js'
  ]
  return {
    esbuildVersion: version,
    baselineModules: BASELINE_MODULES,
    sha256: Object.fromEntries(
      await Promise.all(
        paths.map(async (path) => [
          path,
          createHash('sha256')
            .update(await readFile(join(ROOT, ...path.split('/'))))
            .digest('hex')
        ])
      )
    )
  }
}

export async function readClaudeUsageBenchmarkBaselineSources() {
  const { runProcessSync } = await import('@orca/process-host')
  const readGit = (args) => {
    const result = runProcessSync({
      program: 'git',
      args,
      cwd: ROOT,
      timeoutMs: 10_000,
      maxOutputBytes: MIB,
      env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' }
    })
    assert.equal(result.code, 0, `Git baseline read failed: ${result.stderr}`)
    return result.stdout
  }
  const baselineRef = process.env.ORCA_CLAUDE_USAGE_APPEND_BENCH_BASELINE ?? 'HEAD'
  const baselineCommit = readGit(['rev-parse', '--verify', `${baselineRef}^{commit}`]).trim()
  const sources = new Map(
    BASELINE_MODULES.map((modulePath) => [
      join(ROOT, ...modulePath.split('/')),
      readGit(['show', `${baselineCommit}:${modulePath}`])
    ])
  )
  return { baselineRef, baselineCommit, sources }
}

export async function loadClaudeUsageBenchmarkScanner(
  home,
  baselineSources = new Map(),
  worktreeSources = new Map(),
  baselineCommit = null
) {
  const sourceFingerprints = {}
  const { runProcessSync } = await import('@orca/process-host')
  const result = await build({
    stdin: {
      contents: `export { scanClaudeUsageFiles } from './src/main/claude-usage/scanner';
        export { resetReadMetrics, readMetrics } from 'orca-benchmark-fs';
        export { open as openForBenchmark } from 'node:fs/promises';
        export { createReadStream as createReadStreamForBenchmark } from 'node:fs';`,
      resolveDir: ROOT,
      loader: 'ts'
    },
    platform: 'node',
    format: 'esm',
    bundle: true,
    write: false,
    plugins: [
      {
        name: 'isolated-transcript-filesystem',
        setup(bundler) {
          bundler.onLoad({ filter: /[/\\]src[/\\].*\.[cm]?[jt]sx?$/ }, async (args) => {
            const modulePath = relative(ROOT, args.path).split(sep).join('/')
            if (!modulePath.startsWith('src/')) {
              return undefined
            }
            if (typeof baselineCommit === 'string' && !baselineSources.has(args.path)) {
              const pinned = runProcessSync({
                program: 'git',
                args: ['show', `${String(baselineCommit)}:${modulePath}`],
                cwd: ROOT,
                timeoutMs: 10_000,
                maxOutputBytes: MIB,
                env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' }
              })
              assert.equal(pinned.code, 0, `Pinned source read failed: ${pinned.stderr}`)
              assert.equal(pinned.outputTruncated, false, `Pinned source truncated: ${modulePath}`)
              baselineSources.set(args.path, pinned.stdout)
            }
            if (!baselineSources.has(args.path) && !worktreeSources.has(args.path)) {
              worktreeSources.set(args.path, readFile(args.path, 'utf8'))
            }
            const contents =
              baselineSources.get(args.path) ?? (await worktreeSources.get(args.path))
            sourceFingerprints[modulePath] = {
              source: baselineSources.has(args.path)
                ? (baselineCommit ?? 'baseline-map')
                : 'worktree',
              sha256: createHash('sha256').update(contents).digest('hex')
            }
            const extension = extname(args.path).slice(1)
            const loader =
              extension === 'tsx' || extension === 'jsx'
                ? extension
                : extension.endsWith('ts')
                  ? 'ts'
                  : 'js'
            return { contents, loader, resolveDir: dirname(args.path) }
          })
          bundler.onResolve(
            { filter: /^node:(fs(?:\/promises)?|os)$|^orca-benchmark-fs$/ },
            (args) => {
              if (args.path === 'orca-benchmark-fs') {
                return { path: 'node:fs', namespace: 'benchmark' }
              }
              if (args.namespace === 'benchmark') {
                return { path: args.path, external: true }
              }
              return { path: args.path, namespace: 'benchmark' }
            }
          )
          bundler.onResolve({ filter: /claude-profile-installed-router$/ }, () => ({
            path: 'profiles',
            namespace: 'benchmark'
          }))
          bundler.onLoad({ filter: /.*/, namespace: 'benchmark' }, (args) => {
            if (args.path === 'profiles') {
              return { contents: 'export const claudeProfileHistoryDirs = () => [];' }
            }
            if (args.path === 'node:os') {
              return {
                contents: `export * from 'node:os';
                  export const homedir = () => ${JSON.stringify(home)};`
              }
            }
            if (args.path === 'node:fs/promises') {
              return {
                contents: `import * as fs from 'node:fs/promises';
                  import { recordStream, recordRead } from 'orca-benchmark-fs';
                  export * from 'node:fs/promises';
                  export function readFile() {
                    throw new Error('Extend benchmark accounting before using fs.promises.readFile');
                  }
                  export async function open(path, ...args) {
                    const handle = await fs.open(path, ...args);
                    return new Proxy(handle, {
                      get(target, key) {
                        if (key === 'readFile' || key === 'readv') {
                          throw new Error('Extend benchmark accounting before using FileHandle.' + key);
                        }
                        const value = Reflect.get(target, key, target);
                        if (key === 'createReadStream') {
                          return (options) => recordStream(path, options, value.call(target, options));
                        }
                        if (key === 'read') {
                          return async (...readArgs) => {
                            const result = await value.apply(target, readArgs);
                            const options = Buffer.isBuffer(readArgs[0]) ? null : readArgs[0];
                            recordRead(path, options?.position ?? readArgs[3] ?? null, result.bytesRead);
                            return result;
                          };
                        }
                        return typeof value === 'function' ? value.bind(target) : value;
                      }
                    });
                  }`
              }
            }
            return {
              contents: `import * as fs from 'node:fs';
                export * from 'node:fs';
                let reads = [];
                export function readFile() {
                  throw new Error('Extend benchmark accounting before using fs.readFile');
                }
                export function readFileSync() {
                  throw new Error('Extend benchmark accounting before using fs.readFileSync');
                }
                export function recordStream(path, options, stream) {
                  reads.push({ stream, kind: 'stream', path: String(path), start: options?.start ?? 0,
                    end: options?.end ?? null });
                  return stream;
                }
                export function recordRead(path, start, bytes) {
                  reads.push({ kind: 'read', path: String(path), start, bytes });
                }
                export function createReadStream(path, options) {
                  return recordStream(path, options, fs.createReadStream(path, options));
                }
                export function resetReadMetrics() { reads = []; }
                export function readMetrics() {
                  return reads.map(({ stream, ...range }) => ({ ...range, bytes: stream?.bytesRead ?? range.bytes }));
                }`
            }
          })
        }
      }
    ]
  })
  return {
    bundleSha256: createHash('sha256').update(result.outputFiles[0].text).digest('hex'),
    scanner: await import(
      `data:text/javascript;base64,${Buffer.from(result.outputFiles[0].text).toString('base64')}`
    ),
    sourceFingerprints: Object.fromEntries(
      Object.entries(sourceFingerprints).sort(([left], [right]) => left.localeCompare(right))
    )
  }
}

export async function verifyClaudeUsageBenchmarkReadAccounting(scanner, transcriptPath) {
  const payload = Buffer.from('工具結果 🌊'.repeat(5_000))
  await writeFile(transcriptPath, payload)
  scanner.resetReadMetrics()
  const handle = await scanner.openForBenchmark(transcriptPath, 'r')
  try {
    await handle.read(Buffer.alloc(11), 0, 11, 1)
    const stream = handle.createReadStream({ start: 0, end: payload.length - 1, autoClose: false })
    for await (const chunk of stream) {
      void chunk
    }
    const reads = scanner.readMetrics()
    assert.equal(
      reads.reduce((sum, read) => sum + read.bytes, 0),
      payload.length + 11
    )
    assert.equal(reads.filter((read) => read.kind === 'read').length, 1)
    assert.equal(reads.filter((read) => read.kind === 'stream').length, 1)
  } finally {
    await handle.close()
  }
  for await (const chunk of scanner.createReadStreamForBenchmark(transcriptPath)) {
    void chunk
  }
  assert.equal(
    scanner.readMetrics().reduce((sum, read) => sum + read.bytes, 0),
    payload.length * 2 + 11
  )
}

export async function measureClaudeUsageBenchmarkScan(
  scanner,
  transcriptPaths,
  previousProcessedFiles = []
) {
  scanner.resetReadMetrics()
  const start = performance.now()
  const result = await scanner.scanClaudeUsageFiles([], previousProcessedFiles, undefined, [])
  const durationMs = performance.now() - start
  const reads = scanner.readMetrics()
  const allowedPaths = new Set(
    typeof transcriptPaths === 'string' ? [transcriptPaths] : transcriptPaths
  )
  assert(
    reads.every((read) => allowedPaths.has(read.path)),
    'scanner read an unexpected file'
  )
  const serializationStart = performance.now()
  const serializedCache = JSON.stringify(result.processedFiles)
  const cacheSerializationMs = performance.now() - serializationStart
  return {
    result,
    cache: JSON.parse(serializedCache),
    metrics: {
      durationMs,
      cacheSerializationMs,
      totalDurationMs: durationMs + cacheSerializationMs,
      bytesRead: reads.reduce((sum, read) => sum + read.bytes, 0),
      directReadBytes: reads
        .filter((read) => read.kind === 'read')
        .reduce((sum, read) => sum + read.bytes, 0),
      readCount: reads.length,
      streamCount: reads.filter((read) => read.kind === 'stream').length,
      cachedStateBytes: Buffer.byteLength(serializedCache)
    }
  }
}