feat: add cli benchmark history commands

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

ai_evals/README.md

@@ -60,6 +60,8 @@ bun run cli -- snapshot-variant --surface cli --variant candidate
 bun run cli -- list-cases --surface cli
 bun run cli -- run --surface cli --case bun-hello-script --variant baseline --runs 5
 bun run cli -- compare --surface cli --case bun-hello-script --variant baseline --variant baseline --runs 5
+bun run cli -- compare --surface cli --case bun-hello-script --variant baseline-frozen --variant candidate --runs 5 --write-history
+bun run cli -- history --view latest
 ```
 
 At the moment this is still intentionally small, but it is the only benchmark

ai_evals/adapters/cli/runtime.ts

@@ -17,6 +17,8 @@ export interface PromptRunResult {
 }
 
 const REPO_ROOT = fileURLToPath(new URL("../../../", import.meta.url));
+export const CLI_BENCHMARK_PROVIDER = "anthropic";
+export const CLI_BENCHMARK_MODEL = "haiku";
 
 export function getGeneratedSkillsSource(): string {
   return join(REPO_ROOT, "system_prompts", "auto-generated", "skills");
@@ -35,7 +37,7 @@ export async function runPromptAndCapture(
 
   const options: Options = {
     cwd,
-    model: "haiku",
+    model: CLI_BENCHMARK_MODEL,
     maxTurns,
     settingSources: ["project"],
     allowedTools: ["Skill", "Read", "Glob", "Grep", "Bash", "Write", "Edit"]

ai_evals/cli/README.md

@@ -9,6 +9,7 @@ The current implementation is intentionally small:
 
 - `run` command
 - `compare` command
+- `history` command
 - `list-cases` and `list-variants` discovery commands
 - `cli` surface only
 
@@ -83,6 +84,21 @@ cd ai_evals
 bun run cli -- compare --surface cli --case bun-hello-script --variant baseline --variant baseline --json
 ```
 
+Write official benchmark snapshots while comparing distinct variants:
+
+```bash
+cd ai_evals
+bun run cli -- compare --surface cli --case bun-hello-script --variant baseline-frozen --variant candidate --runs 5 --write-history
+```
+
+Inspect the tracked history:
+
+```bash
+cd ai_evals
+bun run cli -- history --view latest
+bun run cli -- history --view summary --limit 10
+```
+
 ## Benchmarking A CLI Skill Change
 
 If you change one of the generated CLI skills and want to know whether the
@@ -130,6 +146,15 @@ for repeated runs:
 - average skill-invocation count
 - aggregated required-check failures
 
+When `--write-history` is used on `compare`, the benchmark CLI also writes one
+official snapshot per compared variant into `ai_evals/history/` and rebuilds:
+
+- `summary.jsonl`
+- `rollups/latest.json`
+- `rollups/by_surface.json`
+- `rollups/by_variant.json`
+- `rollups/by_model.json`
+
 The compare output also includes tool usage and invoked skills as diagnostics.
 
 True efficiency metrics such as latency, token usage, and cost are planned, but
@@ -168,8 +193,7 @@ instead of the generated default.
 
 Later iterations should add:
 
-- `history` command
 - frontend adapters
 - variant cleanup and diff helpers
 - token and cost metrics in compare output
-- shared result/history writing from this entrypoint
+- richer history views and filtering

ai_evals/cli/index.ts

@@ -1,16 +1,29 @@
 #!/usr/bin/env bun
 
+import { execFileSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
 import {
   cleanupWorkspace,
   loadCliArtifactEvalCases,
   runCliArtifactEvalCase
 } from "../adapters/cli/artifact-eval";
+import {
+  CLI_BENCHMARK_MODEL,
+  CLI_BENCHMARK_PROVIDER,
+} from "../adapters/cli/runtime";
 import {
   loadCliVariantById,
   loadCliVariants,
   snapshotCliVariant,
   type CliVariant
 } from "../adapters/cli/variants";
+import {
+  appendOfficialRun,
+  DEFAULT_HISTORY_DIR,
+  loadHistoryRollup,
+  loadLatestHistoryRollup,
+  loadSummaryHistory
+} from "../history/writer.mjs";
 
 type CommandName =
   | "run"
@@ -30,6 +43,10 @@ interface ParsedArgs {
   runs: number;
   json: boolean;
   keepWorkspace: boolean;
+  writeHistory: boolean;
+  historyDir?: string;
+  historyView: "latest" | "summary" | "surface" | "variant" | "model";
+  limit: number;
 }
 
 interface AttemptSummary {
@@ -44,6 +61,7 @@ interface AttemptSummary {
   checks: Array<{ name: string; passed: boolean; required?: boolean }>;
   requiredFailedChecks: string[];
   expectedFiles: Array<{ path: string; exists: boolean }>;
+  pathSignature: string;
 }
 
 interface AggregateMetrics {
@@ -51,6 +69,8 @@ interface AggregateMetrics {
   passedRuns: number;
   passRate: number;
   averageDurationMs: number;
+  medianDurationMs: number;
+  latencyPerSuccessMs: number;
   averageAssistantMessages: number;
   averageToolCalls: number;
   averageSkillInvocations: number;
@@ -59,6 +79,43 @@ interface AggregateMetrics {
   requiredFailureCounts: Array<{ name: string; count: number }>;
 }
 
+interface CompareCaseResult {
+  caseId: string;
+  totalRuns: number;
+  passedRuns: number;
+  passRate: number;
+  averageDurationMs: number;
+  medianDurationMs: number;
+  latencyPerSuccessMs: number;
+  averageAssistantMessages: number;
+  averageToolCalls: number;
+  averageSkillInvocations: number;
+  pathConsistency: number;
+  distinctSkillsInvoked: string[];
+  distinctToolsUsed: string[];
+  requiredFailureCounts: Array<{ name: string; count: number }>;
+}
+
+interface CompareVariantResult {
+  label: string;
+  variant: string;
+  totalCases: number;
+  fullyPassedCases: number;
+  totalRuns: number;
+  passedRuns: number;
+  passRate: number;
+  averageDurationMs: number;
+  medianDurationMs: number;
+  latencyPerSuccessMs: number;
+  averageAssistantMessages: number;
+  averageToolCalls: number;
+  averageSkillInvocations: number;
+  distinctSkillsInvoked: string[];
+  distinctToolsUsed: string[];
+  requiredFailureCounts: Array<{ name: string; count: number }>;
+  caseResults: CompareCaseResult[];
+}
+
 async function main() {
   const args = parseArgs(process.argv.slice(2));
 
@@ -79,9 +136,8 @@ async function main() {
       await handleCompare(args);
       return;
     case "history":
-      throw new Error(
-        `'${args.command}' is not implemented yet in the repo-level benchmark CLI`
-      );
+      await handleHistory(args);
+      return;
     default:
       printHelp();
       throw new Error(`Unknown command: ${String(args.command)}`);
@@ -157,6 +213,9 @@ async function handleRun(args: ParsedArgs) {
   if (args.keepWorkspace && args.runs !== 1) {
     throw new Error("--keep-workspace is only supported when --runs is 1");
   }
+  if (args.writeHistory) {
+    throw new Error("--write-history is currently supported on compare only");
+  }
 
   switch (surface) {
     case "cli": {
@@ -265,11 +324,16 @@ async function handleCompare(args: ParsedArgs) {
       const variants = await Promise.all(
         args.variantIds.map((variantId) => loadCliVariantById(variantId))
       );
+      if (args.writeHistory && new Set(variants.map((variant) => variant.id)).size !== variants.length) {
+        throw new Error(
+          "--write-history requires distinct variant ids so official snapshots stay unambiguous"
+        );
+      }
       const labeledVariants = labelVariantSelections(variants);
-      const variantResults = [];
+      const internalVariantResults: Array<CompareVariantResult & { allAttempts: AttemptSummary[] }> = [];
 
       for (const labeledVariant of labeledVariants) {
-        const caseResults = [];
+        const caseResults: CompareCaseResult[] = [];
         const allAttempts: AttemptSummary[] = [];
 
         for (const evalCase of selectedCases) {
@@ -286,9 +350,12 @@ async function handleCompare(args: ParsedArgs) {
             passedRuns: execution.aggregate.passedRuns,
             passRate: execution.aggregate.passRate,
             averageDurationMs: execution.aggregate.averageDurationMs,
+            medianDurationMs: execution.aggregate.medianDurationMs,
+            latencyPerSuccessMs: execution.aggregate.latencyPerSuccessMs,
             averageAssistantMessages: execution.aggregate.averageAssistantMessages,
             averageToolCalls: execution.aggregate.averageToolCalls,
             averageSkillInvocations: execution.aggregate.averageSkillInvocations,
+            pathConsistency: computePathConsistency(execution.attempts),
             distinctSkillsInvoked: execution.aggregate.distinctSkillsInvoked,
             distinctToolsUsed: execution.aggregate.distinctToolsUsed,
             requiredFailureCounts: execution.aggregate.requiredFailureCounts
@@ -297,7 +364,7 @@ async function handleCompare(args: ParsedArgs) {
 
         const aggregate = aggregateAttempts(allAttempts);
 
-        variantResults.push({
+        internalVariantResults.push({
           label: labeledVariant.label,
           variant: labeledVariant.variant.id,
           totalCases: caseResults.length,
@@ -308,22 +375,35 @@ async function handleCompare(args: ParsedArgs) {
           passedRuns: aggregate.passedRuns,
           passRate: aggregate.passRate,
           averageDurationMs: aggregate.averageDurationMs,
+          medianDurationMs: aggregate.medianDurationMs,
+          latencyPerSuccessMs: aggregate.latencyPerSuccessMs,
           averageAssistantMessages: aggregate.averageAssistantMessages,
           averageToolCalls: aggregate.averageToolCalls,
           averageSkillInvocations: aggregate.averageSkillInvocations,
           distinctSkillsInvoked: aggregate.distinctSkillsInvoked,
           distinctToolsUsed: aggregate.distinctToolsUsed,
           requiredFailureCounts: aggregate.requiredFailureCounts,
-          caseResults
+          caseResults,
+          allAttempts
         });
       }
 
+      const historyWrites = args.writeHistory
+        ? await writeCompareHistorySnapshots({
+            surface,
+            runs: args.runs,
+            variants: internalVariantResults,
+            historyDir: args.historyDir
+          })
+        : [];
+
       const payload = {
         command: "compare",
         surface,
         caseIds: selectedCases.map((entry) => entry.id),
         runs: args.runs,
-        variants: variantResults
+        variants: internalVariantResults.map(({ allAttempts: _allAttempts, ...variant }) => variant),
+        historyWrites
       };
 
       if (args.json) {
@@ -339,6 +419,62 @@ async function handleCompare(args: ParsedArgs) {
   }
 }
 
+async function handleHistory(args: ParsedArgs) {
+  const historyDir = args.historyDir ?? DEFAULT_HISTORY_DIR;
+
+  switch (args.historyView) {
+    case "latest": {
+      const payload = await loadLatestHistoryRollup(historyDir);
+      if (args.json) {
+        process.stdout.write(JSON.stringify(payload, null, 2) + "\n");
+      } else {
+        printHistoryLatestSummary(payload);
+      }
+      return;
+    }
+    case "summary": {
+      const summaries = await loadSummaryHistory(historyDir);
+      const payload = {
+        view: "summary",
+        historyDir,
+        totalEntries: summaries.length,
+        entries: summaries.slice(-args.limit).reverse()
+      };
+      if (args.json) {
+        process.stdout.write(JSON.stringify(payload, null, 2) + "\n");
+      } else {
+        printHistorySummary(payload);
+      }
+      return;
+    }
+    case "surface":
+    case "variant":
+    case "model": {
+      const filename =
+        args.historyView === "surface"
+          ? "by_surface.json"
+          : args.historyView === "variant"
+            ? "by_variant.json"
+            : "by_model.json";
+      const rollup = await loadHistoryRollup(filename, historyDir);
+      const payload = {
+        view: args.historyView,
+        historyDir,
+        generatedAt: rollup.generatedAt,
+        groups: rollup.groups
+      };
+      if (args.json) {
+        process.stdout.write(JSON.stringify(payload, null, 2) + "\n");
+      } else {
+        printHistoryGroupSummary(payload, args.limit);
+      }
+      return;
+    }
+    default:
+      assertNever(args.historyView);
+  }
+}
+
 async function runCaseAttempts(
   evalCase: Awaited<ReturnType<typeof loadCliArtifactEvalCases>>[number],
   variant: CliVariant,
@@ -374,6 +510,9 @@ function summarizeAttempt(
   result: Awaited<ReturnType<typeof runCliArtifactEvalCase>>,
   attempt: number
 ): AttemptSummary {
+  const skillsInvoked = uniqueStrings(result.run.skillsInvoked);
+  const toolsUsed = uniqueStrings(result.run.toolsUsed.map((tool) => tool.tool));
+
   return {
     attempt,
     passed: result.passed,
@@ -381,8 +520,8 @@ function summarizeAttempt(
     assistantMessageCount: result.run.assistantMessageCount,
     toolCallCount: result.run.toolsUsed.length,
     skillInvocationCount: result.run.skillsInvoked.length,
-    skillsInvoked: uniqueStrings(result.run.skillsInvoked),
-    toolsUsed: uniqueStrings(result.run.toolsUsed.map((tool) => tool.tool)),
+    skillsInvoked,
+    toolsUsed,
     checks: result.checks.map((check) => ({
       name: check.name,
       passed: check.passed,
@@ -394,12 +533,14 @@ function summarizeAttempt(
     expectedFiles: result.expectedFiles.map((file) => ({
       path: file.path,
       exists: file.exists
-    }))
+    })),
+    pathSignature: buildPathSignature(skillsInvoked, toolsUsed)
   };
 }
 
 function aggregateAttempts(attempts: AttemptSummary[]): AggregateMetrics {
   const requiredFailureCounts = new Map<string, number>();
+  const successfulAttempts = attempts.filter((attempt) => attempt.passed);
 
   for (const attempt of attempts) {
     for (const failure of attempt.requiredFailedChecks) {
@@ -412,6 +553,8 @@ function aggregateAttempts(attempts: AttemptSummary[]): AggregateMetrics {
     passedRuns: attempts.filter((attempt) => attempt.passed).length,
     passRate: attempts.length === 0 ? 0 : attempts.filter((attempt) => attempt.passed).length / attempts.length,
     averageDurationMs: average(attempts.map((attempt) => attempt.durationMs)),
+    medianDurationMs: median(attempts.map((attempt) => attempt.durationMs)),
+    latencyPerSuccessMs: average(successfulAttempts.map((attempt) => attempt.durationMs)),
     averageAssistantMessages: average(attempts.map((attempt) => attempt.assistantMessageCount)),
     averageToolCalls: average(attempts.map((attempt) => attempt.toolCallCount)),
     averageSkillInvocations: average(attempts.map((attempt) => attempt.skillInvocationCount)),
@@ -442,7 +585,11 @@ function parseArgs(argv: string[]): ParsedArgs {
     description: undefined,
     runs: 1,
     json: false,
-    keepWorkspace: false
+    keepWorkspace: false,
+    writeHistory: false,
+    historyDir: undefined,
+    historyView: "latest",
+    limit: 10
   };
 
   for (let index = 0; index < rest.length; index += 1) {
@@ -478,6 +625,29 @@ function parseArgs(argv: string[]): ParsedArgs {
       continue;
     }
 
+    if (arg === "--write-history") {
+      parsed.writeHistory = true;
+      continue;
+    }
+
+    if (arg === "--history-dir") {
+      parsed.historyDir = rest[index + 1];
+      index += 1;
+      continue;
+    }
+
+    if (arg === "--view") {
+      parsed.historyView = parseHistoryView(rest[index + 1]);
+      index += 1;
+      continue;
+    }
+
+    if (arg === "--limit") {
+      parsed.limit = parsePositiveInteger(rest[index + 1], "--limit");
+      index += 1;
+      continue;
+    }
+
     if (arg === "--json") {
       parsed.json = true;
       continue;
@@ -556,6 +726,22 @@ function parsePositiveInteger(value: string | undefined, flagName: string): numb
   return parsed;
 }
 
+function parseHistoryView(
+  value: string | undefined
+): ParsedArgs["historyView"] {
+  if (
+    value === "latest" ||
+    value === "summary" ||
+    value === "surface" ||
+    value === "variant" ||
+    value === "model"
+  ) {
+    return value;
+  }
+
+  throw new Error("--view must be one of: latest, summary, surface, variant, model");
+}
+
 function printRunSummary(payload: {
   surface: SurfaceName;
   variant: string;
@@ -626,30 +812,8 @@ function printCompareSummary(payload: {
   surface: SurfaceName;
   caseIds: string[];
   runs: number;
-  variants: Array<{
-    label: string;
-    variant: string;
-    totalCases: number;
-    fullyPassedCases: number;
-    totalRuns: number;
-    passedRuns: number;
-    passRate: number;
-    averageDurationMs: number;
-    averageAssistantMessages: number;
-    averageToolCalls: number;
-    averageSkillInvocations: number;
-    caseResults: Array<{
-      caseId: string;
-      totalRuns: number;
-      passedRuns: number;
-      passRate: number;
-      averageDurationMs: number;
-      averageAssistantMessages: number;
-      averageToolCalls: number;
-      averageSkillInvocations: number;
-      requiredFailureCounts: Array<{ name: string; count: number }>;
-    }>;
-  }>;
+  variants: CompareVariantResult[];
+  historyWrites: Array<{ variant: string; runId: string; runPath: string }>;
 }) {
   process.stdout.write(`Surface: ${payload.surface}\n`);
   process.stdout.write(`Cases: ${payload.caseIds.join(", ")}\n`);
@@ -669,6 +833,13 @@ function printCompareSummary(payload: {
       }
     }
   }
+
+  if (payload.historyWrites.length > 0) {
+    process.stdout.write("History writes:\n");
+    for (const write of payload.historyWrites) {
+      process.stdout.write(`- ${write.variant}: ${write.runPath} (${write.runId})\n`);
+    }
+  }
 }
 
 function printSnapshotSummary(payload: {
@@ -716,6 +887,241 @@ function labelVariantSelections(
   });
 }
 
+async function writeCompareHistorySnapshots(input: {
+  surface: SurfaceName;
+  runs: number;
+  variants: Array<CompareVariantResult & { allAttempts: AttemptSummary[] }>;
+  historyDir?: string;
+}) {
+  const timestamp = new Date().toISOString();
+  const gitSha = getGitSha();
+  const writes = [];
+
+  for (const variant of input.variants) {
+    const result = await appendOfficialRun(
+      buildOfficialRun({
+        timestamp,
+        gitSha,
+        surface: input.surface,
+        runs: input.runs,
+        variant
+      }),
+      {
+        historyDir: input.historyDir
+      }
+    );
+
+    writes.push({
+      variant: variant.variant,
+      runId: result.runId,
+      runPath: result.runPath
+    });
+  }
+
+  return writes;
+}
+
+function buildOfficialRun(input: {
+  timestamp: string;
+  gitSha: string;
+  surface: SurfaceName;
+  runs: number;
+  variant: CompareVariantResult & { allAttempts: AttemptSummary[] };
+}) {
+  const flakeRate =
+    input.variant.caseResults.length === 0
+      ? 0
+      : input.variant.caseResults.filter(
+          (entry) => entry.passRate > 0 && entry.passRate < 1
+        ).length / input.variant.caseResults.length;
+  const pathConsistency = average(
+    input.variant.caseResults.map((entry) => entry.pathConsistency)
+  );
+  const qualityScore = input.variant.passRate * 100;
+  const efficiencyScore = computeEfficiencyScore(input.variant);
+
+  return {
+    timestamp: input.timestamp,
+    git_sha: input.gitSha,
+    suite_version: "cli-benchmark-v1",
+    scoring_version: "cli-deterministic-v1",
+    surface: input.surface,
+    variant_name: input.variant.variant,
+    provider: CLI_BENCHMARK_PROVIDER,
+    model: CLI_BENCHMARK_MODEL,
+    judge_model: null,
+    runs_per_case: input.runs,
+    case_count: input.variant.caseResults.length,
+    metrics: {
+      quality: {
+        pass_rate: input.variant.passRate,
+        deterministic_pass_rate: input.variant.passRate,
+        judge_score_mean: 0,
+        judge_score_median: 0,
+        judge_score_p10: 0,
+        quality_score: qualityScore
+      },
+      reliability: {
+        runs_per_case: input.runs,
+        flake_rate: flakeRate,
+        path_consistency: pathConsistency
+      },
+      efficiency: {
+        latency_ms_mean: input.variant.averageDurationMs,
+        latency_ms_median: input.variant.medianDurationMs,
+        tokens_total_mean: 0,
+        tool_calls_mean: input.variant.averageToolCalls,
+        iterations_mean: input.variant.averageAssistantMessages,
+        estimated_cost_mean: 0,
+        cost_per_success: 0,
+        latency_per_success: input.variant.latencyPerSuccessMs,
+        efficiency_score: efficiencyScore,
+        value_score: (qualityScore * efficiencyScore) / 100
+      }
+    },
+    cases: input.variant.caseResults.map((entry) => ({
+      id: entry.caseId,
+      pass_rate: entry.passRate,
+      average_duration_ms: entry.averageDurationMs,
+      median_duration_ms: entry.medianDurationMs,
+      latency_per_success_ms: entry.latencyPerSuccessMs,
+      average_assistant_messages: entry.averageAssistantMessages,
+      average_tool_calls: entry.averageToolCalls,
+      average_skill_invocations: entry.averageSkillInvocations,
+      path_consistency: entry.pathConsistency,
+      distinct_skills_invoked: entry.distinctSkillsInvoked,
+      distinct_tools_used: entry.distinctToolsUsed,
+      required_failure_counts: entry.requiredFailureCounts
+    }))
+  };
+}
+
+function computeEfficiencyScore(variant: CompareVariantResult): number {
+  const latencyFactor = 1 / (1 + variant.averageDurationMs / 20000);
+  const toolFactor = 1 / (1 + variant.averageToolCalls / 10);
+  const iterationFactor = 1 / (1 + variant.averageAssistantMessages / 10);
+
+  return ((latencyFactor + toolFactor + iterationFactor) / 3) * 100;
+}
+
+function getGitSha(): string {
+  try {
+    return execFileSync("git", ["rev-parse", "HEAD"], {
+      cwd: fileURLToPath(new URL("../..", import.meta.url)),
+      encoding: "utf8"
+    }).trim();
+  } catch {
+    return "0000000";
+  }
+}
+
+function buildPathSignature(skillsInvoked: string[], toolsUsed: string[]): string {
+  return `skills:${skillsInvoked.join(",") || "(none)"}|tools:${toolsUsed.join(",") || "(none)"}`;
+}
+
+function computePathConsistency(attempts: AttemptSummary[]): number {
+  if (attempts.length === 0) {
+    return 0;
+  }
+
+  const counts = new Map<string, number>();
+  for (const attempt of attempts) {
+    counts.set(attempt.pathSignature, (counts.get(attempt.pathSignature) ?? 0) + 1);
+  }
+
+  return Math.max(...counts.values()) / attempts.length;
+}
+
+function printHistoryLatestSummary(payload: {
+  generatedAt: string;
+  latestRun: null | {
+    timestamp: string;
+    surface: string;
+    variant_name: string;
+    provider: string;
+    model: string;
+    metrics: {
+      quality: { pass_rate: number };
+      efficiency: { latency_ms_mean: number };
+    };
+  };
+  latestBySurface: Record<string, { variant_name: string; metrics: { quality: { pass_rate: number } } }>;
+}) {
+  process.stdout.write(`Generated: ${payload.generatedAt}\n`);
+  if (!payload.latestRun) {
+    process.stdout.write("Latest run: none\n");
+    return;
+  }
+
+  process.stdout.write(
+    `Latest run: ${payload.latestRun.timestamp} | ${payload.latestRun.surface} | ${payload.latestRun.variant_name} | ${formatPercent(payload.latestRun.metrics.quality.pass_rate)} | avg ${formatNumber(payload.latestRun.metrics.efficiency.latency_ms_mean)} ms\n`
+  );
+  process.stdout.write("Latest by surface:\n");
+  for (const [surface, summary] of Object.entries(payload.latestBySurface)) {
+    process.stdout.write(
+      `- ${surface}: ${summary.variant_name} (${formatPercent(summary.metrics.quality.pass_rate)})\n`
+    );
+  }
+}
+
+function printHistorySummary(payload: {
+  view: "summary";
+  historyDir: string;
+  totalEntries: number;
+  entries: Array<{
+    timestamp: string;
+    surface: string;
+    variant_name: string;
+    provider: string;
+    model: string;
+    runs_per_case: number;
+    case_count: number;
+    metrics: {
+      quality: { pass_rate: number };
+      reliability: { flake_rate: number };
+      efficiency: { latency_ms_mean: number };
+    };
+  }>;
+}) {
+  process.stdout.write(`History: ${payload.historyDir}\n`);
+  process.stdout.write(`Entries: ${payload.totalEntries}\n`);
+  for (const entry of payload.entries) {
+    process.stdout.write(
+      `- ${entry.timestamp} | ${entry.surface} | ${entry.variant_name} | ${formatPercent(entry.metrics.quality.pass_rate)} | flake ${formatPercent(entry.metrics.reliability.flake_rate)} | avg ${formatNumber(entry.metrics.efficiency.latency_ms_mean)} ms | ${entry.case_count} cases x ${entry.runs_per_case} runs\n`
+    );
+  }
+}
+
+function printHistoryGroupSummary(
+  payload: {
+    view: "surface" | "variant" | "model";
+    historyDir: string;
+    generatedAt: string;
+    groups: Record<string, Array<{
+      timestamp: string;
+      variant_name: string;
+      surface: string;
+      metrics: {
+        quality: { pass_rate: number };
+        efficiency: { latency_ms_mean: number };
+      };
+    }>>;
+  },
+  limit: number
+) {
+  process.stdout.write(`History: ${payload.historyDir}\n`);
+  process.stdout.write(`View: ${payload.view}\n`);
+  process.stdout.write(`Generated: ${payload.generatedAt}\n`);
+  for (const [group, entries] of Object.entries(payload.groups)
+    .sort((left, right) => left[0].localeCompare(right[0]))
+    .slice(0, limit)) {
+    const latest = entries[entries.length - 1];
+    process.stdout.write(
+      `- ${group}: ${entries.length} runs | latest ${latest.timestamp} | ${latest.variant_name} | ${formatPercent(latest.metrics.quality.pass_rate)} | avg ${formatNumber(latest.metrics.efficiency.latency_ms_mean)} ms\n`
+    );
+  }
+}
+
 function printHelp() {
   process.stdout.write(
     [
@@ -724,8 +1130,8 @@ function printHelp() {
       "  cd ai_evals && bun run cli -- list-variants --surface cli [--json]",
       "  cd ai_evals && bun run cli -- snapshot-variant --surface cli --variant <id> [--description <text>] [--json]",
       "  cd ai_evals && bun run cli -- run --surface cli --case <id> [--variant <id>] [--runs <n>] [--json] [--keep-workspace]",
-      "  cd ai_evals && bun run cli -- compare --surface cli [--case <id> ...] [--variant <id> ...] [--runs <n>] [--json]",
-      "  cd ai_evals && bun run cli -- history",
+      "  cd ai_evals && bun run cli -- compare --surface cli [--case <id> ...] [--variant <id> ...] [--runs <n>] [--write-history] [--history-dir <path>] [--json]",
+      "  cd ai_evals && bun run cli -- history [--view latest|summary|surface|variant|model] [--limit <n>] [--history-dir <path>] [--json]",
       "",
       "Current support:",
       "  surfaces: cli"
@@ -744,6 +1150,21 @@ function average(values: number[]): number {
   return values.reduce((sum, value) => sum + value, 0) / values.length;
 }
 
+function median(values: number[]): number {
+  if (values.length === 0) {
+    return 0;
+  }
+
+  const sorted = [...values].sort((left, right) => left - right);
+  const middle = Math.floor(sorted.length / 2);
+
+  if (sorted.length % 2 === 0) {
+    return (sorted[middle - 1] + sorted[middle]) / 2;
+  }
+
+  return sorted[middle];
+}
+
 function uniqueStrings(values: string[]): string[] {
   return [...new Set(values)].sort((left, right) => left.localeCompare(right));
 }

ai_evals/history/writer.mjs

@@ -0,0 +1,430 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url));
+
+export const DEFAULT_HISTORY_DIR = MODULE_DIR;
+const SUMMARY_FILENAME = "summary.jsonl";
+const RUNS_DIRNAME = "runs";
+const ROLLUPS_DIRNAME = "rollups";
+
+export async function appendOfficialRun(input, options = {}) {
+  const absoluteHistoryDir = path.resolve(options.historyDir ?? DEFAULT_HISTORY_DIR);
+  const normalizedRun = normalizeRun(input);
+  await ensureHistoryLayout(absoluteHistoryDir);
+
+  const runFilename = `${normalizedRun.run_id}.json`;
+  const runRelativePath = path.posix.join(RUNS_DIRNAME, runFilename);
+  const runFilePath = path.join(absoluteHistoryDir, runRelativePath);
+
+  await writeJsonFile(runFilePath, normalizedRun);
+
+  const summaryPath = path.join(absoluteHistoryDir, SUMMARY_FILENAME);
+  const summaries = await loadSummaryEntries(summaryPath);
+  const summaryEntry = buildSummaryEntry(normalizedRun, runRelativePath);
+  const nextSummaries = upsertSummaryEntry(summaries, summaryEntry);
+
+  await writeSummaryEntries(summaryPath, nextSummaries);
+  await writeRollups(absoluteHistoryDir, nextSummaries);
+
+  return {
+    status: "ok",
+    runId: normalizedRun.run_id,
+    runPath: runRelativePath,
+    summaryEntries: nextSummaries.length
+  };
+}
+
+export async function loadLatestHistoryRollup(historyDir = DEFAULT_HISTORY_DIR) {
+  const rollupPath = path.join(path.resolve(historyDir), ROLLUPS_DIRNAME, "latest.json");
+  return await loadJsonFile(rollupPath);
+}
+
+export async function loadHistoryRollup(
+  name,
+  historyDir = DEFAULT_HISTORY_DIR
+) {
+  const rollupPath = path.join(path.resolve(historyDir), ROLLUPS_DIRNAME, name);
+  return await loadJsonFile(rollupPath);
+}
+
+export async function loadSummaryHistory(historyDir = DEFAULT_HISTORY_DIR) {
+  const summaryPath = path.join(path.resolve(historyDir), SUMMARY_FILENAME);
+  return await loadSummaryEntries(summaryPath);
+}
+
+async function loadJsonFile(filePath) {
+  const raw = await readFile(filePath, "utf8");
+  try {
+    return JSON.parse(raw);
+  } catch (error) {
+    throw new Error(`Failed to parse JSON from ${filePath}: ${error.message}`);
+  }
+}
+
+function normalizeRun(input) {
+  assertPlainObject(input, "benchmark run");
+
+  const timestamp = assertIsoDateTime(input.timestamp, "timestamp");
+  const gitSha = assertNonEmptyString(input.git_sha, "git_sha");
+  const suiteVersion = assertNonEmptyString(input.suite_version, "suite_version");
+  const scoringVersion = assertNonEmptyString(
+    input.scoring_version,
+    "scoring_version"
+  );
+  const surface = assertNonEmptyString(input.surface, "surface");
+  const variantName = assertNonEmptyString(input.variant_name, "variant_name");
+  const provider = assertNonEmptyString(input.provider, "provider");
+  const model = assertNonEmptyString(input.model, "model");
+  const judgeModel =
+    input.judge_model === null || input.judge_model === undefined
+      ? null
+      : assertNonEmptyString(input.judge_model, "judge_model");
+  const runsPerCase = assertPositiveInteger(input.runs_per_case, "runs_per_case");
+  const caseCount = assertPositiveInteger(input.case_count, "case_count");
+  const metrics = normalizeMetrics(input.metrics, runsPerCase);
+  const cases = normalizeCases(input.cases);
+
+  if (cases.length !== caseCount) {
+    throw new Error(
+      `case_count (${caseCount}) does not match cases.length (${cases.length})`
+    );
+  }
+
+  const runId =
+    input.run_id && typeof input.run_id === "string" && input.run_id.trim()
+      ? input.run_id.trim()
+      : buildRunId({
+          timestamp,
+          surface,
+          variantName,
+          provider,
+          model,
+          gitSha
+        });
+
+  return {
+    ...input,
+    run_id: runId,
+    timestamp,
+    git_sha: gitSha,
+    suite_version: suiteVersion,
+    scoring_version: scoringVersion,
+    surface,
+    variant_name: variantName,
+    provider,
+    model,
+    judge_model: judgeModel,
+    runs_per_case: runsPerCase,
+    case_count: caseCount,
+    metrics,
+    cases
+  };
+}
+
+function normalizeMetrics(input, runsPerCase) {
+  assertPlainObject(input, "metrics");
+
+  const quality = normalizeMetricGroup(
+    input.quality,
+    "metrics.quality",
+    [
+      "pass_rate",
+      "deterministic_pass_rate",
+      "judge_score_mean",
+      "judge_score_median",
+      "judge_score_p10",
+      "quality_score"
+    ],
+    new Set(["pass_rate", "deterministic_pass_rate"])
+  );
+  const reliability = normalizeMetricGroup(
+    input.reliability,
+    "metrics.reliability",
+    ["runs_per_case", "flake_rate", "path_consistency"],
+    new Set(["flake_rate", "path_consistency"])
+  );
+  const efficiency = normalizeMetricGroup(
+    input.efficiency,
+    "metrics.efficiency",
+    [
+      "latency_ms_mean",
+      "latency_ms_median",
+      "tokens_total_mean",
+      "tool_calls_mean",
+      "iterations_mean",
+      "estimated_cost_mean",
+      "cost_per_success",
+      "latency_per_success",
+      "efficiency_score",
+      "value_score"
+    ]
+  );
+
+  if (reliability.runs_per_case !== runsPerCase) {
+    throw new Error(
+      `metrics.reliability.runs_per_case (${reliability.runs_per_case}) does not match runs_per_case (${runsPerCase})`
+    );
+  }
+
+  if (quality.category_pass_rate !== undefined) {
+    assertPlainObject(quality.category_pass_rate, "metrics.quality.category_pass_rate");
+    for (const [category, value] of Object.entries(quality.category_pass_rate)) {
+      assertRatio(value, `metrics.quality.category_pass_rate.${category}`);
+    }
+  }
+
+  return { quality, reliability, efficiency };
+}
+
+function normalizeMetricGroup(input, groupName, requiredFields, ratioFields = new Set()) {
+  assertPlainObject(input, groupName);
+  const normalized = { ...input };
+
+  for (const field of requiredFields) {
+    if (!(field in normalized)) {
+      throw new Error(`Missing required ${groupName}.${field}`);
+    }
+    if (field === "runs_per_case") {
+      normalized[field] = assertPositiveInteger(
+        normalized[field],
+        `${groupName}.${field}`
+      );
+      continue;
+    }
+    normalized[field] = assertFiniteNumber(normalized[field], `${groupName}.${field}`);
+    if (ratioFields.has(field)) {
+      assertRatio(normalized[field], `${groupName}.${field}`);
+    }
+  }
+
+  for (const [field, value] of Object.entries(normalized)) {
+    if (value === null || value === undefined) {
+      continue;
+    }
+    if (typeof value === "number") {
+      normalized[field] = assertFiniteNumber(value, `${groupName}.${field}`);
+      if (ratioFields.has(field)) {
+        assertRatio(normalized[field], `${groupName}.${field}`);
+      }
+    }
+  }
+
+  return normalized;
+}
+
+function normalizeCases(input) {
+  if (!Array.isArray(input) || input.length === 0) {
+    throw new Error("cases must be a non-empty array");
+  }
+
+  return input.map((entry, index) => {
+    assertPlainObject(entry, `cases[${index}]`);
+
+    return {
+      ...entry,
+      id: assertNonEmptyString(entry.id, `cases[${index}].id`),
+      pass_rate: assertRatio(entry.pass_rate, `cases[${index}].pass_rate`)
+    };
+  });
+}
+
+function buildRunId({ timestamp, surface, variantName, provider, model, gitSha }) {
+  const timestampSlug = timestamp.replaceAll(":", "-").replaceAll(".", "-");
+  const shortSha = gitSha.slice(0, 12);
+
+  return [
+    slugify(timestampSlug),
+    slugify(surface),
+    slugify(variantName),
+    slugify(provider),
+    slugify(model),
+    shortSha
+  ]
+    .filter(Boolean)
+    .join("__");
+}
+
+function buildSummaryEntry(run, runRelativePath) {
+  return {
+    run_id: run.run_id,
+    timestamp: run.timestamp,
+    git_sha: run.git_sha,
+    suite_version: run.suite_version,
+    scoring_version: run.scoring_version,
+    surface: run.surface,
+    variant_name: run.variant_name,
+    provider: run.provider,
+    model: run.model,
+    judge_model: run.judge_model,
+    runs_per_case: run.runs_per_case,
+    case_count: run.case_count,
+    run_path: runRelativePath,
+    metrics: run.metrics
+  };
+}
+
+function upsertSummaryEntry(entries, nextEntry) {
+  const remainingEntries = entries.filter((entry) => entry.run_id !== nextEntry.run_id);
+  const nextEntries = [...remainingEntries, nextEntry];
+  nextEntries.sort((left, right) => left.timestamp.localeCompare(right.timestamp));
+  return nextEntries;
+}
+
+async function loadSummaryEntries(summaryPath) {
+  try {
+    const raw = await readFile(summaryPath, "utf8");
+    return raw
+      .split("\n")
+      .map((line) => line.trim())
+      .filter(Boolean)
+      .map((line, index) => {
+        try {
+          return JSON.parse(line);
+        } catch (error) {
+          throw new Error(
+            `Failed to parse ${SUMMARY_FILENAME} line ${index + 1}: ${error.message}`
+          );
+        }
+      });
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      return [];
+    }
+    throw error;
+  }
+}
+
+async function writeSummaryEntries(summaryPath, entries) {
+  const content =
+    entries.map((entry) => JSON.stringify(entry)).join("\n") +
+    (entries.length > 0 ? "\n" : "");
+  await writeFile(summaryPath, content, "utf8");
+}
+
+async function writeRollups(historyDir, summaries) {
+  const rollupsDir = path.join(historyDir, ROLLUPS_DIRNAME);
+  const generatedAt = new Date().toISOString();
+  const latestFirst = [...summaries].sort((left, right) =>
+    right.timestamp.localeCompare(left.timestamp)
+  );
+  const latestBySurface = {};
+
+  for (const summary of latestFirst) {
+    if (!(summary.surface in latestBySurface)) {
+      latestBySurface[summary.surface] = summary;
+    }
+  }
+
+  const latestRollup = {
+    generatedAt,
+    latestRun: latestFirst[0] ?? null,
+    latestBySurface
+  };
+  const bySurfaceRollup = {
+    generatedAt,
+    groupKey: "surface",
+    groups: groupSummaries(summaries, (summary) => summary.surface)
+  };
+  const byVariantRollup = {
+    generatedAt,
+    groupKey: "surface:variant_name",
+    groups: groupSummaries(
+      summaries,
+      (summary) => `${summary.surface}:${summary.variant_name}`
+    )
+  };
+  const byModelRollup = {
+    generatedAt,
+    groupKey: "provider:model",
+    groups: groupSummaries(
+      summaries,
+      (summary) => `${summary.provider}:${summary.model}`
+    )
+  };
+
+  await Promise.all([
+    writeJsonFile(path.join(rollupsDir, "latest.json"), latestRollup),
+    writeJsonFile(path.join(rollupsDir, "by_surface.json"), bySurfaceRollup),
+    writeJsonFile(path.join(rollupsDir, "by_variant.json"), byVariantRollup),
+    writeJsonFile(path.join(rollupsDir, "by_model.json"), byModelRollup)
+  ]);
+}
+
+function groupSummaries(summaries, getGroupKey) {
+  const groups = {};
+
+  for (const summary of summaries) {
+    const key = getGroupKey(summary);
+    groups[key] ??= [];
+    groups[key].push(summary);
+  }
+
+  for (const groupEntries of Object.values(groups)) {
+    groupEntries.sort((left, right) => left.timestamp.localeCompare(right.timestamp));
+  }
+
+  return groups;
+}
+
+async function ensureHistoryLayout(historyDir) {
+  await Promise.all([
+    mkdir(path.join(historyDir, RUNS_DIRNAME), { recursive: true }),
+    mkdir(path.join(historyDir, ROLLUPS_DIRNAME), { recursive: true })
+  ]);
+}
+
+async function writeJsonFile(filePath, value) {
+  await writeFile(filePath, JSON.stringify(value, null, 2) + "\n", "utf8");
+}
+
+function assertPlainObject(value, fieldName) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error(`${fieldName} must be an object`);
+  }
+}
+
+function assertNonEmptyString(value, fieldName) {
+  if (typeof value !== "string" || value.trim().length === 0) {
+    throw new Error(`${fieldName} must be a non-empty string`);
+  }
+  return value.trim();
+}
+
+function assertPositiveInteger(value, fieldName) {
+  if (!Number.isInteger(value) || value < 1) {
+    throw new Error(`${fieldName} must be a positive integer`);
+  }
+  return value;
+}
+
+function assertFiniteNumber(value, fieldName) {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    throw new Error(`${fieldName} must be a finite number`);
+  }
+  return value;
+}
+
+function assertRatio(value, fieldName) {
+  const normalized = assertFiniteNumber(value, fieldName);
+  if (normalized < 0 || normalized > 1) {
+    throw new Error(`${fieldName} must be between 0 and 1`);
+  }
+  return normalized;
+}
+
+function assertIsoDateTime(value, fieldName) {
+  const normalized = assertNonEmptyString(value, fieldName);
+  if (Number.isNaN(Date.parse(normalized))) {
+    throw new Error(`${fieldName} must be a valid ISO date-time string`);
+  }
+  return normalized;
+}
+
+function slugify(value) {
+  return value
+    .toLowerCase()
+    .replaceAll(/[^a-z0-9]+/g, "-")
+    .replaceAll(/^-+|-+$/g, "");
+}

ai_evals/scripts/append-official-run.mjs

@@ -1,46 +1,17 @@
 #!/usr/bin/env node
 
-import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { readFile } from "node:fs/promises";
 import path from "node:path";
-
-const DEFAULT_HISTORY_DIR = path.resolve("ai_evals/history");
-const SUMMARY_FILENAME = "summary.jsonl";
-const RUNS_DIRNAME = "runs";
-const ROLLUPS_DIRNAME = "rollups";
+import { appendOfficialRun, DEFAULT_HISTORY_DIR } from "../history/writer.mjs";
 
 async function main() {
   const { inputPath, historyDir } = parseArgs(process.argv.slice(2));
-  const absoluteHistoryDir = path.resolve(historyDir);
-  const input = await loadJsonFile(path.resolve(inputPath));
-  const normalizedRun = normalizeRun(input);
-  await ensureHistoryLayout(absoluteHistoryDir);
+  const input = JSON.parse(await readFile(path.resolve(inputPath), "utf8"));
+  const result = await appendOfficialRun(input, {
+    historyDir
+  });
 
-  const runFilename = `${normalizedRun.run_id}.json`;
-  const runRelativePath = path.posix.join(RUNS_DIRNAME, runFilename);
-  const runFilePath = path.join(absoluteHistoryDir, runRelativePath);
-
-  await writeJsonFile(runFilePath, normalizedRun);
-
-  const summaryPath = path.join(absoluteHistoryDir, SUMMARY_FILENAME);
-  const summaries = await loadSummaryEntries(summaryPath);
-  const summaryEntry = buildSummaryEntry(normalizedRun, runRelativePath);
-  const nextSummaries = upsertSummaryEntry(summaries, summaryEntry);
-
-  await writeSummaryEntries(summaryPath, nextSummaries);
-  await writeRollups(absoluteHistoryDir, nextSummaries);
-
-  process.stdout.write(
-    JSON.stringify(
-      {
-        status: "ok",
-        runId: normalizedRun.run_id,
-        runPath: runRelativePath,
-        summaryEntries: nextSummaries.length
-      },
-      null,
-      2
-    ) + "\n"
-  );
+  process.stdout.write(JSON.stringify(result, null, 2) + "\n");
 }
 
 function parseArgs(argv) {
@@ -86,381 +57,6 @@ function printHelp() {
   );
 }
 
-async function loadJsonFile(filePath) {
-  const raw = await readFile(filePath, "utf8");
-  try {
-    return JSON.parse(raw);
-  } catch (error) {
-    throw new Error(`Failed to parse JSON from ${filePath}: ${error.message}`);
-  }
-}
-
-function normalizeRun(input) {
-  assertPlainObject(input, "benchmark run");
-
-  const timestamp = assertIsoDateTime(input.timestamp, "timestamp");
-  const gitSha = assertNonEmptyString(input.git_sha, "git_sha");
-  const suiteVersion = assertNonEmptyString(input.suite_version, "suite_version");
-  const scoringVersion = assertNonEmptyString(
-    input.scoring_version,
-    "scoring_version"
-  );
-  const surface = assertNonEmptyString(input.surface, "surface");
-  const variantName = assertNonEmptyString(input.variant_name, "variant_name");
-  const provider = assertNonEmptyString(input.provider, "provider");
-  const model = assertNonEmptyString(input.model, "model");
-  const judgeModel =
-    input.judge_model === null || input.judge_model === undefined
-      ? null
-      : assertNonEmptyString(input.judge_model, "judge_model");
-  const runsPerCase = assertPositiveInteger(input.runs_per_case, "runs_per_case");
-  const caseCount = assertPositiveInteger(input.case_count, "case_count");
-  const metrics = normalizeMetrics(input.metrics, runsPerCase);
-  const cases = normalizeCases(input.cases);
-
-  if (cases.length !== caseCount) {
-    throw new Error(
-      `case_count (${caseCount}) does not match cases.length (${cases.length})`
-    );
-  }
-
-  const runId =
-    input.run_id && typeof input.run_id === "string" && input.run_id.trim()
-      ? input.run_id.trim()
-      : buildRunId({
-          timestamp,
-          surface,
-          variantName,
-          provider,
-          model,
-          gitSha
-        });
-
-  return {
-    ...input,
-    run_id: runId,
-    timestamp,
-    git_sha: gitSha,
-    suite_version: suiteVersion,
-    scoring_version: scoringVersion,
-    surface,
-    variant_name: variantName,
-    provider,
-    model,
-    judge_model: judgeModel,
-    runs_per_case: runsPerCase,
-    case_count: caseCount,
-    metrics,
-    cases
-  };
-}
-
-function normalizeMetrics(input, runsPerCase) {
-  assertPlainObject(input, "metrics");
-
-  const quality = normalizeMetricGroup(
-    input.quality,
-    "metrics.quality",
-    [
-      "pass_rate",
-      "deterministic_pass_rate",
-      "judge_score_mean",
-      "judge_score_median",
-      "judge_score_p10",
-      "quality_score"
-    ],
-    new Set(["pass_rate", "deterministic_pass_rate"])
-  );
-  const reliability = normalizeMetricGroup(
-    input.reliability,
-    "metrics.reliability",
-    ["runs_per_case", "flake_rate", "path_consistency"],
-    new Set(["flake_rate", "path_consistency"])
-  );
-  const efficiency = normalizeMetricGroup(
-    input.efficiency,
-    "metrics.efficiency",
-    [
-      "latency_ms_mean",
-      "latency_ms_median",
-      "tokens_total_mean",
-      "tool_calls_mean",
-      "iterations_mean",
-      "estimated_cost_mean",
-      "cost_per_success",
-      "latency_per_success",
-      "efficiency_score",
-      "value_score"
-    ]
-  );
-
-  if (reliability.runs_per_case !== runsPerCase) {
-    throw new Error(
-      `metrics.reliability.runs_per_case (${reliability.runs_per_case}) does not match runs_per_case (${runsPerCase})`
-    );
-  }
-
-  if (quality.category_pass_rate !== undefined) {
-    assertPlainObject(quality.category_pass_rate, "metrics.quality.category_pass_rate");
-    for (const [category, value] of Object.entries(quality.category_pass_rate)) {
-      assertRatio(value, `metrics.quality.category_pass_rate.${category}`);
-    }
-  }
-
-  return { quality, reliability, efficiency };
-}
-
-function normalizeMetricGroup(input, groupName, requiredFields, ratioFields = new Set()) {
-  assertPlainObject(input, groupName);
-  const normalized = { ...input };
-
-  for (const field of requiredFields) {
-    if (!(field in normalized)) {
-      throw new Error(`Missing required ${groupName}.${field}`);
-    }
-    if (field === "runs_per_case") {
-      normalized[field] = assertPositiveInteger(
-        normalized[field],
-        `${groupName}.${field}`
-      );
-      continue;
-    }
-    normalized[field] = assertFiniteNumber(normalized[field], `${groupName}.${field}`);
-    if (ratioFields.has(field)) {
-      assertRatio(normalized[field], `${groupName}.${field}`);
-    }
-  }
-
-  for (const [field, value] of Object.entries(normalized)) {
-    if (value === null || value === undefined) {
-      continue;
-    }
-    if (typeof value === "number") {
-      normalized[field] = assertFiniteNumber(value, `${groupName}.${field}`);
-      if (ratioFields.has(field)) {
-        assertRatio(normalized[field], `${groupName}.${field}`);
-      }
-    }
-  }
-
-  return normalized;
-}
-
-function normalizeCases(input) {
-  if (!Array.isArray(input) || input.length === 0) {
-    throw new Error("cases must be a non-empty array");
-  }
-
-  return input.map((entry, index) => {
-    assertPlainObject(entry, `cases[${index}]`);
-
-    return {
-      ...entry,
-      id: assertNonEmptyString(entry.id, `cases[${index}].id`),
-      pass_rate: assertRatio(entry.pass_rate, `cases[${index}].pass_rate`)
-    };
-  });
-}
-
-function buildRunId({ timestamp, surface, variantName, provider, model, gitSha }) {
-  const timestampSlug = timestamp.replaceAll(":", "-").replaceAll(".", "-");
-  const shortSha = gitSha.slice(0, 12);
-
-  return [
-    slugify(timestampSlug),
-    slugify(surface),
-    slugify(variantName),
-    slugify(provider),
-    slugify(model),
-    shortSha
-  ]
-    .filter(Boolean)
-    .join("__");
-}
-
-function buildSummaryEntry(run, runRelativePath) {
-  return {
-    run_id: run.run_id,
-    timestamp: run.timestamp,
-    git_sha: run.git_sha,
-    suite_version: run.suite_version,
-    scoring_version: run.scoring_version,
-    surface: run.surface,
-    variant_name: run.variant_name,
-    provider: run.provider,
-    model: run.model,
-    judge_model: run.judge_model,
-    runs_per_case: run.runs_per_case,
-    case_count: run.case_count,
-    run_path: runRelativePath,
-    metrics: run.metrics
-  };
-}
-
-function upsertSummaryEntry(entries, nextEntry) {
-  const remainingEntries = entries.filter((entry) => entry.run_id !== nextEntry.run_id);
-  const nextEntries = [...remainingEntries, nextEntry];
-  nextEntries.sort((left, right) => left.timestamp.localeCompare(right.timestamp));
-  return nextEntries;
-}
-
-async function loadSummaryEntries(summaryPath) {
-  try {
-    const raw = await readFile(summaryPath, "utf8");
-    return raw
-      .split("\n")
-      .map((line) => line.trim())
-      .filter(Boolean)
-      .map((line, index) => {
-        try {
-          return JSON.parse(line);
-        } catch (error) {
-          throw new Error(
-            `Failed to parse ${SUMMARY_FILENAME} line ${index + 1}: ${error.message}`
-          );
-        }
-      });
-  } catch (error) {
-    if (error.code === "ENOENT") {
-      return [];
-    }
-    throw error;
-  }
-}
-
-async function writeSummaryEntries(summaryPath, entries) {
-  const content =
-    entries.map((entry) => JSON.stringify(entry)).join("\n") +
-    (entries.length > 0 ? "\n" : "");
-  await writeFile(summaryPath, content, "utf8");
-}
-
-async function writeRollups(historyDir, summaries) {
-  const rollupsDir = path.join(historyDir, ROLLUPS_DIRNAME);
-  const generatedAt = new Date().toISOString();
-  const latestFirst = [...summaries].sort((left, right) =>
-    right.timestamp.localeCompare(left.timestamp)
-  );
-  const latestBySurface = {};
-
-  for (const summary of latestFirst) {
-    if (!(summary.surface in latestBySurface)) {
-      latestBySurface[summary.surface] = summary;
-    }
-  }
-
-  const latestRollup = {
-    generatedAt,
-    latestRun: latestFirst[0] ?? null,
-    latestBySurface
-  };
-  const bySurfaceRollup = {
-    generatedAt,
-    groupKey: "surface",
-    groups: groupSummaries(summaries, (summary) => summary.surface)
-  };
-  const byVariantRollup = {
-    generatedAt,
-    groupKey: "surface:variant_name",
-    groups: groupSummaries(
-      summaries,
-      (summary) => `${summary.surface}:${summary.variant_name}`
-    )
-  };
-  const byModelRollup = {
-    generatedAt,
-    groupKey: "provider:model",
-    groups: groupSummaries(
-      summaries,
-      (summary) => `${summary.provider}:${summary.model}`
-    )
-  };
-
-  await Promise.all([
-    writeJsonFile(path.join(rollupsDir, "latest.json"), latestRollup),
-    writeJsonFile(path.join(rollupsDir, "by_surface.json"), bySurfaceRollup),
-    writeJsonFile(path.join(rollupsDir, "by_variant.json"), byVariantRollup),
-    writeJsonFile(path.join(rollupsDir, "by_model.json"), byModelRollup)
-  ]);
-}
-
-function groupSummaries(summaries, getGroupKey) {
-  const groups = {};
-
-  for (const summary of summaries) {
-    const key = getGroupKey(summary);
-    groups[key] ??= [];
-    groups[key].push(summary);
-  }
-
-  for (const groupEntries of Object.values(groups)) {
-    groupEntries.sort((left, right) => left.timestamp.localeCompare(right.timestamp));
-  }
-
-  return groups;
-}
-
-async function ensureHistoryLayout(historyDir) {
-  await Promise.all([
-    mkdir(path.join(historyDir, RUNS_DIRNAME), { recursive: true }),
-    mkdir(path.join(historyDir, ROLLUPS_DIRNAME), { recursive: true })
-  ]);
-}
-
-async function writeJsonFile(filePath, value) {
-  await writeFile(filePath, JSON.stringify(value, null, 2) + "\n", "utf8");
-}
-
-function assertPlainObject(value, fieldName) {
-  if (!value || typeof value !== "object" || Array.isArray(value)) {
-    throw new Error(`${fieldName} must be an object`);
-  }
-}
-
-function assertNonEmptyString(value, fieldName) {
-  if (typeof value !== "string" || value.trim().length === 0) {
-    throw new Error(`${fieldName} must be a non-empty string`);
-  }
-  return value.trim();
-}
-
-function assertPositiveInteger(value, fieldName) {
-  if (!Number.isInteger(value) || value < 1) {
-    throw new Error(`${fieldName} must be a positive integer`);
-  }
-  return value;
-}
-
-function assertFiniteNumber(value, fieldName) {
-  if (typeof value !== "number" || !Number.isFinite(value)) {
-    throw new Error(`${fieldName} must be a finite number`);
-  }
-  return value;
-}
-
-function assertRatio(value, fieldName) {
-  const normalized = assertFiniteNumber(value, fieldName);
-  if (normalized < 0 || normalized > 1) {
-    throw new Error(`${fieldName} must be between 0 and 1`);
-  }
-  return normalized;
-}
-
-function assertIsoDateTime(value, fieldName) {
-  const normalized = assertNonEmptyString(value, fieldName);
-  if (Number.isNaN(Date.parse(normalized))) {
-    throw new Error(`${fieldName} must be a valid ISO date-time string`);
-  }
-  return normalized;
-}
-
-function slugify(value) {
-  return value
-    .toLowerCase()
-    .replaceAll(/[^a-z0-9]+/g, "-")
-    .replaceAll(/^-+|-+$/g, "");
-}
-
 main().catch((error) => {
   process.stderr.write(`${error.message}\n`);
   process.exit(1);

docs/system-prompt-testing-status.md

@@ -23,13 +23,12 @@ What is true today:
 - variants can be frozen as named snapshots through the benchmark CLI
 - repeated-run CLI benchmarking exists through `--runs`
 - basic CLI reliability metrics now exist
-- benchmark history scaffolding exists in `ai_evals/history`
+- benchmark history writing and reading now exist in the benchmark CLI
 
 What is not true yet:
 
 - frontend is not yet exposed through the benchmark CLI
 - token and cost metrics are not implemented
-- official history writing is not yet wired into normal benchmark commands
 - no UI studio exists yet
 
 ## Comparison To The Plan
@@ -84,11 +83,11 @@ Implemented:
 - `run`
 - `compare`
 - `snapshot-variant`
+- `history`
 - CLI adapter selection through `--surface cli`
 
 Still missing:
 
-- `history` command implementation
 - frontend adapter selection
 
 ### Phase 3: Replace the CLI smoke suite with real artifact evaluation
@@ -149,7 +148,7 @@ Planned:
 
 Status:
 
-- partially done
+- mostly done
 
 Implemented:
 
@@ -157,14 +156,18 @@ Implemented:
 - official run schema scaffold in `benchmark-run.schema.json`
 - `summary.jsonl`
 - rollup placeholders
+- shared history writer under `ai_evals/history/writer.mjs`
 - snapshot writer script under `ai_evals/scripts/append-official-run.mjs`
+- benchmark CLI wiring for `compare --write-history`
+- benchmark CLI `history --view latest|summary|surface|variant|model`
+- official run snapshots generated from repeated CLI benchmark results
+- history rollups rebuilt from real benchmark writes
 
 Still missing:
 
-- benchmark CLI wiring to emit official history snapshots
 - pass-rate summaries across repeated runs
 - worst-failure reporting
-- chart-ready data generated from real benchmark runs instead of placeholders
+- richer history filtering and reporting around those snapshots
 
 ### Phase 5: Finish the frontend black-box harness on top of the shared model
 
@@ -253,7 +256,7 @@ The most important implemented changes so far are:
   - `wmill init`
 - Moved `wmill init` testing overrides to internal env vars instead of public flags
 - Added docs for variant workflows and benchmark usage
-- Added benchmark history scaffolding
+- Added official benchmark history writing and reading
 
 ## What Is Left To Do
 
@@ -265,8 +268,11 @@ The highest-priority remaining work is:
    - latency
    - tool-call count
    - token and cost metrics if available
-2. Implement official history writing from the benchmark CLI.
-3. Expand the CLI case corpus to cover more real skill behavior.
+2. Expand the CLI case corpus to cover more real skill behavior.
+3. Add stronger official history summaries:
+   - worst-failure views
+   - better pass-rate rollups
+   - more filtering
 4. Bring frontend behind the same benchmark CLI.
 5. Add CI tiers.
 6. Build the UI last.
@@ -275,13 +281,13 @@ The highest-priority remaining work is:
 
 The best next implementation step is:
 
-- official history writing from repeated CLI benchmark runs
+- expand CLI coverage and enrich the official history metrics
 
 Reason:
 
-- the current CLI harness can now detect meaningful skill regressions and basic flakiness
-- the next missing layer is persisted benchmark history, not basic run aggregation
-- history wiring is needed before CI and before any useful trend dashboard
+- the current CLI harness can now detect meaningful skill regressions, write official snapshots, and read them back
+- the next missing layer is better coverage and stronger metrics inside that tracked history
+- CI and the future trend dashboard should build on the existing history path instead of inventing a second one
 
 ## Relevant Files