diff --git a/ai_evals/README.md b/ai_evals/README.md index d63c38b862..988c4dd760 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -60,6 +60,8 @@ bun run cli -- snapshot-variant --surface cli --variant candidate bun run cli -- list-cases --surface cli bun run cli -- run --surface cli --case bun-hello-script --variant baseline --runs 5 bun run cli -- compare --surface cli --case bun-hello-script --variant baseline --variant baseline --runs 5 +bun run cli -- compare --surface cli --case bun-hello-script --variant baseline-frozen --variant candidate --runs 5 --write-history +bun run cli -- history --view latest ``` At the moment this is still intentionally small, but it is the only benchmark diff --git a/ai_evals/adapters/cli/runtime.ts b/ai_evals/adapters/cli/runtime.ts index e0558aa1eb..ba144a305b 100644 --- a/ai_evals/adapters/cli/runtime.ts +++ b/ai_evals/adapters/cli/runtime.ts @@ -17,6 +17,8 @@ export interface PromptRunResult { } const REPO_ROOT = fileURLToPath(new URL("../../../", import.meta.url)); +export const CLI_BENCHMARK_PROVIDER = "anthropic"; +export const CLI_BENCHMARK_MODEL = "haiku"; export function getGeneratedSkillsSource(): string { return join(REPO_ROOT, "system_prompts", "auto-generated", "skills"); @@ -35,7 +37,7 @@ export async function runPromptAndCapture( const options: Options = { cwd, - model: "haiku", + model: CLI_BENCHMARK_MODEL, maxTurns, settingSources: ["project"], allowedTools: ["Skill", "Read", "Glob", "Grep", "Bash", "Write", "Edit"] diff --git a/ai_evals/cli/README.md b/ai_evals/cli/README.md index e7e9420898..92849c7646 100644 --- a/ai_evals/cli/README.md +++ b/ai_evals/cli/README.md @@ -9,6 +9,7 @@ The current implementation is intentionally small: - `run` command - `compare` command +- `history` command - `list-cases` and `list-variants` discovery commands - `cli` surface only @@ -83,6 +84,21 @@ cd ai_evals bun run cli -- compare --surface cli --case bun-hello-script --variant baseline --variant baseline --json ``` +Write official benchmark snapshots while comparing distinct variants: + +```bash +cd ai_evals +bun run cli -- compare --surface cli --case bun-hello-script --variant baseline-frozen --variant candidate --runs 5 --write-history +``` + +Inspect the tracked history: + +```bash +cd ai_evals +bun run cli -- history --view latest +bun run cli -- history --view summary --limit 10 +``` + ## Benchmarking A CLI Skill Change If you change one of the generated CLI skills and want to know whether the @@ -130,6 +146,15 @@ for repeated runs: - average skill-invocation count - aggregated required-check failures +When `--write-history` is used on `compare`, the benchmark CLI also writes one +official snapshot per compared variant into `ai_evals/history/` and rebuilds: + +- `summary.jsonl` +- `rollups/latest.json` +- `rollups/by_surface.json` +- `rollups/by_variant.json` +- `rollups/by_model.json` + The compare output also includes tool usage and invoked skills as diagnostics. True efficiency metrics such as latency, token usage, and cost are planned, but @@ -168,8 +193,7 @@ instead of the generated default. Later iterations should add: -- `history` command - frontend adapters - variant cleanup and diff helpers - token and cost metrics in compare output -- shared result/history writing from this entrypoint +- richer history views and filtering diff --git a/ai_evals/cli/index.ts b/ai_evals/cli/index.ts index b55141e405..0ece159f44 100644 --- a/ai_evals/cli/index.ts +++ b/ai_evals/cli/index.ts @@ -1,16 +1,29 @@ #!/usr/bin/env bun +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; import { cleanupWorkspace, loadCliArtifactEvalCases, runCliArtifactEvalCase } from "../adapters/cli/artifact-eval"; +import { + CLI_BENCHMARK_MODEL, + CLI_BENCHMARK_PROVIDER, +} from "../adapters/cli/runtime"; import { loadCliVariantById, loadCliVariants, snapshotCliVariant, type CliVariant } from "../adapters/cli/variants"; +import { + appendOfficialRun, + DEFAULT_HISTORY_DIR, + loadHistoryRollup, + loadLatestHistoryRollup, + loadSummaryHistory +} from "../history/writer.mjs"; type CommandName = | "run" @@ -30,6 +43,10 @@ interface ParsedArgs { runs: number; json: boolean; keepWorkspace: boolean; + writeHistory: boolean; + historyDir?: string; + historyView: "latest" | "summary" | "surface" | "variant" | "model"; + limit: number; } interface AttemptSummary { @@ -44,6 +61,7 @@ interface AttemptSummary { checks: Array<{ name: string; passed: boolean; required?: boolean }>; requiredFailedChecks: string[]; expectedFiles: Array<{ path: string; exists: boolean }>; + pathSignature: string; } interface AggregateMetrics { @@ -51,6 +69,8 @@ interface AggregateMetrics { passedRuns: number; passRate: number; averageDurationMs: number; + medianDurationMs: number; + latencyPerSuccessMs: number; averageAssistantMessages: number; averageToolCalls: number; averageSkillInvocations: number; @@ -59,6 +79,43 @@ interface AggregateMetrics { requiredFailureCounts: Array<{ name: string; count: number }>; } +interface CompareCaseResult { + caseId: string; + totalRuns: number; + passedRuns: number; + passRate: number; + averageDurationMs: number; + medianDurationMs: number; + latencyPerSuccessMs: number; + averageAssistantMessages: number; + averageToolCalls: number; + averageSkillInvocations: number; + pathConsistency: number; + distinctSkillsInvoked: string[]; + distinctToolsUsed: string[]; + requiredFailureCounts: Array<{ name: string; count: number }>; +} + +interface CompareVariantResult { + label: string; + variant: string; + totalCases: number; + fullyPassedCases: number; + totalRuns: number; + passedRuns: number; + passRate: number; + averageDurationMs: number; + medianDurationMs: number; + latencyPerSuccessMs: number; + averageAssistantMessages: number; + averageToolCalls: number; + averageSkillInvocations: number; + distinctSkillsInvoked: string[]; + distinctToolsUsed: string[]; + requiredFailureCounts: Array<{ name: string; count: number }>; + caseResults: CompareCaseResult[]; +} + async function main() { const args = parseArgs(process.argv.slice(2)); @@ -79,9 +136,8 @@ async function main() { await handleCompare(args); return; case "history": - throw new Error( - `'${args.command}' is not implemented yet in the repo-level benchmark CLI` - ); + await handleHistory(args); + return; default: printHelp(); throw new Error(`Unknown command: ${String(args.command)}`); @@ -157,6 +213,9 @@ async function handleRun(args: ParsedArgs) { if (args.keepWorkspace && args.runs !== 1) { throw new Error("--keep-workspace is only supported when --runs is 1"); } + if (args.writeHistory) { + throw new Error("--write-history is currently supported on compare only"); + } switch (surface) { case "cli": { @@ -265,11 +324,16 @@ async function handleCompare(args: ParsedArgs) { const variants = await Promise.all( args.variantIds.map((variantId) => loadCliVariantById(variantId)) ); + if (args.writeHistory && new Set(variants.map((variant) => variant.id)).size !== variants.length) { + throw new Error( + "--write-history requires distinct variant ids so official snapshots stay unambiguous" + ); + } const labeledVariants = labelVariantSelections(variants); - const variantResults = []; + const internalVariantResults: Array = []; for (const labeledVariant of labeledVariants) { - const caseResults = []; + const caseResults: CompareCaseResult[] = []; const allAttempts: AttemptSummary[] = []; for (const evalCase of selectedCases) { @@ -286,9 +350,12 @@ async function handleCompare(args: ParsedArgs) { passedRuns: execution.aggregate.passedRuns, passRate: execution.aggregate.passRate, averageDurationMs: execution.aggregate.averageDurationMs, + medianDurationMs: execution.aggregate.medianDurationMs, + latencyPerSuccessMs: execution.aggregate.latencyPerSuccessMs, averageAssistantMessages: execution.aggregate.averageAssistantMessages, averageToolCalls: execution.aggregate.averageToolCalls, averageSkillInvocations: execution.aggregate.averageSkillInvocations, + pathConsistency: computePathConsistency(execution.attempts), distinctSkillsInvoked: execution.aggregate.distinctSkillsInvoked, distinctToolsUsed: execution.aggregate.distinctToolsUsed, requiredFailureCounts: execution.aggregate.requiredFailureCounts @@ -297,7 +364,7 @@ async function handleCompare(args: ParsedArgs) { const aggregate = aggregateAttempts(allAttempts); - variantResults.push({ + internalVariantResults.push({ label: labeledVariant.label, variant: labeledVariant.variant.id, totalCases: caseResults.length, @@ -308,22 +375,35 @@ async function handleCompare(args: ParsedArgs) { passedRuns: aggregate.passedRuns, passRate: aggregate.passRate, averageDurationMs: aggregate.averageDurationMs, + medianDurationMs: aggregate.medianDurationMs, + latencyPerSuccessMs: aggregate.latencyPerSuccessMs, averageAssistantMessages: aggregate.averageAssistantMessages, averageToolCalls: aggregate.averageToolCalls, averageSkillInvocations: aggregate.averageSkillInvocations, distinctSkillsInvoked: aggregate.distinctSkillsInvoked, distinctToolsUsed: aggregate.distinctToolsUsed, requiredFailureCounts: aggregate.requiredFailureCounts, - caseResults + caseResults, + allAttempts }); } + const historyWrites = args.writeHistory + ? await writeCompareHistorySnapshots({ + surface, + runs: args.runs, + variants: internalVariantResults, + historyDir: args.historyDir + }) + : []; + const payload = { command: "compare", surface, caseIds: selectedCases.map((entry) => entry.id), runs: args.runs, - variants: variantResults + variants: internalVariantResults.map(({ allAttempts: _allAttempts, ...variant }) => variant), + historyWrites }; if (args.json) { @@ -339,6 +419,62 @@ async function handleCompare(args: ParsedArgs) { } } +async function handleHistory(args: ParsedArgs) { + const historyDir = args.historyDir ?? DEFAULT_HISTORY_DIR; + + switch (args.historyView) { + case "latest": { + const payload = await loadLatestHistoryRollup(historyDir); + if (args.json) { + process.stdout.write(JSON.stringify(payload, null, 2) + "\n"); + } else { + printHistoryLatestSummary(payload); + } + return; + } + case "summary": { + const summaries = await loadSummaryHistory(historyDir); + const payload = { + view: "summary", + historyDir, + totalEntries: summaries.length, + entries: summaries.slice(-args.limit).reverse() + }; + if (args.json) { + process.stdout.write(JSON.stringify(payload, null, 2) + "\n"); + } else { + printHistorySummary(payload); + } + return; + } + case "surface": + case "variant": + case "model": { + const filename = + args.historyView === "surface" + ? "by_surface.json" + : args.historyView === "variant" + ? "by_variant.json" + : "by_model.json"; + const rollup = await loadHistoryRollup(filename, historyDir); + const payload = { + view: args.historyView, + historyDir, + generatedAt: rollup.generatedAt, + groups: rollup.groups + }; + if (args.json) { + process.stdout.write(JSON.stringify(payload, null, 2) + "\n"); + } else { + printHistoryGroupSummary(payload, args.limit); + } + return; + } + default: + assertNever(args.historyView); + } +} + async function runCaseAttempts( evalCase: Awaited>[number], variant: CliVariant, @@ -374,6 +510,9 @@ function summarizeAttempt( result: Awaited>, attempt: number ): AttemptSummary { + const skillsInvoked = uniqueStrings(result.run.skillsInvoked); + const toolsUsed = uniqueStrings(result.run.toolsUsed.map((tool) => tool.tool)); + return { attempt, passed: result.passed, @@ -381,8 +520,8 @@ function summarizeAttempt( assistantMessageCount: result.run.assistantMessageCount, toolCallCount: result.run.toolsUsed.length, skillInvocationCount: result.run.skillsInvoked.length, - skillsInvoked: uniqueStrings(result.run.skillsInvoked), - toolsUsed: uniqueStrings(result.run.toolsUsed.map((tool) => tool.tool)), + skillsInvoked, + toolsUsed, checks: result.checks.map((check) => ({ name: check.name, passed: check.passed, @@ -394,12 +533,14 @@ function summarizeAttempt( expectedFiles: result.expectedFiles.map((file) => ({ path: file.path, exists: file.exists - })) + })), + pathSignature: buildPathSignature(skillsInvoked, toolsUsed) }; } function aggregateAttempts(attempts: AttemptSummary[]): AggregateMetrics { const requiredFailureCounts = new Map(); + const successfulAttempts = attempts.filter((attempt) => attempt.passed); for (const attempt of attempts) { for (const failure of attempt.requiredFailedChecks) { @@ -412,6 +553,8 @@ function aggregateAttempts(attempts: AttemptSummary[]): AggregateMetrics { passedRuns: attempts.filter((attempt) => attempt.passed).length, passRate: attempts.length === 0 ? 0 : attempts.filter((attempt) => attempt.passed).length / attempts.length, averageDurationMs: average(attempts.map((attempt) => attempt.durationMs)), + medianDurationMs: median(attempts.map((attempt) => attempt.durationMs)), + latencyPerSuccessMs: average(successfulAttempts.map((attempt) => attempt.durationMs)), averageAssistantMessages: average(attempts.map((attempt) => attempt.assistantMessageCount)), averageToolCalls: average(attempts.map((attempt) => attempt.toolCallCount)), averageSkillInvocations: average(attempts.map((attempt) => attempt.skillInvocationCount)), @@ -442,7 +585,11 @@ function parseArgs(argv: string[]): ParsedArgs { description: undefined, runs: 1, json: false, - keepWorkspace: false + keepWorkspace: false, + writeHistory: false, + historyDir: undefined, + historyView: "latest", + limit: 10 }; for (let index = 0; index < rest.length; index += 1) { @@ -478,6 +625,29 @@ function parseArgs(argv: string[]): ParsedArgs { continue; } + if (arg === "--write-history") { + parsed.writeHistory = true; + continue; + } + + if (arg === "--history-dir") { + parsed.historyDir = rest[index + 1]; + index += 1; + continue; + } + + if (arg === "--view") { + parsed.historyView = parseHistoryView(rest[index + 1]); + index += 1; + continue; + } + + if (arg === "--limit") { + parsed.limit = parsePositiveInteger(rest[index + 1], "--limit"); + index += 1; + continue; + } + if (arg === "--json") { parsed.json = true; continue; @@ -556,6 +726,22 @@ function parsePositiveInteger(value: string | undefined, flagName: string): numb return parsed; } +function parseHistoryView( + value: string | undefined +): ParsedArgs["historyView"] { + if ( + value === "latest" || + value === "summary" || + value === "surface" || + value === "variant" || + value === "model" + ) { + return value; + } + + throw new Error("--view must be one of: latest, summary, surface, variant, model"); +} + function printRunSummary(payload: { surface: SurfaceName; variant: string; @@ -626,30 +812,8 @@ function printCompareSummary(payload: { surface: SurfaceName; caseIds: string[]; runs: number; - variants: Array<{ - label: string; - variant: string; - totalCases: number; - fullyPassedCases: number; - totalRuns: number; - passedRuns: number; - passRate: number; - averageDurationMs: number; - averageAssistantMessages: number; - averageToolCalls: number; - averageSkillInvocations: number; - caseResults: Array<{ - caseId: string; - totalRuns: number; - passedRuns: number; - passRate: number; - averageDurationMs: number; - averageAssistantMessages: number; - averageToolCalls: number; - averageSkillInvocations: number; - requiredFailureCounts: Array<{ name: string; count: number }>; - }>; - }>; + variants: CompareVariantResult[]; + historyWrites: Array<{ variant: string; runId: string; runPath: string }>; }) { process.stdout.write(`Surface: ${payload.surface}\n`); process.stdout.write(`Cases: ${payload.caseIds.join(", ")}\n`); @@ -669,6 +833,13 @@ function printCompareSummary(payload: { } } } + + if (payload.historyWrites.length > 0) { + process.stdout.write("History writes:\n"); + for (const write of payload.historyWrites) { + process.stdout.write(`- ${write.variant}: ${write.runPath} (${write.runId})\n`); + } + } } function printSnapshotSummary(payload: { @@ -716,6 +887,241 @@ function labelVariantSelections( }); } +async function writeCompareHistorySnapshots(input: { + surface: SurfaceName; + runs: number; + variants: Array; + historyDir?: string; +}) { + const timestamp = new Date().toISOString(); + const gitSha = getGitSha(); + const writes = []; + + for (const variant of input.variants) { + const result = await appendOfficialRun( + buildOfficialRun({ + timestamp, + gitSha, + surface: input.surface, + runs: input.runs, + variant + }), + { + historyDir: input.historyDir + } + ); + + writes.push({ + variant: variant.variant, + runId: result.runId, + runPath: result.runPath + }); + } + + return writes; +} + +function buildOfficialRun(input: { + timestamp: string; + gitSha: string; + surface: SurfaceName; + runs: number; + variant: CompareVariantResult & { allAttempts: AttemptSummary[] }; +}) { + const flakeRate = + input.variant.caseResults.length === 0 + ? 0 + : input.variant.caseResults.filter( + (entry) => entry.passRate > 0 && entry.passRate < 1 + ).length / input.variant.caseResults.length; + const pathConsistency = average( + input.variant.caseResults.map((entry) => entry.pathConsistency) + ); + const qualityScore = input.variant.passRate * 100; + const efficiencyScore = computeEfficiencyScore(input.variant); + + return { + timestamp: input.timestamp, + git_sha: input.gitSha, + suite_version: "cli-benchmark-v1", + scoring_version: "cli-deterministic-v1", + surface: input.surface, + variant_name: input.variant.variant, + provider: CLI_BENCHMARK_PROVIDER, + model: CLI_BENCHMARK_MODEL, + judge_model: null, + runs_per_case: input.runs, + case_count: input.variant.caseResults.length, + metrics: { + quality: { + pass_rate: input.variant.passRate, + deterministic_pass_rate: input.variant.passRate, + judge_score_mean: 0, + judge_score_median: 0, + judge_score_p10: 0, + quality_score: qualityScore + }, + reliability: { + runs_per_case: input.runs, + flake_rate: flakeRate, + path_consistency: pathConsistency + }, + efficiency: { + latency_ms_mean: input.variant.averageDurationMs, + latency_ms_median: input.variant.medianDurationMs, + tokens_total_mean: 0, + tool_calls_mean: input.variant.averageToolCalls, + iterations_mean: input.variant.averageAssistantMessages, + estimated_cost_mean: 0, + cost_per_success: 0, + latency_per_success: input.variant.latencyPerSuccessMs, + efficiency_score: efficiencyScore, + value_score: (qualityScore * efficiencyScore) / 100 + } + }, + cases: input.variant.caseResults.map((entry) => ({ + id: entry.caseId, + pass_rate: entry.passRate, + average_duration_ms: entry.averageDurationMs, + median_duration_ms: entry.medianDurationMs, + latency_per_success_ms: entry.latencyPerSuccessMs, + average_assistant_messages: entry.averageAssistantMessages, + average_tool_calls: entry.averageToolCalls, + average_skill_invocations: entry.averageSkillInvocations, + path_consistency: entry.pathConsistency, + distinct_skills_invoked: entry.distinctSkillsInvoked, + distinct_tools_used: entry.distinctToolsUsed, + required_failure_counts: entry.requiredFailureCounts + })) + }; +} + +function computeEfficiencyScore(variant: CompareVariantResult): number { + const latencyFactor = 1 / (1 + variant.averageDurationMs / 20000); + const toolFactor = 1 / (1 + variant.averageToolCalls / 10); + const iterationFactor = 1 / (1 + variant.averageAssistantMessages / 10); + + return ((latencyFactor + toolFactor + iterationFactor) / 3) * 100; +} + +function getGitSha(): string { + try { + return execFileSync("git", ["rev-parse", "HEAD"], { + cwd: fileURLToPath(new URL("../..", import.meta.url)), + encoding: "utf8" + }).trim(); + } catch { + return "0000000"; + } +} + +function buildPathSignature(skillsInvoked: string[], toolsUsed: string[]): string { + return `skills:${skillsInvoked.join(",") || "(none)"}|tools:${toolsUsed.join(",") || "(none)"}`; +} + +function computePathConsistency(attempts: AttemptSummary[]): number { + if (attempts.length === 0) { + return 0; + } + + const counts = new Map(); + for (const attempt of attempts) { + counts.set(attempt.pathSignature, (counts.get(attempt.pathSignature) ?? 0) + 1); + } + + return Math.max(...counts.values()) / attempts.length; +} + +function printHistoryLatestSummary(payload: { + generatedAt: string; + latestRun: null | { + timestamp: string; + surface: string; + variant_name: string; + provider: string; + model: string; + metrics: { + quality: { pass_rate: number }; + efficiency: { latency_ms_mean: number }; + }; + }; + latestBySurface: Record; +}) { + process.stdout.write(`Generated: ${payload.generatedAt}\n`); + if (!payload.latestRun) { + process.stdout.write("Latest run: none\n"); + return; + } + + process.stdout.write( + `Latest run: ${payload.latestRun.timestamp} | ${payload.latestRun.surface} | ${payload.latestRun.variant_name} | ${formatPercent(payload.latestRun.metrics.quality.pass_rate)} | avg ${formatNumber(payload.latestRun.metrics.efficiency.latency_ms_mean)} ms\n` + ); + process.stdout.write("Latest by surface:\n"); + for (const [surface, summary] of Object.entries(payload.latestBySurface)) { + process.stdout.write( + `- ${surface}: ${summary.variant_name} (${formatPercent(summary.metrics.quality.pass_rate)})\n` + ); + } +} + +function printHistorySummary(payload: { + view: "summary"; + historyDir: string; + totalEntries: number; + entries: Array<{ + timestamp: string; + surface: string; + variant_name: string; + provider: string; + model: string; + runs_per_case: number; + case_count: number; + metrics: { + quality: { pass_rate: number }; + reliability: { flake_rate: number }; + efficiency: { latency_ms_mean: number }; + }; + }>; +}) { + process.stdout.write(`History: ${payload.historyDir}\n`); + process.stdout.write(`Entries: ${payload.totalEntries}\n`); + for (const entry of payload.entries) { + process.stdout.write( + `- ${entry.timestamp} | ${entry.surface} | ${entry.variant_name} | ${formatPercent(entry.metrics.quality.pass_rate)} | flake ${formatPercent(entry.metrics.reliability.flake_rate)} | avg ${formatNumber(entry.metrics.efficiency.latency_ms_mean)} ms | ${entry.case_count} cases x ${entry.runs_per_case} runs\n` + ); + } +} + +function printHistoryGroupSummary( + payload: { + view: "surface" | "variant" | "model"; + historyDir: string; + generatedAt: string; + groups: Record>; + }, + limit: number +) { + process.stdout.write(`History: ${payload.historyDir}\n`); + process.stdout.write(`View: ${payload.view}\n`); + process.stdout.write(`Generated: ${payload.generatedAt}\n`); + for (const [group, entries] of Object.entries(payload.groups) + .sort((left, right) => left[0].localeCompare(right[0])) + .slice(0, limit)) { + const latest = entries[entries.length - 1]; + process.stdout.write( + `- ${group}: ${entries.length} runs | latest ${latest.timestamp} | ${latest.variant_name} | ${formatPercent(latest.metrics.quality.pass_rate)} | avg ${formatNumber(latest.metrics.efficiency.latency_ms_mean)} ms\n` + ); + } +} + function printHelp() { process.stdout.write( [ @@ -724,8 +1130,8 @@ function printHelp() { " cd ai_evals && bun run cli -- list-variants --surface cli [--json]", " cd ai_evals && bun run cli -- snapshot-variant --surface cli --variant [--description ] [--json]", " cd ai_evals && bun run cli -- run --surface cli --case [--variant ] [--runs ] [--json] [--keep-workspace]", - " cd ai_evals && bun run cli -- compare --surface cli [--case ...] [--variant ...] [--runs ] [--json]", - " cd ai_evals && bun run cli -- history", + " cd ai_evals && bun run cli -- compare --surface cli [--case ...] [--variant ...] [--runs ] [--write-history] [--history-dir ] [--json]", + " cd ai_evals && bun run cli -- history [--view latest|summary|surface|variant|model] [--limit ] [--history-dir ] [--json]", "", "Current support:", " surfaces: cli" @@ -744,6 +1150,21 @@ function average(values: number[]): number { return values.reduce((sum, value) => sum + value, 0) / values.length; } +function median(values: number[]): number { + if (values.length === 0) { + return 0; + } + + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + + if (sorted.length % 2 === 0) { + return (sorted[middle - 1] + sorted[middle]) / 2; + } + + return sorted[middle]; +} + function uniqueStrings(values: string[]): string[] { return [...new Set(values)].sort((left, right) => left.localeCompare(right)); } diff --git a/ai_evals/history/writer.mjs b/ai_evals/history/writer.mjs new file mode 100644 index 0000000000..0c9c9f6f5f --- /dev/null +++ b/ai_evals/history/writer.mjs @@ -0,0 +1,430 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); + +export const DEFAULT_HISTORY_DIR = MODULE_DIR; +const SUMMARY_FILENAME = "summary.jsonl"; +const RUNS_DIRNAME = "runs"; +const ROLLUPS_DIRNAME = "rollups"; + +export async function appendOfficialRun(input, options = {}) { + const absoluteHistoryDir = path.resolve(options.historyDir ?? DEFAULT_HISTORY_DIR); + const normalizedRun = normalizeRun(input); + await ensureHistoryLayout(absoluteHistoryDir); + + const runFilename = `${normalizedRun.run_id}.json`; + const runRelativePath = path.posix.join(RUNS_DIRNAME, runFilename); + const runFilePath = path.join(absoluteHistoryDir, runRelativePath); + + await writeJsonFile(runFilePath, normalizedRun); + + const summaryPath = path.join(absoluteHistoryDir, SUMMARY_FILENAME); + const summaries = await loadSummaryEntries(summaryPath); + const summaryEntry = buildSummaryEntry(normalizedRun, runRelativePath); + const nextSummaries = upsertSummaryEntry(summaries, summaryEntry); + + await writeSummaryEntries(summaryPath, nextSummaries); + await writeRollups(absoluteHistoryDir, nextSummaries); + + return { + status: "ok", + runId: normalizedRun.run_id, + runPath: runRelativePath, + summaryEntries: nextSummaries.length + }; +} + +export async function loadLatestHistoryRollup(historyDir = DEFAULT_HISTORY_DIR) { + const rollupPath = path.join(path.resolve(historyDir), ROLLUPS_DIRNAME, "latest.json"); + return await loadJsonFile(rollupPath); +} + +export async function loadHistoryRollup( + name, + historyDir = DEFAULT_HISTORY_DIR +) { + const rollupPath = path.join(path.resolve(historyDir), ROLLUPS_DIRNAME, name); + return await loadJsonFile(rollupPath); +} + +export async function loadSummaryHistory(historyDir = DEFAULT_HISTORY_DIR) { + const summaryPath = path.join(path.resolve(historyDir), SUMMARY_FILENAME); + return await loadSummaryEntries(summaryPath); +} + +async function loadJsonFile(filePath) { + const raw = await readFile(filePath, "utf8"); + try { + return JSON.parse(raw); + } catch (error) { + throw new Error(`Failed to parse JSON from ${filePath}: ${error.message}`); + } +} + +function normalizeRun(input) { + assertPlainObject(input, "benchmark run"); + + const timestamp = assertIsoDateTime(input.timestamp, "timestamp"); + const gitSha = assertNonEmptyString(input.git_sha, "git_sha"); + const suiteVersion = assertNonEmptyString(input.suite_version, "suite_version"); + const scoringVersion = assertNonEmptyString( + input.scoring_version, + "scoring_version" + ); + const surface = assertNonEmptyString(input.surface, "surface"); + const variantName = assertNonEmptyString(input.variant_name, "variant_name"); + const provider = assertNonEmptyString(input.provider, "provider"); + const model = assertNonEmptyString(input.model, "model"); + const judgeModel = + input.judge_model === null || input.judge_model === undefined + ? null + : assertNonEmptyString(input.judge_model, "judge_model"); + const runsPerCase = assertPositiveInteger(input.runs_per_case, "runs_per_case"); + const caseCount = assertPositiveInteger(input.case_count, "case_count"); + const metrics = normalizeMetrics(input.metrics, runsPerCase); + const cases = normalizeCases(input.cases); + + if (cases.length !== caseCount) { + throw new Error( + `case_count (${caseCount}) does not match cases.length (${cases.length})` + ); + } + + const runId = + input.run_id && typeof input.run_id === "string" && input.run_id.trim() + ? input.run_id.trim() + : buildRunId({ + timestamp, + surface, + variantName, + provider, + model, + gitSha + }); + + return { + ...input, + run_id: runId, + timestamp, + git_sha: gitSha, + suite_version: suiteVersion, + scoring_version: scoringVersion, + surface, + variant_name: variantName, + provider, + model, + judge_model: judgeModel, + runs_per_case: runsPerCase, + case_count: caseCount, + metrics, + cases + }; +} + +function normalizeMetrics(input, runsPerCase) { + assertPlainObject(input, "metrics"); + + const quality = normalizeMetricGroup( + input.quality, + "metrics.quality", + [ + "pass_rate", + "deterministic_pass_rate", + "judge_score_mean", + "judge_score_median", + "judge_score_p10", + "quality_score" + ], + new Set(["pass_rate", "deterministic_pass_rate"]) + ); + const reliability = normalizeMetricGroup( + input.reliability, + "metrics.reliability", + ["runs_per_case", "flake_rate", "path_consistency"], + new Set(["flake_rate", "path_consistency"]) + ); + const efficiency = normalizeMetricGroup( + input.efficiency, + "metrics.efficiency", + [ + "latency_ms_mean", + "latency_ms_median", + "tokens_total_mean", + "tool_calls_mean", + "iterations_mean", + "estimated_cost_mean", + "cost_per_success", + "latency_per_success", + "efficiency_score", + "value_score" + ] + ); + + if (reliability.runs_per_case !== runsPerCase) { + throw new Error( + `metrics.reliability.runs_per_case (${reliability.runs_per_case}) does not match runs_per_case (${runsPerCase})` + ); + } + + if (quality.category_pass_rate !== undefined) { + assertPlainObject(quality.category_pass_rate, "metrics.quality.category_pass_rate"); + for (const [category, value] of Object.entries(quality.category_pass_rate)) { + assertRatio(value, `metrics.quality.category_pass_rate.${category}`); + } + } + + return { quality, reliability, efficiency }; +} + +function normalizeMetricGroup(input, groupName, requiredFields, ratioFields = new Set()) { + assertPlainObject(input, groupName); + const normalized = { ...input }; + + for (const field of requiredFields) { + if (!(field in normalized)) { + throw new Error(`Missing required ${groupName}.${field}`); + } + if (field === "runs_per_case") { + normalized[field] = assertPositiveInteger( + normalized[field], + `${groupName}.${field}` + ); + continue; + } + normalized[field] = assertFiniteNumber(normalized[field], `${groupName}.${field}`); + if (ratioFields.has(field)) { + assertRatio(normalized[field], `${groupName}.${field}`); + } + } + + for (const [field, value] of Object.entries(normalized)) { + if (value === null || value === undefined) { + continue; + } + if (typeof value === "number") { + normalized[field] = assertFiniteNumber(value, `${groupName}.${field}`); + if (ratioFields.has(field)) { + assertRatio(normalized[field], `${groupName}.${field}`); + } + } + } + + return normalized; +} + +function normalizeCases(input) { + if (!Array.isArray(input) || input.length === 0) { + throw new Error("cases must be a non-empty array"); + } + + return input.map((entry, index) => { + assertPlainObject(entry, `cases[${index}]`); + + return { + ...entry, + id: assertNonEmptyString(entry.id, `cases[${index}].id`), + pass_rate: assertRatio(entry.pass_rate, `cases[${index}].pass_rate`) + }; + }); +} + +function buildRunId({ timestamp, surface, variantName, provider, model, gitSha }) { + const timestampSlug = timestamp.replaceAll(":", "-").replaceAll(".", "-"); + const shortSha = gitSha.slice(0, 12); + + return [ + slugify(timestampSlug), + slugify(surface), + slugify(variantName), + slugify(provider), + slugify(model), + shortSha + ] + .filter(Boolean) + .join("__"); +} + +function buildSummaryEntry(run, runRelativePath) { + return { + run_id: run.run_id, + timestamp: run.timestamp, + git_sha: run.git_sha, + suite_version: run.suite_version, + scoring_version: run.scoring_version, + surface: run.surface, + variant_name: run.variant_name, + provider: run.provider, + model: run.model, + judge_model: run.judge_model, + runs_per_case: run.runs_per_case, + case_count: run.case_count, + run_path: runRelativePath, + metrics: run.metrics + }; +} + +function upsertSummaryEntry(entries, nextEntry) { + const remainingEntries = entries.filter((entry) => entry.run_id !== nextEntry.run_id); + const nextEntries = [...remainingEntries, nextEntry]; + nextEntries.sort((left, right) => left.timestamp.localeCompare(right.timestamp)); + return nextEntries; +} + +async function loadSummaryEntries(summaryPath) { + try { + const raw = await readFile(summaryPath, "utf8"); + return raw + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line, index) => { + try { + return JSON.parse(line); + } catch (error) { + throw new Error( + `Failed to parse ${SUMMARY_FILENAME} line ${index + 1}: ${error.message}` + ); + } + }); + } catch (error) { + if (error.code === "ENOENT") { + return []; + } + throw error; + } +} + +async function writeSummaryEntries(summaryPath, entries) { + const content = + entries.map((entry) => JSON.stringify(entry)).join("\n") + + (entries.length > 0 ? "\n" : ""); + await writeFile(summaryPath, content, "utf8"); +} + +async function writeRollups(historyDir, summaries) { + const rollupsDir = path.join(historyDir, ROLLUPS_DIRNAME); + const generatedAt = new Date().toISOString(); + const latestFirst = [...summaries].sort((left, right) => + right.timestamp.localeCompare(left.timestamp) + ); + const latestBySurface = {}; + + for (const summary of latestFirst) { + if (!(summary.surface in latestBySurface)) { + latestBySurface[summary.surface] = summary; + } + } + + const latestRollup = { + generatedAt, + latestRun: latestFirst[0] ?? null, + latestBySurface + }; + const bySurfaceRollup = { + generatedAt, + groupKey: "surface", + groups: groupSummaries(summaries, (summary) => summary.surface) + }; + const byVariantRollup = { + generatedAt, + groupKey: "surface:variant_name", + groups: groupSummaries( + summaries, + (summary) => `${summary.surface}:${summary.variant_name}` + ) + }; + const byModelRollup = { + generatedAt, + groupKey: "provider:model", + groups: groupSummaries( + summaries, + (summary) => `${summary.provider}:${summary.model}` + ) + }; + + await Promise.all([ + writeJsonFile(path.join(rollupsDir, "latest.json"), latestRollup), + writeJsonFile(path.join(rollupsDir, "by_surface.json"), bySurfaceRollup), + writeJsonFile(path.join(rollupsDir, "by_variant.json"), byVariantRollup), + writeJsonFile(path.join(rollupsDir, "by_model.json"), byModelRollup) + ]); +} + +function groupSummaries(summaries, getGroupKey) { + const groups = {}; + + for (const summary of summaries) { + const key = getGroupKey(summary); + groups[key] ??= []; + groups[key].push(summary); + } + + for (const groupEntries of Object.values(groups)) { + groupEntries.sort((left, right) => left.timestamp.localeCompare(right.timestamp)); + } + + return groups; +} + +async function ensureHistoryLayout(historyDir) { + await Promise.all([ + mkdir(path.join(historyDir, RUNS_DIRNAME), { recursive: true }), + mkdir(path.join(historyDir, ROLLUPS_DIRNAME), { recursive: true }) + ]); +} + +async function writeJsonFile(filePath, value) { + await writeFile(filePath, JSON.stringify(value, null, 2) + "\n", "utf8"); +} + +function assertPlainObject(value, fieldName) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${fieldName} must be an object`); + } +} + +function assertNonEmptyString(value, fieldName) { + if (typeof value !== "string" || value.trim().length === 0) { + throw new Error(`${fieldName} must be a non-empty string`); + } + return value.trim(); +} + +function assertPositiveInteger(value, fieldName) { + if (!Number.isInteger(value) || value < 1) { + throw new Error(`${fieldName} must be a positive integer`); + } + return value; +} + +function assertFiniteNumber(value, fieldName) { + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new Error(`${fieldName} must be a finite number`); + } + return value; +} + +function assertRatio(value, fieldName) { + const normalized = assertFiniteNumber(value, fieldName); + if (normalized < 0 || normalized > 1) { + throw new Error(`${fieldName} must be between 0 and 1`); + } + return normalized; +} + +function assertIsoDateTime(value, fieldName) { + const normalized = assertNonEmptyString(value, fieldName); + if (Number.isNaN(Date.parse(normalized))) { + throw new Error(`${fieldName} must be a valid ISO date-time string`); + } + return normalized; +} + +function slugify(value) { + return value + .toLowerCase() + .replaceAll(/[^a-z0-9]+/g, "-") + .replaceAll(/^-+|-+$/g, ""); +} diff --git a/ai_evals/scripts/append-official-run.mjs b/ai_evals/scripts/append-official-run.mjs index 3ca296f785..0c1f2b3624 100644 --- a/ai_evals/scripts/append-official-run.mjs +++ b/ai_evals/scripts/append-official-run.mjs @@ -1,46 +1,17 @@ #!/usr/bin/env node -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { readFile } from "node:fs/promises"; import path from "node:path"; - -const DEFAULT_HISTORY_DIR = path.resolve("ai_evals/history"); -const SUMMARY_FILENAME = "summary.jsonl"; -const RUNS_DIRNAME = "runs"; -const ROLLUPS_DIRNAME = "rollups"; +import { appendOfficialRun, DEFAULT_HISTORY_DIR } from "../history/writer.mjs"; async function main() { const { inputPath, historyDir } = parseArgs(process.argv.slice(2)); - const absoluteHistoryDir = path.resolve(historyDir); - const input = await loadJsonFile(path.resolve(inputPath)); - const normalizedRun = normalizeRun(input); - await ensureHistoryLayout(absoluteHistoryDir); + const input = JSON.parse(await readFile(path.resolve(inputPath), "utf8")); + const result = await appendOfficialRun(input, { + historyDir + }); - const runFilename = `${normalizedRun.run_id}.json`; - const runRelativePath = path.posix.join(RUNS_DIRNAME, runFilename); - const runFilePath = path.join(absoluteHistoryDir, runRelativePath); - - await writeJsonFile(runFilePath, normalizedRun); - - const summaryPath = path.join(absoluteHistoryDir, SUMMARY_FILENAME); - const summaries = await loadSummaryEntries(summaryPath); - const summaryEntry = buildSummaryEntry(normalizedRun, runRelativePath); - const nextSummaries = upsertSummaryEntry(summaries, summaryEntry); - - await writeSummaryEntries(summaryPath, nextSummaries); - await writeRollups(absoluteHistoryDir, nextSummaries); - - process.stdout.write( - JSON.stringify( - { - status: "ok", - runId: normalizedRun.run_id, - runPath: runRelativePath, - summaryEntries: nextSummaries.length - }, - null, - 2 - ) + "\n" - ); + process.stdout.write(JSON.stringify(result, null, 2) + "\n"); } function parseArgs(argv) { @@ -86,381 +57,6 @@ function printHelp() { ); } -async function loadJsonFile(filePath) { - const raw = await readFile(filePath, "utf8"); - try { - return JSON.parse(raw); - } catch (error) { - throw new Error(`Failed to parse JSON from ${filePath}: ${error.message}`); - } -} - -function normalizeRun(input) { - assertPlainObject(input, "benchmark run"); - - const timestamp = assertIsoDateTime(input.timestamp, "timestamp"); - const gitSha = assertNonEmptyString(input.git_sha, "git_sha"); - const suiteVersion = assertNonEmptyString(input.suite_version, "suite_version"); - const scoringVersion = assertNonEmptyString( - input.scoring_version, - "scoring_version" - ); - const surface = assertNonEmptyString(input.surface, "surface"); - const variantName = assertNonEmptyString(input.variant_name, "variant_name"); - const provider = assertNonEmptyString(input.provider, "provider"); - const model = assertNonEmptyString(input.model, "model"); - const judgeModel = - input.judge_model === null || input.judge_model === undefined - ? null - : assertNonEmptyString(input.judge_model, "judge_model"); - const runsPerCase = assertPositiveInteger(input.runs_per_case, "runs_per_case"); - const caseCount = assertPositiveInteger(input.case_count, "case_count"); - const metrics = normalizeMetrics(input.metrics, runsPerCase); - const cases = normalizeCases(input.cases); - - if (cases.length !== caseCount) { - throw new Error( - `case_count (${caseCount}) does not match cases.length (${cases.length})` - ); - } - - const runId = - input.run_id && typeof input.run_id === "string" && input.run_id.trim() - ? input.run_id.trim() - : buildRunId({ - timestamp, - surface, - variantName, - provider, - model, - gitSha - }); - - return { - ...input, - run_id: runId, - timestamp, - git_sha: gitSha, - suite_version: suiteVersion, - scoring_version: scoringVersion, - surface, - variant_name: variantName, - provider, - model, - judge_model: judgeModel, - runs_per_case: runsPerCase, - case_count: caseCount, - metrics, - cases - }; -} - -function normalizeMetrics(input, runsPerCase) { - assertPlainObject(input, "metrics"); - - const quality = normalizeMetricGroup( - input.quality, - "metrics.quality", - [ - "pass_rate", - "deterministic_pass_rate", - "judge_score_mean", - "judge_score_median", - "judge_score_p10", - "quality_score" - ], - new Set(["pass_rate", "deterministic_pass_rate"]) - ); - const reliability = normalizeMetricGroup( - input.reliability, - "metrics.reliability", - ["runs_per_case", "flake_rate", "path_consistency"], - new Set(["flake_rate", "path_consistency"]) - ); - const efficiency = normalizeMetricGroup( - input.efficiency, - "metrics.efficiency", - [ - "latency_ms_mean", - "latency_ms_median", - "tokens_total_mean", - "tool_calls_mean", - "iterations_mean", - "estimated_cost_mean", - "cost_per_success", - "latency_per_success", - "efficiency_score", - "value_score" - ] - ); - - if (reliability.runs_per_case !== runsPerCase) { - throw new Error( - `metrics.reliability.runs_per_case (${reliability.runs_per_case}) does not match runs_per_case (${runsPerCase})` - ); - } - - if (quality.category_pass_rate !== undefined) { - assertPlainObject(quality.category_pass_rate, "metrics.quality.category_pass_rate"); - for (const [category, value] of Object.entries(quality.category_pass_rate)) { - assertRatio(value, `metrics.quality.category_pass_rate.${category}`); - } - } - - return { quality, reliability, efficiency }; -} - -function normalizeMetricGroup(input, groupName, requiredFields, ratioFields = new Set()) { - assertPlainObject(input, groupName); - const normalized = { ...input }; - - for (const field of requiredFields) { - if (!(field in normalized)) { - throw new Error(`Missing required ${groupName}.${field}`); - } - if (field === "runs_per_case") { - normalized[field] = assertPositiveInteger( - normalized[field], - `${groupName}.${field}` - ); - continue; - } - normalized[field] = assertFiniteNumber(normalized[field], `${groupName}.${field}`); - if (ratioFields.has(field)) { - assertRatio(normalized[field], `${groupName}.${field}`); - } - } - - for (const [field, value] of Object.entries(normalized)) { - if (value === null || value === undefined) { - continue; - } - if (typeof value === "number") { - normalized[field] = assertFiniteNumber(value, `${groupName}.${field}`); - if (ratioFields.has(field)) { - assertRatio(normalized[field], `${groupName}.${field}`); - } - } - } - - return normalized; -} - -function normalizeCases(input) { - if (!Array.isArray(input) || input.length === 0) { - throw new Error("cases must be a non-empty array"); - } - - return input.map((entry, index) => { - assertPlainObject(entry, `cases[${index}]`); - - return { - ...entry, - id: assertNonEmptyString(entry.id, `cases[${index}].id`), - pass_rate: assertRatio(entry.pass_rate, `cases[${index}].pass_rate`) - }; - }); -} - -function buildRunId({ timestamp, surface, variantName, provider, model, gitSha }) { - const timestampSlug = timestamp.replaceAll(":", "-").replaceAll(".", "-"); - const shortSha = gitSha.slice(0, 12); - - return [ - slugify(timestampSlug), - slugify(surface), - slugify(variantName), - slugify(provider), - slugify(model), - shortSha - ] - .filter(Boolean) - .join("__"); -} - -function buildSummaryEntry(run, runRelativePath) { - return { - run_id: run.run_id, - timestamp: run.timestamp, - git_sha: run.git_sha, - suite_version: run.suite_version, - scoring_version: run.scoring_version, - surface: run.surface, - variant_name: run.variant_name, - provider: run.provider, - model: run.model, - judge_model: run.judge_model, - runs_per_case: run.runs_per_case, - case_count: run.case_count, - run_path: runRelativePath, - metrics: run.metrics - }; -} - -function upsertSummaryEntry(entries, nextEntry) { - const remainingEntries = entries.filter((entry) => entry.run_id !== nextEntry.run_id); - const nextEntries = [...remainingEntries, nextEntry]; - nextEntries.sort((left, right) => left.timestamp.localeCompare(right.timestamp)); - return nextEntries; -} - -async function loadSummaryEntries(summaryPath) { - try { - const raw = await readFile(summaryPath, "utf8"); - return raw - .split("\n") - .map((line) => line.trim()) - .filter(Boolean) - .map((line, index) => { - try { - return JSON.parse(line); - } catch (error) { - throw new Error( - `Failed to parse ${SUMMARY_FILENAME} line ${index + 1}: ${error.message}` - ); - } - }); - } catch (error) { - if (error.code === "ENOENT") { - return []; - } - throw error; - } -} - -async function writeSummaryEntries(summaryPath, entries) { - const content = - entries.map((entry) => JSON.stringify(entry)).join("\n") + - (entries.length > 0 ? "\n" : ""); - await writeFile(summaryPath, content, "utf8"); -} - -async function writeRollups(historyDir, summaries) { - const rollupsDir = path.join(historyDir, ROLLUPS_DIRNAME); - const generatedAt = new Date().toISOString(); - const latestFirst = [...summaries].sort((left, right) => - right.timestamp.localeCompare(left.timestamp) - ); - const latestBySurface = {}; - - for (const summary of latestFirst) { - if (!(summary.surface in latestBySurface)) { - latestBySurface[summary.surface] = summary; - } - } - - const latestRollup = { - generatedAt, - latestRun: latestFirst[0] ?? null, - latestBySurface - }; - const bySurfaceRollup = { - generatedAt, - groupKey: "surface", - groups: groupSummaries(summaries, (summary) => summary.surface) - }; - const byVariantRollup = { - generatedAt, - groupKey: "surface:variant_name", - groups: groupSummaries( - summaries, - (summary) => `${summary.surface}:${summary.variant_name}` - ) - }; - const byModelRollup = { - generatedAt, - groupKey: "provider:model", - groups: groupSummaries( - summaries, - (summary) => `${summary.provider}:${summary.model}` - ) - }; - - await Promise.all([ - writeJsonFile(path.join(rollupsDir, "latest.json"), latestRollup), - writeJsonFile(path.join(rollupsDir, "by_surface.json"), bySurfaceRollup), - writeJsonFile(path.join(rollupsDir, "by_variant.json"), byVariantRollup), - writeJsonFile(path.join(rollupsDir, "by_model.json"), byModelRollup) - ]); -} - -function groupSummaries(summaries, getGroupKey) { - const groups = {}; - - for (const summary of summaries) { - const key = getGroupKey(summary); - groups[key] ??= []; - groups[key].push(summary); - } - - for (const groupEntries of Object.values(groups)) { - groupEntries.sort((left, right) => left.timestamp.localeCompare(right.timestamp)); - } - - return groups; -} - -async function ensureHistoryLayout(historyDir) { - await Promise.all([ - mkdir(path.join(historyDir, RUNS_DIRNAME), { recursive: true }), - mkdir(path.join(historyDir, ROLLUPS_DIRNAME), { recursive: true }) - ]); -} - -async function writeJsonFile(filePath, value) { - await writeFile(filePath, JSON.stringify(value, null, 2) + "\n", "utf8"); -} - -function assertPlainObject(value, fieldName) { - if (!value || typeof value !== "object" || Array.isArray(value)) { - throw new Error(`${fieldName} must be an object`); - } -} - -function assertNonEmptyString(value, fieldName) { - if (typeof value !== "string" || value.trim().length === 0) { - throw new Error(`${fieldName} must be a non-empty string`); - } - return value.trim(); -} - -function assertPositiveInteger(value, fieldName) { - if (!Number.isInteger(value) || value < 1) { - throw new Error(`${fieldName} must be a positive integer`); - } - return value; -} - -function assertFiniteNumber(value, fieldName) { - if (typeof value !== "number" || !Number.isFinite(value)) { - throw new Error(`${fieldName} must be a finite number`); - } - return value; -} - -function assertRatio(value, fieldName) { - const normalized = assertFiniteNumber(value, fieldName); - if (normalized < 0 || normalized > 1) { - throw new Error(`${fieldName} must be between 0 and 1`); - } - return normalized; -} - -function assertIsoDateTime(value, fieldName) { - const normalized = assertNonEmptyString(value, fieldName); - if (Number.isNaN(Date.parse(normalized))) { - throw new Error(`${fieldName} must be a valid ISO date-time string`); - } - return normalized; -} - -function slugify(value) { - return value - .toLowerCase() - .replaceAll(/[^a-z0-9]+/g, "-") - .replaceAll(/^-+|-+$/g, ""); -} - main().catch((error) => { process.stderr.write(`${error.message}\n`); process.exit(1); diff --git a/docs/system-prompt-testing-status.md b/docs/system-prompt-testing-status.md index 439c0ecf2c..2f5286d53a 100644 --- a/docs/system-prompt-testing-status.md +++ b/docs/system-prompt-testing-status.md @@ -23,13 +23,12 @@ What is true today: - variants can be frozen as named snapshots through the benchmark CLI - repeated-run CLI benchmarking exists through `--runs` - basic CLI reliability metrics now exist -- benchmark history scaffolding exists in `ai_evals/history` +- benchmark history writing and reading now exist in the benchmark CLI What is not true yet: - frontend is not yet exposed through the benchmark CLI - token and cost metrics are not implemented -- official history writing is not yet wired into normal benchmark commands - no UI studio exists yet ## Comparison To The Plan @@ -84,11 +83,11 @@ Implemented: - `run` - `compare` - `snapshot-variant` +- `history` - CLI adapter selection through `--surface cli` Still missing: -- `history` command implementation - frontend adapter selection ### Phase 3: Replace the CLI smoke suite with real artifact evaluation @@ -149,7 +148,7 @@ Planned: Status: -- partially done +- mostly done Implemented: @@ -157,14 +156,18 @@ Implemented: - official run schema scaffold in `benchmark-run.schema.json` - `summary.jsonl` - rollup placeholders +- shared history writer under `ai_evals/history/writer.mjs` - snapshot writer script under `ai_evals/scripts/append-official-run.mjs` +- benchmark CLI wiring for `compare --write-history` +- benchmark CLI `history --view latest|summary|surface|variant|model` +- official run snapshots generated from repeated CLI benchmark results +- history rollups rebuilt from real benchmark writes Still missing: -- benchmark CLI wiring to emit official history snapshots - pass-rate summaries across repeated runs - worst-failure reporting -- chart-ready data generated from real benchmark runs instead of placeholders +- richer history filtering and reporting around those snapshots ### Phase 5: Finish the frontend black-box harness on top of the shared model @@ -253,7 +256,7 @@ The most important implemented changes so far are: - `wmill init` - Moved `wmill init` testing overrides to internal env vars instead of public flags - Added docs for variant workflows and benchmark usage -- Added benchmark history scaffolding +- Added official benchmark history writing and reading ## What Is Left To Do @@ -265,8 +268,11 @@ The highest-priority remaining work is: - latency - tool-call count - token and cost metrics if available -2. Implement official history writing from the benchmark CLI. -3. Expand the CLI case corpus to cover more real skill behavior. +2. Expand the CLI case corpus to cover more real skill behavior. +3. Add stronger official history summaries: + - worst-failure views + - better pass-rate rollups + - more filtering 4. Bring frontend behind the same benchmark CLI. 5. Add CI tiers. 6. Build the UI last. @@ -275,13 +281,13 @@ The highest-priority remaining work is: The best next implementation step is: -- official history writing from repeated CLI benchmark runs +- expand CLI coverage and enrich the official history metrics Reason: -- the current CLI harness can now detect meaningful skill regressions and basic flakiness -- the next missing layer is persisted benchmark history, not basic run aggregation -- history wiring is needed before CI and before any useful trend dashboard +- the current CLI harness can now detect meaningful skill regressions, write official snapshots, and read them back +- the next missing layer is better coverage and stronger metrics inside that tracked history +- CI and the future trend dashboard should build on the existing history path instead of inventing a second one ## Relevant Files