From 3e214f7992fd75c90fcf053ec528ecf836357618 Mon Sep 17 00:00:00 2001 From: centdix Date: Tue, 31 Mar 2026 18:57:53 +0200 Subject: [PATCH] feat: add benchmark cli compare command Co-Authored-By: Claude Opus 4.5 --- ai_evals/README.md | 4 +- ai_evals/adapters/cli/artifact-eval.ts | 21 ++- ai_evals/adapters/cli/variants.ts | 73 +++++++++ ai_evals/cli/README.md | 20 ++- ai_evals/cli/index.ts | 206 +++++++++++++++++++++++-- ai_evals/variants/cli/baseline.json | 7 + 6 files changed, 310 insertions(+), 21 deletions(-) create mode 100644 ai_evals/adapters/cli/variants.ts create mode 100644 ai_evals/variants/cli/baseline.json diff --git a/ai_evals/README.md b/ai_evals/README.md index 1cb983983c..b03b127387 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -55,8 +55,10 @@ Current usage: ```bash cd ai_evals +bun run cli -- list-variants --surface cli bun run cli -- list-cases --surface cli -bun run cli -- run --surface cli --case bun-hello-script +bun run cli -- run --surface cli --case bun-hello-script --variant baseline +bun run cli -- compare --surface cli --case bun-hello-script --variant baseline --variant baseline ``` At the moment this is still intentionally small, but it is the only benchmark diff --git a/ai_evals/adapters/cli/artifact-eval.ts b/ai_evals/adapters/cli/artifact-eval.ts index 3c1e56e851..805e28dcc8 100644 --- a/ai_evals/adapters/cli/artifact-eval.ts +++ b/ai_evals/adapters/cli/artifact-eval.ts @@ -4,11 +4,11 @@ import { tmpdir } from "os"; import { dirname, join } from "path"; import { fileURLToPath } from "url"; import { - getGeneratedSkillsSource, runPromptAndCapture, type PromptRunResult, wasSkillInvoked } from "./runtime"; +import type { CliVariant } from "./variants"; export interface ExpectedFile { path: string; @@ -46,6 +46,7 @@ export interface CliArtifactEvalResult { checks: ArtifactCheck[]; expectedFiles: FileArtifactResult[]; passed: boolean; + variantId: string; } const CASES_DIR = fileURLToPath(new URL("../../cases/cli", import.meta.url)); @@ -72,9 +73,12 @@ export async function loadCliArtifactEvalCases(): Promise } export async function runCliArtifactEvalCase( - evalCase: CliArtifactEvalCase + evalCase: CliArtifactEvalCase, + options: { + variant: CliVariant; + } ): Promise { - const workspaceDir = await createIsolatedWorkspace(evalCase.id); + const workspaceDir = await createIsolatedWorkspace(evalCase.id, options.variant.skillsSourcePath); try { const renderedPrompt = renderPrompt(evalCase.prompt, workspaceDir); @@ -92,7 +96,8 @@ export async function runCliArtifactEvalCase( run, checks, expectedFiles: fileResults, - passed: checks.every((check) => check.required === false || check.passed) + passed: checks.every((check) => check.required === false || check.passed), + variantId: options.variant.id }; } catch (error) { if (!shouldKeepWorkspace()) { @@ -110,13 +115,15 @@ export function shouldKeepWorkspace(): boolean { return process.env.WMILL_CLI_EVAL_KEEP_WORKSPACE === "1"; } -async function createIsolatedWorkspace(caseId: string): Promise { +async function createIsolatedWorkspace( + caseId: string, + skillsSourcePath: string +): Promise { const workspaceDir = await mkdtemp(join(tmpdir(), `wmill-cli-artifact-${caseId}-`)); const skillsDir = join(workspaceDir, ".claude", "skills"); - const generatedSkillsSource = getGeneratedSkillsSource(); await mkdir(dirname(skillsDir), { recursive: true }); - await cp(generatedSkillsSource, skillsDir, { recursive: true }); + await cp(skillsSourcePath, skillsDir, { recursive: true }); await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8"); return workspaceDir; diff --git a/ai_evals/adapters/cli/variants.ts b/ai_evals/adapters/cli/variants.ts new file mode 100644 index 0000000000..be669595d2 --- /dev/null +++ b/ai_evals/adapters/cli/variants.ts @@ -0,0 +1,73 @@ +import { readdir, readFile } from "fs/promises"; +import { join, resolve } from "path"; +import { fileURLToPath } from "url"; +import { getGeneratedSkillsSource } from "./runtime"; + +type CliVariantSource = + | { + type: "generated"; + } + | { + type: "path"; + path: string; + }; + +interface CliVariantManifest { + id: string; + description?: string; + skillsSource: CliVariantSource; +} + +export interface CliVariant { + id: string; + description?: string; + skillsSourcePath: string; +} + +const VARIANTS_DIR = fileURLToPath(new URL("../../variants/cli", import.meta.url)); + +export async function loadCliVariants(): Promise { + const filenames = (await readdir(VARIANTS_DIR)) + .filter((entry) => entry.endsWith(".json")) + .sort((left, right) => left.localeCompare(right)); + + const variants: CliVariant[] = []; + + for (const filename of filenames) { + const manifestPath = join(VARIANTS_DIR, filename); + const raw = await readFile(manifestPath, "utf8"); + const parsed = JSON.parse(raw) as CliVariantManifest; + + if (!parsed.id) { + throw new Error(`Missing variant id in ${manifestPath}`); + } + + variants.push({ + id: parsed.id, + description: parsed.description, + skillsSourcePath: resolveVariantSkillsSource(parsed.skillsSource, manifestPath) + }); + } + + return variants; +} + +export async function loadCliVariantById(variantId: string): Promise { + const variants = await loadCliVariants(); + const variant = variants.find((entry) => entry.id === variantId); + if (!variant) { + throw new Error(`Unknown CLI variant: ${variantId}`); + } + return variant; +} + +function resolveVariantSkillsSource( + skillsSource: CliVariantSource | undefined, + manifestPath: string +): string { + if (!skillsSource || skillsSource.type === "generated") { + return getGeneratedSkillsSource(); + } + + return resolve(join(manifestPath, ".."), skillsSource.path); +} diff --git a/ai_evals/cli/README.md b/ai_evals/cli/README.md index b65975bca6..58a7f602e5 100644 --- a/ai_evals/cli/README.md +++ b/ai_evals/cli/README.md @@ -28,25 +28,39 @@ cd ai_evals bun run cli -- list-cases --surface cli ``` +List available CLI variants: + +```bash +cd ai_evals +bun run cli -- list-variants --surface cli +``` + Run one CLI case: ```bash cd ai_evals -bun run cli -- run --surface cli --case bun-hello-script +bun run cli -- run --surface cli --case bun-hello-script --variant baseline ``` Keep the temp workspace for inspection: ```bash cd ai_evals -bun run cli -- run --surface cli --case bun-hello-script --keep-workspace +bun run cli -- run --surface cli --case bun-hello-script --variant baseline --keep-workspace ``` Print machine-readable output: ```bash cd ai_evals -bun run cli -- run --surface cli --case bun-hello-script --json +bun run cli -- run --surface cli --case bun-hello-script --variant baseline --json +``` + +Compare two variant selections on one or more cases: + +```bash +cd ai_evals +bun run cli -- compare --surface cli --case bun-hello-script --variant baseline --variant baseline --json ``` ## Next Steps diff --git a/ai_evals/cli/index.ts b/ai_evals/cli/index.ts index 44ef8c8e02..7a877deebc 100644 --- a/ai_evals/cli/index.ts +++ b/ai_evals/cli/index.ts @@ -5,14 +5,16 @@ import { loadCliArtifactEvalCases, runCliArtifactEvalCase } from "../adapters/cli/artifact-eval"; +import { loadCliVariantById, loadCliVariants, type CliVariant } from "../adapters/cli/variants"; -type CommandName = "run" | "list-cases" | "compare" | "history"; +type CommandName = "run" | "list-cases" | "list-variants" | "compare" | "history"; type SurfaceName = "cli"; interface ParsedArgs { command: CommandName; surface?: string; - caseId?: string; + caseIds: string[]; + variantIds: string[]; json: boolean; keepWorkspace: boolean; } @@ -24,10 +26,15 @@ async function main() { case "list-cases": await handleListCases(args); return; + case "list-variants": + await handleListVariants(args); + return; case "run": await handleRun(args); return; case "compare": + await handleCompare(args); + return; case "history": throw new Error( `'${args.command}' is not implemented yet in the repo-level benchmark CLI` @@ -69,9 +76,41 @@ async function handleListCases(args: ParsedArgs) { } } +async function handleListVariants(args: ParsedArgs) { + const surface = requireSurface(args.surface); + + switch (surface) { + case "cli": { + const variants = await loadCliVariants(); + const payload = { + surface, + variants: variants.map((entry) => ({ + id: entry.id, + description: entry.description ?? null + })) + }; + + if (args.json) { + process.stdout.write(JSON.stringify(payload, null, 2) + "\n"); + return; + } + + process.stdout.write(`Surface: ${surface}\n`); + for (const entry of payload.variants) { + process.stdout.write( + `- ${entry.id}${entry.description ? `: ${entry.description}` : ""}\n` + ); + } + return; + } + default: + assertNever(surface); + } +} + async function handleRun(args: ParsedArgs) { const surface = requireSurface(args.surface); - const caseId = requireCaseId(args.caseId); + const caseId = requireSingleCaseId(args.caseIds); switch (surface) { case "cli": { @@ -80,11 +119,13 @@ async function handleRun(args: ParsedArgs) { if (!evalCase) { throw new Error(`Unknown CLI case: ${caseId}`); } + const variant = await loadCliVariantById(requireSingleVariantId(args.variantIds, "baseline")); - const result = await runCliArtifactEvalCase(evalCase); + const result = await runCliArtifactEvalCase(evalCase, { variant }); const payload = { command: "run", surface, + variant: variant.id, caseId: evalCase.id, passed: result.passed, workspaceKept: args.keepWorkspace, @@ -120,6 +161,83 @@ async function handleRun(args: ParsedArgs) { } } +async function handleCompare(args: ParsedArgs) { + const surface = requireSurface(args.surface); + + switch (surface) { + case "cli": { + const allCases = await loadCliArtifactEvalCases(); + const selectedCases = + args.caseIds.length === 0 + ? allCases + : args.caseIds.map((caseId) => { + const evalCase = allCases.find((entry) => entry.id === caseId); + if (!evalCase) { + throw new Error(`Unknown CLI case: ${caseId}`); + } + return evalCase; + }); + + if (args.variantIds.length < 2) { + throw new Error("compare requires at least two --variant values"); + } + + const variants = await Promise.all( + args.variantIds.map((variantId) => loadCliVariantById(variantId)) + ); + const labeledVariants = labelVariantSelections(variants); + const variantResults = []; + + for (const labeledVariant of labeledVariants) { + const caseResults = []; + + for (const evalCase of selectedCases) { + const result = await runCliArtifactEvalCase(evalCase, { + variant: labeledVariant.variant + }); + + try { + caseResults.push({ + caseId: evalCase.id, + passed: result.passed, + skillsInvoked: result.run.skillsInvoked, + toolsUsed: result.run.toolsUsed.map((tool) => tool.tool), + checks: result.checks + }); + } finally { + await cleanupWorkspace(result.workspaceDir); + } + } + + variantResults.push({ + label: labeledVariant.label, + variant: labeledVariant.variant.id, + totalCases: caseResults.length, + passedCases: caseResults.filter((entry) => entry.passed).length, + caseResults + }); + } + + const payload = { + command: "compare", + surface, + caseIds: selectedCases.map((entry) => entry.id), + variants: variantResults + }; + + if (args.json) { + process.stdout.write(JSON.stringify(payload, null, 2) + "\n"); + } else { + printCompareSummary(payload); + } + + return; + } + default: + assertNever(surface); + } +} + function parseArgs(argv: string[]): ParsedArgs { const [commandArg, ...rest] = argv; @@ -134,6 +252,8 @@ function parseArgs(argv: string[]): ParsedArgs { const parsed: ParsedArgs = { command: commandArg, + caseIds: [], + variantIds: [], json: false, keepWorkspace: false }; @@ -148,7 +268,13 @@ function parseArgs(argv: string[]): ParsedArgs { } if (arg === "--case") { - parsed.caseId = rest[index + 1]; + parsed.caseIds.push(rest[index + 1]); + index += 1; + continue; + } + + if (arg === "--variant") { + parsed.variantIds.push(rest[index + 1]); index += 1; continue; } @@ -178,6 +304,7 @@ function isCommandName(value: string): value is CommandName { return ( value === "run" || value === "list-cases" || + value === "list-variants" || value === "compare" || value === "history" ); @@ -193,15 +320,29 @@ function requireSurface(surface: string | undefined): SurfaceName { return surface; } -function requireCaseId(caseId: string | undefined): string { - if (!caseId) { +function requireSingleCaseId(caseIds: string[]): string { + if (caseIds.length === 0) { throw new Error("Missing required --case argument"); } - return caseId; + if (caseIds.length > 1) { + throw new Error("run accepts only one --case value"); + } + return caseIds[0]; +} + +function requireSingleVariantId(variantIds: string[], fallback: string): string { + if (variantIds.length === 0) { + return fallback; + } + if (variantIds.length > 1) { + throw new Error("run accepts only one --variant value"); + } + return variantIds[0]; } function printRunSummary(payload: { surface: SurfaceName; + variant: string; caseId: string; passed: boolean; workspaceKept: boolean; @@ -212,6 +353,7 @@ function printRunSummary(payload: { expectedFiles: Array<{ path: string; exists: boolean }>; }) { process.stdout.write(`Surface: ${payload.surface}\n`); + process.stdout.write(`Variant: ${payload.variant}\n`); process.stdout.write(`Case: ${payload.caseId}\n`); process.stdout.write(`Passed: ${payload.passed ? "yes" : "no"}\n`); process.stdout.write(`Skills: ${payload.skillsInvoked.join(", ") || "(none)"}\n`); @@ -233,13 +375,57 @@ function printRunSummary(payload: { } } +function printCompareSummary(payload: { + surface: SurfaceName; + caseIds: string[]; + variants: Array<{ + label: string; + variant: string; + totalCases: number; + passedCases: number; + caseResults: Array<{ caseId: string; passed: boolean }>; + }>; +}) { + process.stdout.write(`Surface: ${payload.surface}\n`); + process.stdout.write(`Cases: ${payload.caseIds.join(", ")}\n`); + process.stdout.write("Variants:\n"); + + for (const variant of payload.variants) { + process.stdout.write( + `- ${variant.label}: ${variant.passedCases}/${variant.totalCases} cases passed\n` + ); + for (const caseResult of variant.caseResults) { + process.stdout.write( + ` ${caseResult.caseId}: ${caseResult.passed ? "pass" : "fail"}\n` + ); + } + } +} + +function labelVariantSelections( + variants: CliVariant[] +): Array<{ label: string; variant: CliVariant }> { + const seenCounts = new Map(); + + return variants.map((variant) => { + const count = (seenCounts.get(variant.id) ?? 0) + 1; + seenCounts.set(variant.id, count); + + return { + label: count === 1 ? variant.id : `${variant.id}#${count}`, + variant + }; + }); +} + function printHelp() { process.stdout.write( [ "Usage:", " cd ai_evals && bun run cli -- list-cases --surface cli [--json]", - " cd ai_evals && bun run cli -- run --surface cli --case [--json] [--keep-workspace]", - " cd ai_evals && bun run cli -- compare", + " cd ai_evals && bun run cli -- list-variants --surface cli [--json]", + " cd ai_evals && bun run cli -- run --surface cli --case [--variant ] [--json] [--keep-workspace]", + " cd ai_evals && bun run cli -- compare --surface cli [--case ...] [--variant ...] [--json]", " cd ai_evals && bun run cli -- history", "", "Current support:", diff --git a/ai_evals/variants/cli/baseline.json b/ai_evals/variants/cli/baseline.json new file mode 100644 index 0000000000..de48e0d512 --- /dev/null +++ b/ai_evals/variants/cli/baseline.json @@ -0,0 +1,7 @@ +{ + "id": "baseline", + "description": "Uses the repo's generated Windmill CLI skills.", + "skillsSource": { + "type": "generated" + } +}