From 7da096fee370795daf00e43a34338a7f5fc9529e Mon Sep 17 00:00:00 2001 From: centdix Date: Tue, 31 Mar 2026 18:31:40 +0200 Subject: [PATCH] feat: add benchmark cli shell Co-Authored-By: Claude Opus 4.5 --- ai_evals/README.md | 14 +++ ai_evals/cli/README.md | 50 ++++++++ ai_evals/cli/index.ts | 257 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 321 insertions(+) create mode 100644 ai_evals/cli/README.md create mode 100644 ai_evals/cli/index.ts diff --git a/ai_evals/README.md b/ai_evals/README.md index c5b0ae23c7..0d00508482 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -42,3 +42,17 @@ Later phases should add: - official benchmark history snapshots and rollups - frontend `script` cases - frontend adapters aligned to the shared CLI-proven scoring model + +## Current Entry Point + +The benchmark CLI shell now lives under `ai_evals/cli/`. + +Current usage: + +```bash +bun ai_evals/cli/index.ts list-cases --surface cli +bun ai_evals/cli/index.ts run --surface cli --case bun-hello-script +``` + +At the moment this is intentionally thin and delegates to the bootstrap CLI +adapter code under `cli/test-skills/`. diff --git a/ai_evals/cli/README.md b/ai_evals/cli/README.md new file mode 100644 index 0000000000..3742421349 --- /dev/null +++ b/ai_evals/cli/README.md @@ -0,0 +1,50 @@ +# Benchmark CLI + +This directory is the start of the repo-level benchmark CLI described in +`docs/system-prompt-testing-plan.md`. + +## Current Scope + +The current implementation is intentionally small: + +- `run` command +- `cli` surface only +- runs the existing CLI artifact-eval adapter from `cli/test-skills/` + +This is a migration step away from using `cli/test-skills/` as the primary +entrypoint. + +## Usage + +List available CLI cases: + +```bash +bun ai_evals/cli/index.ts list-cases --surface cli +``` + +Run one CLI case: + +```bash +bun ai_evals/cli/index.ts run --surface cli --case bun-hello-script +``` + +Keep the temp workspace for inspection: + +```bash +bun ai_evals/cli/index.ts run --surface cli --case bun-hello-script --keep-workspace +``` + +Print machine-readable output: + +```bash +bun ai_evals/cli/index.ts run --surface cli --case bun-hello-script --json +``` + +## Next Steps + +Later iterations should add: + +- `compare` command +- `history` command +- frontend adapters +- shared result/history writing from this entrypoint diff --git a/ai_evals/cli/index.ts b/ai_evals/cli/index.ts new file mode 100644 index 0000000000..699af22441 --- /dev/null +++ b/ai_evals/cli/index.ts @@ -0,0 +1,257 @@ +#!/usr/bin/env bun + +import { + cleanupWorkspace, + loadCliArtifactEvalCases, + runCliArtifactEvalCase +} from "../../cli/test-skills/src/artifact-eval"; + +type CommandName = "run" | "list-cases" | "compare" | "history"; +type SurfaceName = "cli"; + +interface ParsedArgs { + command: CommandName; + surface?: string; + caseId?: string; + json: boolean; + keepWorkspace: boolean; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + switch (args.command) { + case "list-cases": + await handleListCases(args); + return; + case "run": + await handleRun(args); + return; + case "compare": + case "history": + throw new Error( + `'${args.command}' is not implemented yet in the repo-level benchmark CLI` + ); + default: + printHelp(); + throw new Error(`Unknown command: ${String(args.command)}`); + } +} + +async function handleListCases(args: ParsedArgs) { + const surface = requireSurface(args.surface); + switch (surface) { + case "cli": { + const cases = await loadCliArtifactEvalCases(); + const payload = { + surface, + cases: cases.map((entry) => ({ + id: entry.id, + description: entry.description ?? null + })) + }; + + if (args.json) { + process.stdout.write(JSON.stringify(payload, null, 2) + "\n"); + return; + } + + process.stdout.write(`Surface: ${surface}\n`); + for (const entry of payload.cases) { + process.stdout.write( + `- ${entry.id}${entry.description ? `: ${entry.description}` : ""}\n` + ); + } + return; + } + default: + assertNever(surface); + } +} + +async function handleRun(args: ParsedArgs) { + const surface = requireSurface(args.surface); + const caseId = requireCaseId(args.caseId); + + switch (surface) { + case "cli": { + const cases = await loadCliArtifactEvalCases(); + const evalCase = cases.find((entry) => entry.id === caseId); + if (!evalCase) { + throw new Error(`Unknown CLI case: ${caseId}`); + } + + const result = await runCliArtifactEvalCase(evalCase); + const payload = { + command: "run", + surface, + caseId: evalCase.id, + passed: result.passed, + workspaceKept: args.keepWorkspace, + workspaceDir: args.keepWorkspace ? result.workspaceDir : null, + checks: result.checks, + skillsInvoked: result.run.skillsInvoked, + toolsUsed: result.run.toolsUsed.map((tool) => tool.tool), + expectedFiles: result.expectedFiles.map((file) => ({ + path: file.path, + exists: file.exists + })) + }; + + try { + if (args.json) { + process.stdout.write(JSON.stringify(payload, null, 2) + "\n"); + } else { + printRunSummary(payload); + } + } finally { + if (!args.keepWorkspace) { + await cleanupWorkspace(result.workspaceDir); + } + } + + if (!result.passed) { + process.exitCode = 1; + } + return; + } + default: + assertNever(surface); + } +} + +function parseArgs(argv: string[]): ParsedArgs { + const [commandArg, ...rest] = argv; + + if (!commandArg || commandArg === "--help" || commandArg === "-h") { + printHelp(); + process.exit(0); + } + + if (!isCommandName(commandArg)) { + throw new Error(`Unknown command: ${commandArg}`); + } + + const parsed: ParsedArgs = { + command: commandArg, + json: false, + keepWorkspace: false + }; + + for (let index = 0; index < rest.length; index += 1) { + const arg = rest[index]; + + if (arg === "--surface") { + parsed.surface = rest[index + 1]; + index += 1; + continue; + } + + if (arg === "--case") { + parsed.caseId = rest[index + 1]; + index += 1; + continue; + } + + if (arg === "--json") { + parsed.json = true; + continue; + } + + if (arg === "--keep-workspace") { + parsed.keepWorkspace = true; + continue; + } + + if (arg === "--help" || arg === "-h") { + printHelp(); + process.exit(0); + } + + throw new Error(`Unknown argument: ${arg}`); + } + + return parsed; +} + +function isCommandName(value: string): value is CommandName { + return ( + value === "run" || + value === "list-cases" || + value === "compare" || + value === "history" + ); +} + +function requireSurface(surface: string | undefined): SurfaceName { + if (!surface) { + throw new Error("Missing required --surface argument"); + } + if (surface !== "cli") { + throw new Error(`Unsupported surface for now: ${surface}`); + } + return surface; +} + +function requireCaseId(caseId: string | undefined): string { + if (!caseId) { + throw new Error("Missing required --case argument"); + } + return caseId; +} + +function printRunSummary(payload: { + surface: SurfaceName; + caseId: string; + passed: boolean; + workspaceKept: boolean; + workspaceDir: string | null; + checks: Array<{ name: string; passed: boolean }>; + skillsInvoked: string[]; + toolsUsed: string[]; + expectedFiles: Array<{ path: string; exists: boolean }>; +}) { + process.stdout.write(`Surface: ${payload.surface}\n`); + process.stdout.write(`Case: ${payload.caseId}\n`); + process.stdout.write(`Passed: ${payload.passed ? "yes" : "no"}\n`); + process.stdout.write(`Skills: ${payload.skillsInvoked.join(", ") || "(none)"}\n`); + process.stdout.write(`Tools: ${payload.toolsUsed.join(", ") || "(none)"}\n`); + + if (payload.workspaceKept && payload.workspaceDir) { + process.stdout.write(`Workspace: ${payload.workspaceDir}\n`); + } + + process.stdout.write("Checks:\n"); + for (const check of payload.checks) { + process.stdout.write(`- [${check.passed ? "x" : " "}] ${check.name}\n`); + } + + process.stdout.write("Files:\n"); + for (const file of payload.expectedFiles) { + process.stdout.write(`- ${file.path}: ${file.exists ? "present" : "missing"}\n`); + } +} + +function printHelp() { + process.stdout.write( + [ + "Usage:", + " bun ai_evals/cli/index.ts list-cases --surface cli [--json]", + " bun ai_evals/cli/index.ts run --surface cli --case [--json] [--keep-workspace]", + " bun ai_evals/cli/index.ts compare", + " bun ai_evals/cli/index.ts history", + "", + "Current support:", + " surfaces: cli" + ].join("\n") + "\n" + ); +} + +function assertNever(value: never): never { + throw new Error(`Unexpected value: ${String(value)}`); +} + +main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exit(1); +});