From ef9f38082116aa5649e650dc87d2cf0d7f22d826 Mon Sep 17 00:00:00 2001 From: centdix Date: Wed, 1 Apr 2026 00:52:14 +0200 Subject: [PATCH] feat: add cli variant snapshot helper Co-Authored-By: Claude Opus 4.5 --- ai_evals/README.md | 1 + ai_evals/adapters/cli/variants.ts | 78 ++++++++++++++++++++++++- ai_evals/cli/README.md | 60 +++++++------------ ai_evals/cli/index.ts | 96 ++++++++++++++++++++++++++++++- cli/src/commands/init/init.ts | 17 +++--- cli/src/guidance/writer.ts | 4 ++ 6 files changed, 204 insertions(+), 52 deletions(-) diff --git a/ai_evals/README.md b/ai_evals/README.md index 9678dad82c..25cd332790 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -56,6 +56,7 @@ Current usage: ```bash cd ai_evals bun run cli -- list-variants --surface cli +bun run cli -- snapshot-variant --surface cli --variant candidate bun run cli -- list-cases --surface cli bun run cli -- run --surface cli --case bun-hello-script --variant baseline bun run cli -- compare --surface cli --case bun-hello-script --variant baseline --variant baseline diff --git a/ai_evals/adapters/cli/variants.ts b/ai_evals/adapters/cli/variants.ts index 9ed75ec8b5..6b523af78e 100644 --- a/ai_evals/adapters/cli/variants.ts +++ b/ai_evals/adapters/cli/variants.ts @@ -1,7 +1,13 @@ -import { readdir, readFile } from "fs/promises"; +import { mkdir, readFile, readdir, rm, writeFile } from "fs/promises"; import { join, resolve } from "path"; import { fileURLToPath } from "url"; import { getGeneratedSkillsSource } from "./runtime"; +import { + WMILL_INIT_AI_AGENTS_SOURCE_ENV, + WMILL_INIT_AI_CLAUDE_SOURCE_ENV, + WMILL_INIT_AI_SKILLS_SOURCE_ENV, + writeAiGuidanceFiles +} from "../../../cli/src/guidance/writer.ts"; type CliVariantSource = | { @@ -28,7 +34,20 @@ export interface CliVariant { claudeSourcePath?: string; } +export interface CliVariantSnapshotResult { + variantId: string; + manifestPath: string; + snapshotDir: string; + description: string; + usedOverrides: { + skillsSourcePath?: string; + agentsSourcePath?: string; + claudeSourcePath?: string; + }; +} + const VARIANTS_DIR = fileURLToPath(new URL("../../variants/cli", import.meta.url)); +const SNAPSHOTS_DIR = join(VARIANTS_DIR, "snapshots"); export async function loadCliVariants(): Promise { const filenames = (await readdir(VARIANTS_DIR)) @@ -67,6 +86,55 @@ export async function loadCliVariantById(variantId: string): Promise return variant; } +export async function snapshotCliVariant(options: { + variantId: string; + description?: string; +}): Promise { + validateVariantId(options.variantId); + + const snapshotDir = join(SNAPSHOTS_DIR, options.variantId); + const manifestPath = join(VARIANTS_DIR, `${options.variantId}.json`); + const usedOverrides = { + skillsSourcePath: process.env[WMILL_INIT_AI_SKILLS_SOURCE_ENV], + agentsSourcePath: process.env[WMILL_INIT_AI_AGENTS_SOURCE_ENV], + claudeSourcePath: process.env[WMILL_INIT_AI_CLAUDE_SOURCE_ENV] + }; + + await rm(snapshotDir, { recursive: true, force: true }); + await mkdir(snapshotDir, { recursive: true }); + await writeAiGuidanceFiles({ + targetDir: snapshotDir, + nonDottedPaths: true, + overwriteProjectGuidance: true, + skillsSourcePath: usedOverrides.skillsSourcePath, + agentsSourcePath: usedOverrides.agentsSourcePath, + claudeSourcePath: usedOverrides.claudeSourcePath + }); + + const manifest: CliVariantManifest = { + id: options.variantId, + description: + options.description ?? + `Snapshot of the current CLI guidance bundle stored under snapshots/${options.variantId}.`, + skillsSource: { + type: "path", + path: `./snapshots/${options.variantId}/.claude/skills` + }, + agentsSourcePath: `./snapshots/${options.variantId}/AGENTS.md`, + claudeSourcePath: `./snapshots/${options.variantId}/CLAUDE.md` + }; + + await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); + + return { + variantId: options.variantId, + manifestPath, + snapshotDir, + description: manifest.description ?? "", + usedOverrides + }; +} + function resolveVariantSkillsSource( skillsSource: CliVariantSource | undefined, manifestPath: string @@ -88,3 +156,11 @@ function resolveOptionalManifestPath( return resolve(join(manifestPath, ".."), inputPath); } + +function validateVariantId(variantId: string): void { + if (!/^[a-z0-9][a-z0-9_-]*$/i.test(variantId)) { + throw new Error( + `Invalid variant id '${variantId}'. Use only letters, numbers, '-' and '_'` + ); + } +} diff --git a/ai_evals/cli/README.md b/ai_evals/cli/README.md index 5cb8e8538b..9e6f4fcae2 100644 --- a/ai_evals/cli/README.md +++ b/ai_evals/cli/README.md @@ -41,6 +41,13 @@ cd ai_evals bun run cli -- list-variants --surface cli ``` +Snapshot the current guidance bundle into a named CLI variant: + +```bash +cd ai_evals +bun run cli -- snapshot-variant --surface cli --variant candidate --description "Candidate CLI guidance bundle" +``` + Run one CLI case: ```bash @@ -76,52 +83,21 @@ change improved the system, do not compare against the moving `baseline` variant alone. `baseline` points at the repo's current generated skills, so it changes when -the repo changes. To make a real before-vs-after comparison, freeze both sides -as path-based variants. +the repo changes. To make a real before-vs-after comparison, snapshot both +sides as named variants. -Create a snapshot directory: +Before changing the skills, freeze the current bundle: ```bash -mkdir -p ai_evals/variants/cli/snapshots +cd ai_evals +bun run cli -- snapshot-variant --surface cli --variant baseline-frozen --description "Frozen CLI skills before the change" ``` -Before changing the skill, snapshot the current generated skills: +After changing and regenerating the guidance, snapshot the candidate: ```bash -cp -R system_prompts/auto-generated/skills ai_evals/variants/cli/snapshots/baseline-skills -``` - -Create a frozen baseline variant manifest in -`ai_evals/variants/cli/baseline-frozen.json`: - -```json -{ - "id": "baseline-frozen", - "description": "Frozen CLI skills before the change", - "skillsSource": { - "type": "path", - "path": "./snapshots/baseline-skills" - } -} -``` - -After changing and regenerating the skills, snapshot the candidate: - -```bash -cp -R system_prompts/auto-generated/skills ai_evals/variants/cli/snapshots/candidate-skills -``` - -Create `ai_evals/variants/cli/candidate.json`: - -```json -{ - "id": "candidate", - "description": "CLI skills after the change", - "skillsSource": { - "type": "path", - "path": "./snapshots/candidate-skills" - } -} +cd ai_evals +bun run cli -- snapshot-variant --surface cli --variant candidate --description "CLI skills after the change" ``` Then compare them on one or more cases: @@ -167,6 +143,10 @@ WMILL_INIT_AI_SKILLS_SOURCE=./ai_evals/variants/cli/snapshots/candidate-skills w WMILL_INIT_AI_SKILLS_SOURCE=./ai_evals/variants/cli/snapshots/candidate-skills WMILL_INIT_AI_AGENTS_SOURCE=./my-candidate-AGENTS.md wmill init --use-default ``` +The `snapshot-variant` command also honors those same env vars. If they are +set when you run the snapshot command, it will freeze that overridden bundle +instead of the generated default. + ## Next Steps Later iterations should add: @@ -174,6 +154,6 @@ Later iterations should add: - `history` command - frontend adapters - repeated-run reliability mode -- frozen-variant helper commands +- variant cleanup and diff helpers - latency, token, and cost metrics in compare output - shared result/history writing from this entrypoint diff --git a/ai_evals/cli/index.ts b/ai_evals/cli/index.ts index 7a877deebc..6acbf95562 100644 --- a/ai_evals/cli/index.ts +++ b/ai_evals/cli/index.ts @@ -5,9 +5,20 @@ import { loadCliArtifactEvalCases, runCliArtifactEvalCase } from "../adapters/cli/artifact-eval"; -import { loadCliVariantById, loadCliVariants, type CliVariant } from "../adapters/cli/variants"; +import { + loadCliVariantById, + loadCliVariants, + snapshotCliVariant, + type CliVariant +} from "../adapters/cli/variants"; -type CommandName = "run" | "list-cases" | "list-variants" | "compare" | "history"; +type CommandName = + | "run" + | "list-cases" + | "list-variants" + | "snapshot-variant" + | "compare" + | "history"; type SurfaceName = "cli"; interface ParsedArgs { @@ -15,6 +26,7 @@ interface ParsedArgs { surface?: string; caseIds: string[]; variantIds: string[]; + description?: string; json: boolean; keepWorkspace: boolean; } @@ -29,6 +41,9 @@ async function main() { case "list-variants": await handleListVariants(args); return; + case "snapshot-variant": + await handleSnapshotVariant(args); + return; case "run": await handleRun(args); return; @@ -161,6 +176,40 @@ async function handleRun(args: ParsedArgs) { } } +async function handleSnapshotVariant(args: ParsedArgs) { + const surface = requireSurface(args.surface); + const variantId = requireSingleVariantId(args.variantIds, ""); + + switch (surface) { + case "cli": { + const result = await snapshotCliVariant({ + variantId, + description: args.description + }); + + const payload = { + command: "snapshot-variant", + surface, + variant: result.variantId, + description: result.description, + manifestPath: result.manifestPath, + snapshotDir: result.snapshotDir, + usedOverrides: result.usedOverrides + }; + + if (args.json) { + process.stdout.write(JSON.stringify(payload, null, 2) + "\n"); + } else { + printSnapshotSummary(payload); + } + + return; + } + default: + assertNever(surface); + } +} + async function handleCompare(args: ParsedArgs) { const surface = requireSurface(args.surface); @@ -254,6 +303,7 @@ function parseArgs(argv: string[]): ParsedArgs { command: commandArg, caseIds: [], variantIds: [], + description: undefined, json: false, keepWorkspace: false }; @@ -279,6 +329,12 @@ function parseArgs(argv: string[]): ParsedArgs { continue; } + if (arg === "--description") { + parsed.description = rest[index + 1]; + index += 1; + continue; + } + if (arg === "--json") { parsed.json = true; continue; @@ -305,6 +361,7 @@ function isCommandName(value: string): value is CommandName { value === "run" || value === "list-cases" || value === "list-variants" || + value === "snapshot-variant" || value === "compare" || value === "history" ); @@ -332,10 +389,13 @@ function requireSingleCaseId(caseIds: string[]): string { function requireSingleVariantId(variantIds: string[], fallback: string): string { if (variantIds.length === 0) { + if (!fallback) { + throw new Error("Missing required --variant argument"); + } return fallback; } if (variantIds.length > 1) { - throw new Error("run accepts only one --variant value"); + throw new Error("this command accepts only one --variant value"); } return variantIds[0]; } @@ -402,6 +462,35 @@ function printCompareSummary(payload: { } } +function printSnapshotSummary(payload: { + surface: SurfaceName; + variant: string; + description: string; + manifestPath: string; + snapshotDir: string; + usedOverrides: { + skillsSourcePath?: string; + agentsSourcePath?: string; + claudeSourcePath?: string; + }; +}) { + process.stdout.write(`Surface: ${payload.surface}\n`); + process.stdout.write(`Variant: ${payload.variant}\n`); + process.stdout.write(`Description: ${payload.description}\n`); + process.stdout.write(`Manifest: ${payload.manifestPath}\n`); + process.stdout.write(`Snapshot: ${payload.snapshotDir}\n`); + process.stdout.write("Overrides:\n"); + process.stdout.write( + `- skills: ${payload.usedOverrides.skillsSourcePath ?? "(generated default)"}\n` + ); + process.stdout.write( + `- agents: ${payload.usedOverrides.agentsSourcePath ?? "(generated default)"}\n` + ); + process.stdout.write( + `- claude: ${payload.usedOverrides.claudeSourcePath ?? "(generated default)"}\n` + ); +} + function labelVariantSelections( variants: CliVariant[] ): Array<{ label: string; variant: CliVariant }> { @@ -424,6 +513,7 @@ function printHelp() { "Usage:", " cd ai_evals && bun run cli -- list-cases --surface cli [--json]", " cd ai_evals && bun run cli -- list-variants --surface cli [--json]", + " cd ai_evals && bun run cli -- snapshot-variant --surface cli --variant [--description ] [--json]", " cd ai_evals && bun run cli -- run --surface cli --case [--variant ] [--json] [--keep-workspace]", " cd ai_evals && bun run cli -- compare --surface cli [--case ...] [--variant ...] [--json]", " cd ai_evals && bun run cli -- history", diff --git a/cli/src/commands/init/init.ts b/cli/src/commands/init/init.ts index 8117dbdd3e..73bc35c9c2 100644 --- a/cli/src/commands/init/init.ts +++ b/cli/src/commands/init/init.ts @@ -8,7 +8,12 @@ import { GlobalOptions } from "../../types.ts"; import { readLockfile } from "../../utils/metadata.ts"; import { getActiveWorkspaceOrFallback } from "../workspace/workspace.ts"; import { generateRTNamespace } from "../resource-type/resource-type.ts"; -import { writeAiGuidanceFiles } from "../../guidance/writer.ts"; +import { + WMILL_INIT_AI_AGENTS_SOURCE_ENV, + WMILL_INIT_AI_CLAUDE_SOURCE_ENV, + WMILL_INIT_AI_SKILLS_SOURCE_ENV, + writeAiGuidanceFiles, +} from "../../guidance/writer.ts"; import { generateCommentedTemplate } from "./template.ts"; export interface InitOptions { @@ -24,10 +29,6 @@ export interface InitOptions { bindProfile?: boolean; } -const WMILL_INIT_AI_SKILLS_SOURCE = "WMILL_INIT_AI_SKILLS_SOURCE"; -const WMILL_INIT_AI_AGENTS_SOURCE = "WMILL_INIT_AI_AGENTS_SOURCE"; -const WMILL_INIT_AI_CLAUDE_SOURCE = "WMILL_INIT_AI_CLAUDE_SOURCE"; - /** * Bootstrap a windmill project with a wmill.yaml file */ @@ -229,9 +230,9 @@ async function initAction(opts: InitOptions) { targetDir: ".", nonDottedPaths, overwriteProjectGuidance: false, - skillsSourcePath: process.env[WMILL_INIT_AI_SKILLS_SOURCE], - agentsSourcePath: process.env[WMILL_INIT_AI_AGENTS_SOURCE], - claudeSourcePath: process.env[WMILL_INIT_AI_CLAUDE_SOURCE], + skillsSourcePath: process.env[WMILL_INIT_AI_SKILLS_SOURCE_ENV], + agentsSourcePath: process.env[WMILL_INIT_AI_AGENTS_SOURCE_ENV], + claudeSourcePath: process.env[WMILL_INIT_AI_CLAUDE_SOURCE_ENV], }); if (guidanceResult.agentsWritten) { diff --git a/cli/src/guidance/writer.ts b/cli/src/guidance/writer.ts index dd75e83013..cebcf1560e 100644 --- a/cli/src/guidance/writer.ts +++ b/cli/src/guidance/writer.ts @@ -25,6 +25,10 @@ export interface WriteAiGuidanceResult { skillCount: number; } +export const WMILL_INIT_AI_SKILLS_SOURCE_ENV = "WMILL_INIT_AI_SKILLS_SOURCE"; +export const WMILL_INIT_AI_AGENTS_SOURCE_ENV = "WMILL_INIT_AI_AGENTS_SOURCE"; +export const WMILL_INIT_AI_CLAUDE_SOURCE_ENV = "WMILL_INIT_AI_CLAUDE_SOURCE"; + const CLAUDE_MD_DEFAULT = "Instructions are in @AGENTS.md\n"; export async function writeAiGuidanceFiles(