starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-12 00:06:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add cli variant snapshot helper

Browse Source 
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
centdix
2026-04-01 00:52:14 +02:00
co-authored by Claude Opus 4.5
parent 876943468b
commit ef9f380821
6 changed files with 204 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ai_evals/README.md
+1
View File
@@ -56,6 +56,7 @@ Current usage:
```bash
cd ai_evals
bun run cli -- list-variants --surface cli
bun run cli -- snapshot-variant --surface cli --variant candidate
bun run cli -- list-cases --surface cli
bun run cli -- run --surface cli --case bun-hello-script --variant baseline
bun run cli -- compare --surface cli --case bun-hello-script --variant baseline --variant baseline
ai_evals/adapters/cli/variants.ts
+77 -1
View File
@@ -1,7 +1,13 @@
import { readdir, readFile } from "fs/promises";
import { mkdir, readFile, readdir, rm, writeFile } from "fs/promises";
import { join, resolve } from "path";
import { fileURLToPath } from "url";
import { getGeneratedSkillsSource } from "./runtime";
import {
WMILL_INIT_AI_AGENTS_SOURCE_ENV,
WMILL_INIT_AI_CLAUDE_SOURCE_ENV,
WMILL_INIT_AI_SKILLS_SOURCE_ENV,
writeAiGuidanceFiles
} from "../../../cli/src/guidance/writer.ts";
type CliVariantSource =
| {
@@ -28,7 +34,20 @@ export interface CliVariant {
claudeSourcePath?: string;
}
export interface CliVariantSnapshotResult {
variantId: string;
manifestPath: string;
snapshotDir: string;
description: string;
usedOverrides: {
skillsSourcePath?: string;
agentsSourcePath?: string;
claudeSourcePath?: string;
};
}
const VARIANTS_DIR = fileURLToPath(new URL("../../variants/cli", import.meta.url));
const SNAPSHOTS_DIR = join(VARIANTS_DIR, "snapshots");
export async function loadCliVariants(): Promise<CliVariant[]> {
const filenames = (await readdir(VARIANTS_DIR))
@@ -67,6 +86,55 @@ export async function loadCliVariantById(variantId: string): Promise<CliVariant>
return variant;
}
export async function snapshotCliVariant(options: {
variantId: string;
description?: string;
}): Promise<CliVariantSnapshotResult> {
validateVariantId(options.variantId);
const snapshotDir = join(SNAPSHOTS_DIR, options.variantId);
const manifestPath = join(VARIANTS_DIR, `${options.variantId}.json`);
const usedOverrides = {
skillsSourcePath: process.env[WMILL_INIT_AI_SKILLS_SOURCE_ENV],
agentsSourcePath: process.env[WMILL_INIT_AI_AGENTS_SOURCE_ENV],
claudeSourcePath: process.env[WMILL_INIT_AI_CLAUDE_SOURCE_ENV]
};
await rm(snapshotDir, { recursive: true, force: true });
await mkdir(snapshotDir, { recursive: true });
await writeAiGuidanceFiles({
targetDir: snapshotDir,
nonDottedPaths: true,
overwriteProjectGuidance: true,
skillsSourcePath: usedOverrides.skillsSourcePath,
agentsSourcePath: usedOverrides.agentsSourcePath,
claudeSourcePath: usedOverrides.claudeSourcePath
});
const manifest: CliVariantManifest = {
id: options.variantId,
description:
options.description ??
`Snapshot of the current CLI guidance bundle stored under snapshots/${options.variantId}.`,
skillsSource: {
type: "path",
path: `./snapshots/${options.variantId}/.claude/skills`
},
agentsSourcePath: `./snapshots/${options.variantId}/AGENTS.md`,
claudeSourcePath: `./snapshots/${options.variantId}/CLAUDE.md`
};
await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
return {
variantId: options.variantId,
manifestPath,
snapshotDir,
description: manifest.description ?? "",
usedOverrides
};
}
function resolveVariantSkillsSource(
skillsSource: CliVariantSource | undefined,
manifestPath: string
@@ -88,3 +156,11 @@ function resolveOptionalManifestPath(
return resolve(join(manifestPath, ".."), inputPath);
}
function validateVariantId(variantId: string): void {
if (!/^[a-z0-9][a-z0-9_-]*$/i.test(variantId)) {
throw new Error(
`Invalid variant id '${variantId}'. Use only letters, numbers, '-' and '_'`
);
}
}
ai_evals/cli/README.md
+20 -40
View File
@@ -41,6 +41,13 @@ cd ai_evals
bun run cli -- list-variants --surface cli
```
Snapshot the current guidance bundle into a named CLI variant:
```bash
cd ai_evals
bun run cli -- snapshot-variant --surface cli --variant candidate --description "Candidate CLI guidance bundle"
```
Run one CLI case:
```bash
@@ -76,52 +83,21 @@ change improved the system, do not compare against the moving `baseline`
variant alone.
`baseline` points at the repo's current generated skills, so it changes when
the repo changes. To make a real before-vs-after comparison, freeze both sides
as path-based variants.
the repo changes. To make a real before-vs-after comparison, snapshot both
sides as named variants.
Create a snapshot directory:
Before changing the skills, freeze the current bundle:
```bash
mkdir -p ai_evals/variants/cli/snapshots
cd ai_evals
bun run cli -- snapshot-variant --surface cli --variant baseline-frozen --description "Frozen CLI skills before the change"
```
Before changing the skill, snapshot the current generated skills:
After changing and regenerating the guidance, snapshot the candidate:
```bash
cp -R system_prompts/auto-generated/skills ai_evals/variants/cli/snapshots/baseline-skills
```
Create a frozen baseline variant manifest in
`ai_evals/variants/cli/baseline-frozen.json`:
```json
{
"id": "baseline-frozen",
"description": "Frozen CLI skills before the change",
"skillsSource": {
"type": "path",
"path": "./snapshots/baseline-skills"
}
}
```
After changing and regenerating the skills, snapshot the candidate:
```bash
cp -R system_prompts/auto-generated/skills ai_evals/variants/cli/snapshots/candidate-skills
```
Create `ai_evals/variants/cli/candidate.json`:
```json
{
"id": "candidate",
"description": "CLI skills after the change",
"skillsSource": {
"type": "path",
"path": "./snapshots/candidate-skills"
}
}
cd ai_evals
bun run cli -- snapshot-variant --surface cli --variant candidate --description "CLI skills after the change"
```
Then compare them on one or more cases:
@@ -167,6 +143,10 @@ WMILL_INIT_AI_SKILLS_SOURCE=./ai_evals/variants/cli/snapshots/candidate-skills w
WMILL_INIT_AI_SKILLS_SOURCE=./ai_evals/variants/cli/snapshots/candidate-skills WMILL_INIT_AI_AGENTS_SOURCE=./my-candidate-AGENTS.md wmill init --use-default
```
The `snapshot-variant` command also honors those same env vars. If they are
set when you run the snapshot command, it will freeze that overridden bundle
instead of the generated default.
## Next Steps
Later iterations should add:
@@ -174,6 +154,6 @@ Later iterations should add:
- `history` command
- frontend adapters
- repeated-run reliability mode
- frozen-variant helper commands
- variant cleanup and diff helpers
- latency, token, and cost metrics in compare output
- shared result/history writing from this entrypoint
ai_evals/cli/index.ts
+93 -3
View File
@@ -5,9 +5,20 @@ import {
loadCliArtifactEvalCases,
runCliArtifactEvalCase
} from "../adapters/cli/artifact-eval";
import { loadCliVariantById, loadCliVariants, type CliVariant } from "../adapters/cli/variants";
import {
loadCliVariantById,
loadCliVariants,
snapshotCliVariant,
type CliVariant
} from "../adapters/cli/variants";
type CommandName = "run" | "list-cases" | "list-variants" | "compare" | "history";
type CommandName =
| "run"
| "list-cases"
| "list-variants"
| "snapshot-variant"
| "compare"
| "history";
type SurfaceName = "cli";
interface ParsedArgs {
@@ -15,6 +26,7 @@ interface ParsedArgs {
surface?: string;
caseIds: string[];
variantIds: string[];
description?: string;
json: boolean;
keepWorkspace: boolean;
}
@@ -29,6 +41,9 @@ async function main() {
case "list-variants":
await handleListVariants(args);
return;
case "snapshot-variant":
await handleSnapshotVariant(args);
return;
case "run":
await handleRun(args);
return;
@@ -161,6 +176,40 @@ async function handleRun(args: ParsedArgs) {
}
}
async function handleSnapshotVariant(args: ParsedArgs) {
const surface = requireSurface(args.surface);
const variantId = requireSingleVariantId(args.variantIds, "");
switch (surface) {
case "cli": {
const result = await snapshotCliVariant({
variantId,
description: args.description
});
const payload = {
command: "snapshot-variant",
surface,
variant: result.variantId,
description: result.description,
manifestPath: result.manifestPath,
snapshotDir: result.snapshotDir,
usedOverrides: result.usedOverrides
};
if (args.json) {
process.stdout.write(JSON.stringify(payload, null, 2) + "\n");
} else {
printSnapshotSummary(payload);
}
return;
}
default:
assertNever(surface);
}
}
async function handleCompare(args: ParsedArgs) {
const surface = requireSurface(args.surface);
@@ -254,6 +303,7 @@ function parseArgs(argv: string[]): ParsedArgs {
command: commandArg,
caseIds: [],
variantIds: [],
description: undefined,
json: false,
keepWorkspace: false
};
@@ -279,6 +329,12 @@ function parseArgs(argv: string[]): ParsedArgs {
continue;
}
if (arg === "--description") {
parsed.description = rest[index + 1];
index += 1;
continue;
}
if (arg === "--json") {
parsed.json = true;
continue;
@@ -305,6 +361,7 @@ function isCommandName(value: string): value is CommandName {
value === "run" ||
value === "list-cases" ||
value === "list-variants" ||
value === "snapshot-variant" ||
value === "compare" ||
value === "history"
);
@@ -332,10 +389,13 @@ function requireSingleCaseId(caseIds: string[]): string {
function requireSingleVariantId(variantIds: string[], fallback: string): string {
if (variantIds.length === 0) {
if (!fallback) {
throw new Error("Missing required --variant argument");
}
return fallback;
}
if (variantIds.length > 1) {
throw new Error("run accepts only one --variant value");
throw new Error("this command accepts only one --variant value");
}
return variantIds[0];
}
@@ -402,6 +462,35 @@ function printCompareSummary(payload: {
}
}
function printSnapshotSummary(payload: {
surface: SurfaceName;
variant: string;
description: string;
manifestPath: string;
snapshotDir: string;
usedOverrides: {
skillsSourcePath?: string;
agentsSourcePath?: string;
claudeSourcePath?: string;
};
}) {
process.stdout.write(`Surface: ${payload.surface}\n`);
process.stdout.write(`Variant: ${payload.variant}\n`);
process.stdout.write(`Description: ${payload.description}\n`);
process.stdout.write(`Manifest: ${payload.manifestPath}\n`);
process.stdout.write(`Snapshot: ${payload.snapshotDir}\n`);
process.stdout.write("Overrides:\n");
process.stdout.write(
`- skills: ${payload.usedOverrides.skillsSourcePath ?? "(generated default)"}\n`
);
process.stdout.write(
`- agents: ${payload.usedOverrides.agentsSourcePath ?? "(generated default)"}\n`
);
process.stdout.write(
`- claude: ${payload.usedOverrides.claudeSourcePath ?? "(generated default)"}\n`
);
}
function labelVariantSelections(
variants: CliVariant[]
): Array<{ label: string; variant: CliVariant }> {
@@ -424,6 +513,7 @@ function printHelp() {
"Usage:",
" cd ai_evals && bun run cli -- list-cases --surface cli [--json]",
" cd ai_evals && bun run cli -- list-variants --surface cli [--json]",
" cd ai_evals && bun run cli -- snapshot-variant --surface cli --variant <id> [--description <text>] [--json]",
" cd ai_evals && bun run cli -- run --surface cli --case <id> [--variant <id>] [--json] [--keep-workspace]",
" cd ai_evals && bun run cli -- compare --surface cli [--case <id> ...] [--variant <id> ...] [--json]",
" cd ai_evals && bun run cli -- history",
cli/src/commands/init/init.ts
+9 -8
View File
@@ -8,7 +8,12 @@ import { GlobalOptions } from "../../types.ts";
import { readLockfile } from "../../utils/metadata.ts";
import { getActiveWorkspaceOrFallback } from "../workspace/workspace.ts";
import { generateRTNamespace } from "../resource-type/resource-type.ts";
import { writeAiGuidanceFiles } from "../../guidance/writer.ts";
import {
WMILL_INIT_AI_AGENTS_SOURCE_ENV,
WMILL_INIT_AI_CLAUDE_SOURCE_ENV,
WMILL_INIT_AI_SKILLS_SOURCE_ENV,
writeAiGuidanceFiles,
} from "../../guidance/writer.ts";
import { generateCommentedTemplate } from "./template.ts";
export interface InitOptions {
@@ -24,10 +29,6 @@ export interface InitOptions {
bindProfile?: boolean;
}
const WMILL_INIT_AI_SKILLS_SOURCE = "WMILL_INIT_AI_SKILLS_SOURCE";
const WMILL_INIT_AI_AGENTS_SOURCE = "WMILL_INIT_AI_AGENTS_SOURCE";
const WMILL_INIT_AI_CLAUDE_SOURCE = "WMILL_INIT_AI_CLAUDE_SOURCE";
/**
* Bootstrap a windmill project with a wmill.yaml file
*/
@@ -229,9 +230,9 @@ async function initAction(opts: InitOptions) {
targetDir: ".",
nonDottedPaths,
overwriteProjectGuidance: false,
skillsSourcePath: process.env[WMILL_INIT_AI_SKILLS_SOURCE],
agentsSourcePath: process.env[WMILL_INIT_AI_AGENTS_SOURCE],
claudeSourcePath: process.env[WMILL_INIT_AI_CLAUDE_SOURCE],
skillsSourcePath: process.env[WMILL_INIT_AI_SKILLS_SOURCE_ENV],
agentsSourcePath: process.env[WMILL_INIT_AI_AGENTS_SOURCE_ENV],
claudeSourcePath: process.env[WMILL_INIT_AI_CLAUDE_SOURCE_ENV],
});
if (guidanceResult.agentsWritten) {
cli/src/guidance/writer.ts
+4
View File
@@ -25,6 +25,10 @@ export interface WriteAiGuidanceResult {
skillCount: number;
}
export const WMILL_INIT_AI_SKILLS_SOURCE_ENV = "WMILL_INIT_AI_SKILLS_SOURCE";
export const WMILL_INIT_AI_AGENTS_SOURCE_ENV = "WMILL_INIT_AI_AGENTS_SOURCE";
export const WMILL_INIT_AI_CLAUDE_SOURCE_ENV = "WMILL_INIT_AI_CLAUDE_SOURCE";
const CLAUDE_MD_DEFAULT = "Instructions are in @AGENTS.md\n";
export async function writeAiGuidanceFiles(