feat: share cli ai guidance generation

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

ai_evals/adapters/cli/artifact-eval.ts

@@ -1,10 +1,9 @@
 import { existsSync } from "fs";
-import { cp, mkdtemp, mkdir, readdir, readFile, rm, writeFile } from "fs/promises";
+import { mkdtemp, mkdir, readdir, readFile, rm, writeFile } from "fs/promises";
 import { tmpdir } from "os";
 import { dirname, join } from "path";
 import { fileURLToPath } from "url";
-import { generateAgentsMdContent } from "../../../cli/src/guidance/core.ts";
-import { SKILLS } from "../../../cli/src/guidance/skills.ts";
+import { writeAiGuidanceFiles } from "../../../cli/src/guidance/writer.ts";
 import {
   runPromptAndCapture,
   type PromptRunResult,
@@ -86,7 +85,7 @@ export async function runCliArtifactEvalCase(
     variant: CliVariant;
   }
 ): Promise<CliArtifactEvalResult> {
-  const workspaceDir = await createIsolatedWorkspace(evalCase.id, options.variant.skillsSourcePath);
+  const workspaceDir = await createIsolatedWorkspace(evalCase.id, options.variant);
 
   try {
     const renderedPrompt = await renderPrompt(evalCase.prompt, workspaceDir);
@@ -125,33 +124,23 @@ export function shouldKeepWorkspace(): boolean {
 
 async function createIsolatedWorkspace(
   caseId: string,
-  skillsSourcePath: string
+  variant: CliVariant
 ): Promise<string> {
   const workspaceDir = await mkdtemp(join(tmpdir(), `wmill-cli-artifact-${caseId}-`));
-  const skillsDir = join(workspaceDir, ".claude", "skills");
-
-  await mkdir(dirname(skillsDir), { recursive: true });
-  await cp(skillsSourcePath, skillsDir, { recursive: true });
-  await writeProjectGuidance(workspaceDir);
+  await mkdir(dirname(join(workspaceDir, ".claude", "skills")), { recursive: true });
+  await writeAiGuidanceFiles({
+    targetDir: workspaceDir,
+    nonDottedPaths: true,
+    overwriteProjectGuidance: true,
+    skillsSourcePath: variant.skillsSourcePath,
+    agentsSourcePath: variant.agentsSourcePath,
+    claudeSourcePath: variant.claudeSourcePath,
+  });
   await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8");
 
   return workspaceDir;
 }
 
-async function writeProjectGuidance(workspaceDir: string): Promise<void> {
-  const skillsBaseDir = ".claude/skills";
-  const skillsReference = SKILLS.map(
-    (skill) => `- \`${skillsBaseDir}/${skill.name}/SKILL.md\` - ${skill.description}`
-  ).join("\n");
-
-  await writeFile(
-    join(workspaceDir, "AGENTS.md"),
-    generateAgentsMdContent(skillsReference),
-    "utf8"
-  );
-  await writeFile(join(workspaceDir, "CLAUDE.md"), "Instructions are in @AGENTS.md\n", "utf8");
-}
-
 async function renderPrompt(prompt: string, workspaceDir: string): Promise<string> {
   const renderedUserPrompt = prompt.replaceAll("{{workspace_root}}", workspaceDir);
   const agentsInstructions = await readFile(join(workspaceDir, "AGENTS.md"), "utf8");

ai_evals/adapters/cli/variants.ts

@@ -16,12 +16,16 @@ interface CliVariantManifest {
   id: string;
   description?: string;
   skillsSource: CliVariantSource;
+  agentsSourcePath?: string;
+  claudeSourcePath?: string;
 }
 
 export interface CliVariant {
   id: string;
   description?: string;
   skillsSourcePath: string;
+  agentsSourcePath?: string;
+  claudeSourcePath?: string;
 }
 
 const VARIANTS_DIR = fileURLToPath(new URL("../../variants/cli", import.meta.url));
@@ -45,7 +49,9 @@ export async function loadCliVariants(): Promise<CliVariant[]> {
     variants.push({
       id: parsed.id,
       description: parsed.description,
-      skillsSourcePath: resolveVariantSkillsSource(parsed.skillsSource, manifestPath)
+      skillsSourcePath: resolveVariantSkillsSource(parsed.skillsSource, manifestPath),
+      agentsSourcePath: resolveOptionalManifestPath(parsed.agentsSourcePath, manifestPath),
+      claudeSourcePath: resolveOptionalManifestPath(parsed.claudeSourcePath, manifestPath)
     });
   }
 
@@ -71,3 +77,14 @@ function resolveVariantSkillsSource(
 
   return resolve(join(manifestPath, ".."), skillsSource.path);
 }
+
+function resolveOptionalManifestPath(
+  inputPath: string | undefined,
+  manifestPath: string
+): string | undefined {
+  if (!inputPath) {
+    return undefined;
+  }
+
+  return resolve(join(manifestPath, ".."), inputPath);
+}

ai_evals/cli/README.md

@@ -14,6 +14,10 @@ The current implementation is intentionally small:
 
 This is the benchmark entrypoint for prompt and artifact evaluation.
 
+The CLI benchmark workspace now uses the same shared AI-guidance writer as
+`wmill init`, so benchmark runs and local CLI bootstraps go through the same
+project-guidance generation path.
+
 ## Usage
 
 Install dependencies once:
@@ -140,6 +144,29 @@ the current CLI does not emit them yet. Until that lands, use `compare`
 primarily to answer "did this skill bundle produce better artifacts on the same
 cases?"
 
+Variant manifests can also override the top-level project instructions in
+addition to the skills bundle:
+
+```json
+{
+  "id": "candidate",
+  "description": "Candidate guidance bundle",
+  "skillsSource": {
+    "type": "path",
+    "path": "./snapshots/candidate-skills"
+  },
+  "agentsSourcePath": "./snapshots/candidate-AGENTS.md",
+  "claudeSourcePath": "./snapshots/candidate-CLAUDE.md"
+}
+```
+
+That matches the new `wmill init` overrides:
+
+```bash
+wmill init --use-default --ai-skills-source ./ai_evals/variants/cli/snapshots/candidate-skills
+wmill init --use-default --ai-skills-source ./ai_evals/variants/cli/snapshots/candidate-skills --ai-agents-source ./my-candidate-AGENTS.md
+```
+
 ## Next Steps
 
 Later iterations should add:

cli/README.md

@@ -110,6 +110,27 @@ source <(wmill completions zsh)
 
 ## Development
 
+### AI Guidance Variants
+
+`wmill init` can now materialize alternate AI guidance bundles without changing
+the generated defaults in the repo.
+
+Examples:
+
+```bash
+wmill init --use-default --ai-skills-source /path/to/custom/skills
+wmill init --use-default --ai-skills-source /path/to/custom/skills --ai-agents-source /path/to/AGENTS.md
+wmill init --use-default --ai-skills-source /path/to/custom/skills --ai-claude-source /path/to/CLAUDE.md
+```
+
+This is the same guidance-writing path used by the benchmark CLI under
+`ai_evals/`, so the benchmark harness and `wmill init` now generate the same
+project guidance shape:
+
+- `AGENTS.md`
+- `CLAUDE.md`
+- `.claude/skills/*`
+
 ### Testing with a local `windmill-yaml-validator`
 
 To test local changes to the validator before publishing, use `npm link`:

cli/src/commands/init/init.ts

@@ -1,4 +1,4 @@
-import { stat, writeFile, rm, mkdir } from "node:fs/promises";
+import { stat, writeFile, rm } from "node:fs/promises";
 import { colors } from "@cliffy/ansi/colors";
 import { Command } from "@cliffy/command";
 import { Confirm } from "@cliffy/prompt/confirm";
@@ -8,23 +8,9 @@ import { GlobalOptions } from "../../types.ts";
 import { readLockfile } from "../../utils/metadata.ts";
 import { getActiveWorkspaceOrFallback } from "../workspace/workspace.ts";
 import { generateRTNamespace } from "../resource-type/resource-type.ts";
-import { SKILLS, SKILL_CONTENT, SCHEMAS, SCHEMA_MAPPINGS } from "../../guidance/skills.ts";
-import { generateAgentsMdContent } from "../../guidance/core.ts";
+import { writeAiGuidanceFiles } from "../../guidance/writer.ts";
 import { generateCommentedTemplate } from "./template.ts";
 
-/**
- * Format a YAML schema for inclusion in skill markdown files.
- */
-function formatSchemaForMarkdown(schemaYaml: string, schemaName: string, filePattern: string): string {
-  return `## ${schemaName} (\`${filePattern}\`)
-
-Must be a YAML file that adheres to the following schema:
-
-\`\`\`yaml
-${schemaYaml.trim()}
-\`\`\``;
-}
-
 export interface InitOptions {
   useDefault?: boolean;
   useBackend?: boolean;
@@ -36,6 +22,9 @@ export interface InitOptions {
   baseUrl?: string;
   configDir?: string;
   bindProfile?: boolean;
+  aiSkillsSource?: string;
+  aiAgentsSource?: string;
+  aiClaudeSource?: string;
 }
 
 /**
@@ -235,88 +224,24 @@ async function initAction(opts: InitOptions) {
 
   // Create guidance files (AGENTS.md, CLAUDE.md, and Claude skills)
   try {
-    // Generate skills reference section for AGENTS.md
-    const skills_base_dir = ".claude/skills";
-    const skillsReference = SKILLS.map(
-      (s) => `- \`${skills_base_dir}/${s.name}/SKILL.md\` - ${s.description}`
-    ).join("\n");
+    const guidanceResult = await writeAiGuidanceFiles({
+      targetDir: ".",
+      nonDottedPaths,
+      overwriteProjectGuidance: false,
+      skillsSourcePath: opts.aiSkillsSource,
+      agentsSourcePath: opts.aiAgentsSource,
+      claudeSourcePath: opts.aiClaudeSource,
+    });
 
-    // Create AGENTS.md file with minimal instructions
-    if (!(await stat("AGENTS.md").catch(() => null))) {
-      await writeFile(
-        "AGENTS.md",
-        generateAgentsMdContent(skillsReference), "utf-8"
-      );
+    if (guidanceResult.agentsWritten) {
       log.info(colors.green("Created AGENTS.md"));
     }
-
-    // Create CLAUDE.md file, referencing AGENTS.md
-    if (!(await stat("CLAUDE.md").catch(() => null))) {
-      await writeFile(
-        "CLAUDE.md",
-        `Instructions are in @AGENTS.md
-`, "utf-8"
-      );
+    if (guidanceResult.claudeWritten) {
       log.info(colors.green("Created CLAUDE.md"));
     }
-
-    // Create .claude/skills/ directory and skill files
-    try {
-      await mkdir(".claude/skills", { recursive: true });
-
-      await Promise.all(
-        SKILLS.map(async (skill) => {
-          const skillDir = `.claude/skills/${skill.name}`;
-          await mkdir(skillDir, { recursive: true });
-
-          let skillContent = SKILL_CONTENT[skill.name];
-          if (skillContent) {
-            // Replace placeholders with actual suffixes based on nonDottedPaths
-            if (nonDottedPaths) {
-              skillContent = skillContent
-                .replaceAll("{{FLOW_SUFFIX}}", "__flow")
-                .replaceAll("{{APP_SUFFIX}}", "__app")
-                .replaceAll("{{RAW_APP_SUFFIX}}", "__raw_app")
-                .replaceAll("{{INLINE_SCRIPT_NAMING}}", "Inline script files should NOT include `.inline_script.` in their names (e.g. use `a.ts`, not `a.inline_script.ts`).");
-            } else {
-              skillContent = skillContent
-                .replaceAll("{{FLOW_SUFFIX}}", ".flow")
-                .replaceAll("{{APP_SUFFIX}}", ".app")
-                .replaceAll("{{RAW_APP_SUFFIX}}", ".raw_app")
-                .replaceAll("{{INLINE_SCRIPT_NAMING}}", "Inline script files use the `.inline_script.` naming convention (e.g. `a.inline_script.ts`).");
-            }
-            // Check if this skill has schemas that need to be appended
-            const schemaMappings = SCHEMA_MAPPINGS[skill.name];
-            if (schemaMappings && schemaMappings.length > 0) {
-              // Combine base content with schemas
-              const schemaDocs = schemaMappings
-                .map((mapping) => {
-                  const schemaYaml = SCHEMAS[mapping.schemaKey];
-                  if (schemaYaml) {
-                    return formatSchemaForMarkdown(schemaYaml, mapping.name, mapping.filePattern);
-                  }
-                  return null;
-                })
-                .filter((doc): doc is string => doc !== null);
-
-              if (schemaDocs.length > 0) {
-                skillContent = skillContent + "\n\n" + schemaDocs.join("\n\n");
-              }
-            }
-
-            await writeFile(`${skillDir}/SKILL.md`, skillContent, "utf-8");
-          }
-        })
-      );
-
-      log.info(colors.green(`Created .claude/skills/ with ${SKILLS.length} skills`));
-    } catch (skillError) {
-      if (skillError instanceof Error) {
-        log.warn(`Could not create skills: ${skillError.message}`);
-      } else {
-        log.warn(`Could not create skills: ${skillError}`);
-      }
-    }
+    log.info(
+      colors.green(`Created .claude/skills/ with ${guidanceResult.skillCount} skills`)
+    );
   } catch (error) {
     if (error instanceof Error) {
       log.warn(`Could not create guidance files: ${error.message}`);
@@ -341,6 +266,18 @@ const command = new Command()
   .description("Bootstrap a windmill project with a wmill.yaml file")
   .option("--use-default", "Use default settings without checking backend")
   .option("--use-backend", "Use backend git-sync settings if available")
+  .option(
+    "--ai-skills-source <path:string>",
+    "Use a custom skills directory instead of the generated .claude/skills bundle"
+  )
+  .option(
+    "--ai-agents-source <path:string>",
+    "Use a custom AGENTS.md file for AI guidance"
+  )
+  .option(
+    "--ai-claude-source <path:string>",
+    "Use a custom CLAUDE.md file for AI guidance"
+  )
   .option(
     "--repository <repo:string>",
     "Specify repository path (e.g., u/user/repo) when using backend settings"

cli/src/guidance/writer.ts

@@ -0,0 +1,229 @@
+import { cp, mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { generateAgentsMdContent } from "./core.ts";
+import {
+  SCHEMAS,
+  SCHEMA_MAPPINGS,
+  SKILLS,
+  SKILL_CONTENT,
+  type SkillMetadata,
+} from "./skills.ts";
+
+export interface WriteAiGuidanceOptions {
+  targetDir: string;
+  nonDottedPaths?: boolean;
+  overwriteProjectGuidance?: boolean;
+  skillsSourcePath?: string;
+  agentsSourcePath?: string;
+  claudeSourcePath?: string;
+}
+
+export interface WriteAiGuidanceResult {
+  agentsWritten: boolean;
+  claudeWritten: boolean;
+  skillsWritten: boolean;
+  skillCount: number;
+}
+
+const CLAUDE_MD_DEFAULT = "Instructions are in @AGENTS.md\n";
+
+export async function writeAiGuidanceFiles(
+  options: WriteAiGuidanceOptions
+): Promise<WriteAiGuidanceResult> {
+  const nonDottedPaths = options.nonDottedPaths ?? true;
+  const skillMetadata = options.skillsSourcePath
+    ? await copySkillsFromSource(options.targetDir, options.skillsSourcePath)
+    : await writeGeneratedSkills(options.targetDir, nonDottedPaths);
+
+  const agentsWritten = await writeProjectGuidanceFile({
+    targetPath: join(options.targetDir, "AGENTS.md"),
+    overwrite: options.overwriteProjectGuidance ?? false,
+    content:
+      options.agentsSourcePath != null
+        ? await readFile(options.agentsSourcePath, "utf8")
+        : generateAgentsMdContent(buildSkillsReference(skillMetadata)),
+  });
+
+  const claudeWritten = await writeProjectGuidanceFile({
+    targetPath: join(options.targetDir, "CLAUDE.md"),
+    overwrite: options.overwriteProjectGuidance ?? false,
+    content:
+      options.claudeSourcePath != null
+        ? await readFile(options.claudeSourcePath, "utf8")
+        : CLAUDE_MD_DEFAULT,
+  });
+
+  return {
+    agentsWritten,
+    claudeWritten,
+    skillsWritten: true,
+    skillCount: skillMetadata.length,
+  };
+}
+
+function buildSkillsReference(skills: Pick<SkillMetadata, "name" | "description">[]): string {
+  return skills
+    .map((skill) => `- \`.claude/skills/${skill.name}/SKILL.md\` - ${skill.description}`)
+    .join("\n");
+}
+
+async function copySkillsFromSource(
+  targetDir: string,
+  skillsSourcePath: string
+): Promise<SkillMetadata[]> {
+  const skillsDir = join(targetDir, ".claude", "skills");
+  await mkdir(join(targetDir, ".claude"), { recursive: true });
+  await rm(skillsDir, { recursive: true, force: true });
+  await cp(skillsSourcePath, skillsDir, { recursive: true, force: true });
+  return await readSkillMetadataFromDirectory(skillsDir);
+}
+
+async function writeGeneratedSkills(
+  targetDir: string,
+  nonDottedPaths: boolean
+): Promise<SkillMetadata[]> {
+  const skillsDir = join(targetDir, ".claude", "skills");
+  await rm(skillsDir, { recursive: true, force: true });
+  await mkdir(skillsDir, { recursive: true });
+
+  await Promise.all(
+    SKILLS.map(async (skill) => {
+      const skillDir = join(skillsDir, skill.name);
+      await mkdir(skillDir, { recursive: true });
+      await writeFile(
+        join(skillDir, "SKILL.md"),
+        renderGeneratedSkillContent(skill.name, nonDottedPaths),
+        "utf8"
+      );
+    })
+  );
+
+  return SKILLS;
+}
+
+function renderGeneratedSkillContent(skillName: string, nonDottedPaths: boolean): string {
+  let skillContent = SKILL_CONTENT[skillName];
+  if (!skillContent) {
+    throw new Error(`Missing generated skill content for ${skillName}`);
+  }
+
+  if (nonDottedPaths) {
+    skillContent = skillContent
+      .replaceAll("{{FLOW_SUFFIX}}", "__flow")
+      .replaceAll("{{APP_SUFFIX}}", "__app")
+      .replaceAll("{{RAW_APP_SUFFIX}}", "__raw_app")
+      .replaceAll(
+        "{{INLINE_SCRIPT_NAMING}}",
+        "Inline script files should NOT include `.inline_script.` in their names (e.g. use `a.ts`, not `a.inline_script.ts`)."
+      );
+  } else {
+    skillContent = skillContent
+      .replaceAll("{{FLOW_SUFFIX}}", ".flow")
+      .replaceAll("{{APP_SUFFIX}}", ".app")
+      .replaceAll("{{RAW_APP_SUFFIX}}", ".raw_app")
+      .replaceAll(
+        "{{INLINE_SCRIPT_NAMING}}",
+        "Inline script files use the `.inline_script.` naming convention (e.g. `a.inline_script.ts`)."
+      );
+  }
+
+  const schemaMappings = SCHEMA_MAPPINGS[skillName];
+  if (!schemaMappings || schemaMappings.length === 0) {
+    return skillContent;
+  }
+
+  const schemaDocs = schemaMappings
+    .map((mapping) => {
+      const schemaYaml = SCHEMAS[mapping.schemaKey];
+      if (!schemaYaml) {
+        return null;
+      }
+      return formatSchemaForMarkdown(schemaYaml, mapping.name, mapping.filePattern);
+    })
+    .filter((entry): entry is string => entry !== null);
+
+  if (schemaDocs.length === 0) {
+    return skillContent;
+  }
+
+  return `${skillContent}\n\n${schemaDocs.join("\n\n")}`;
+}
+
+async function readSkillMetadataFromDirectory(skillsDir: string): Promise<SkillMetadata[]> {
+  const entries = await readdir(skillsDir, { withFileTypes: true });
+  const skills: SkillMetadata[] = [];
+
+  for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
+    if (!entry.isDirectory()) {
+      continue;
+    }
+
+    const skillPath = join(skillsDir, entry.name, "SKILL.md");
+    if (!(await stat(skillPath).catch(() => null))) {
+      continue;
+    }
+
+    const content = await readFile(skillPath, "utf8");
+    skills.push(parseSkillMetadata(content, entry.name));
+  }
+
+  return skills;
+}
+
+function parseSkillMetadata(content: string, fallbackName: string): SkillMetadata {
+  const frontMatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
+  if (!frontMatterMatch) {
+    return {
+      name: fallbackName,
+      description: `Skill loaded from ${fallbackName}`,
+    };
+  }
+
+  let name = fallbackName;
+  let description = `Skill loaded from ${fallbackName}`;
+
+  for (const line of frontMatterMatch[1].split("\n")) {
+    const separatorIndex = line.indexOf(":");
+    if (separatorIndex === -1) {
+      continue;
+    }
+
+    const key = line.slice(0, separatorIndex).trim();
+    const value = line.slice(separatorIndex + 1).trim();
+
+    if (key === "name" && value) {
+      name = value;
+    } else if (key === "description" && value) {
+      description = value;
+    }
+  }
+
+  return { name, description };
+}
+
+async function writeProjectGuidanceFile(options: {
+  targetPath: string;
+  content: string;
+  overwrite: boolean;
+}): Promise<boolean> {
+  if (!options.overwrite && (await stat(options.targetPath).catch(() => null))) {
+    return false;
+  }
+
+  await writeFile(options.targetPath, options.content, "utf8");
+  return true;
+}
+
+function formatSchemaForMarkdown(
+  schemaYaml: string,
+  schemaName: string,
+  filePattern: string
+): string {
+  return `## ${schemaName} (\`${filePattern}\`)
+
+Must be a YAML file that adheres to the following schema:
+
+\`\`\`yaml
+${schemaYaml.trim()}
+\`\`\``;
+}