From 5a48886b4ce118f2c64e0e4bd086bef8bac8cb09 Mon Sep 17 00:00:00 2001 From: centdix Date: Tue, 31 Mar 2026 22:53:16 +0200 Subject: [PATCH] feat: share cli ai guidance generation Co-Authored-By: Claude Opus 4.5 --- ai_evals/adapters/cli/artifact-eval.ts | 37 ++-- ai_evals/adapters/cli/variants.ts | 19 +- ai_evals/cli/README.md | 27 +++ cli/README.md | 21 +++ cli/src/commands/init/init.ts | 123 ++++--------- cli/src/guidance/writer.ts | 229 +++++++++++++++++++++++++ 6 files changed, 338 insertions(+), 118 deletions(-) create mode 100644 cli/src/guidance/writer.ts diff --git a/ai_evals/adapters/cli/artifact-eval.ts b/ai_evals/adapters/cli/artifact-eval.ts index 766ef5d0d5..765f6f9815 100644 --- a/ai_evals/adapters/cli/artifact-eval.ts +++ b/ai_evals/adapters/cli/artifact-eval.ts @@ -1,10 +1,9 @@ import { existsSync } from "fs"; -import { cp, mkdtemp, mkdir, readdir, readFile, rm, writeFile } from "fs/promises"; +import { mkdtemp, mkdir, readdir, readFile, rm, writeFile } from "fs/promises"; import { tmpdir } from "os"; import { dirname, join } from "path"; import { fileURLToPath } from "url"; -import { generateAgentsMdContent } from "../../../cli/src/guidance/core.ts"; -import { SKILLS } from "../../../cli/src/guidance/skills.ts"; +import { writeAiGuidanceFiles } from "../../../cli/src/guidance/writer.ts"; import { runPromptAndCapture, type PromptRunResult, @@ -86,7 +85,7 @@ export async function runCliArtifactEvalCase( variant: CliVariant; } ): Promise { - const workspaceDir = await createIsolatedWorkspace(evalCase.id, options.variant.skillsSourcePath); + const workspaceDir = await createIsolatedWorkspace(evalCase.id, options.variant); try { const renderedPrompt = await renderPrompt(evalCase.prompt, workspaceDir); @@ -125,33 +124,23 @@ export function shouldKeepWorkspace(): boolean { async function createIsolatedWorkspace( caseId: string, - skillsSourcePath: string + variant: CliVariant ): Promise { const workspaceDir = await mkdtemp(join(tmpdir(), `wmill-cli-artifact-${caseId}-`)); - const skillsDir = join(workspaceDir, ".claude", "skills"); - - await mkdir(dirname(skillsDir), { recursive: true }); - await cp(skillsSourcePath, skillsDir, { recursive: true }); - await writeProjectGuidance(workspaceDir); + await mkdir(dirname(join(workspaceDir, ".claude", "skills")), { recursive: true }); + await writeAiGuidanceFiles({ + targetDir: workspaceDir, + nonDottedPaths: true, + overwriteProjectGuidance: true, + skillsSourcePath: variant.skillsSourcePath, + agentsSourcePath: variant.agentsSourcePath, + claudeSourcePath: variant.claudeSourcePath, + }); await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8"); return workspaceDir; } -async function writeProjectGuidance(workspaceDir: string): Promise { - const skillsBaseDir = ".claude/skills"; - const skillsReference = SKILLS.map( - (skill) => `- \`${skillsBaseDir}/${skill.name}/SKILL.md\` - ${skill.description}` - ).join("\n"); - - await writeFile( - join(workspaceDir, "AGENTS.md"), - generateAgentsMdContent(skillsReference), - "utf8" - ); - await writeFile(join(workspaceDir, "CLAUDE.md"), "Instructions are in @AGENTS.md\n", "utf8"); -} - async function renderPrompt(prompt: string, workspaceDir: string): Promise { const renderedUserPrompt = prompt.replaceAll("{{workspace_root}}", workspaceDir); const agentsInstructions = await readFile(join(workspaceDir, "AGENTS.md"), "utf8"); diff --git a/ai_evals/adapters/cli/variants.ts b/ai_evals/adapters/cli/variants.ts index be669595d2..9ed75ec8b5 100644 --- a/ai_evals/adapters/cli/variants.ts +++ b/ai_evals/adapters/cli/variants.ts @@ -16,12 +16,16 @@ interface CliVariantManifest { id: string; description?: string; skillsSource: CliVariantSource; + agentsSourcePath?: string; + claudeSourcePath?: string; } export interface CliVariant { id: string; description?: string; skillsSourcePath: string; + agentsSourcePath?: string; + claudeSourcePath?: string; } const VARIANTS_DIR = fileURLToPath(new URL("../../variants/cli", import.meta.url)); @@ -45,7 +49,9 @@ export async function loadCliVariants(): Promise { variants.push({ id: parsed.id, description: parsed.description, - skillsSourcePath: resolveVariantSkillsSource(parsed.skillsSource, manifestPath) + skillsSourcePath: resolveVariantSkillsSource(parsed.skillsSource, manifestPath), + agentsSourcePath: resolveOptionalManifestPath(parsed.agentsSourcePath, manifestPath), + claudeSourcePath: resolveOptionalManifestPath(parsed.claudeSourcePath, manifestPath) }); } @@ -71,3 +77,14 @@ function resolveVariantSkillsSource( return resolve(join(manifestPath, ".."), skillsSource.path); } + +function resolveOptionalManifestPath( + inputPath: string | undefined, + manifestPath: string +): string | undefined { + if (!inputPath) { + return undefined; + } + + return resolve(join(manifestPath, ".."), inputPath); +} diff --git a/ai_evals/cli/README.md b/ai_evals/cli/README.md index 51bd06b097..ad7a262489 100644 --- a/ai_evals/cli/README.md +++ b/ai_evals/cli/README.md @@ -14,6 +14,10 @@ The current implementation is intentionally small: This is the benchmark entrypoint for prompt and artifact evaluation. +The CLI benchmark workspace now uses the same shared AI-guidance writer as +`wmill init`, so benchmark runs and local CLI bootstraps go through the same +project-guidance generation path. + ## Usage Install dependencies once: @@ -140,6 +144,29 @@ the current CLI does not emit them yet. Until that lands, use `compare` primarily to answer "did this skill bundle produce better artifacts on the same cases?" +Variant manifests can also override the top-level project instructions in +addition to the skills bundle: + +```json +{ + "id": "candidate", + "description": "Candidate guidance bundle", + "skillsSource": { + "type": "path", + "path": "./snapshots/candidate-skills" + }, + "agentsSourcePath": "./snapshots/candidate-AGENTS.md", + "claudeSourcePath": "./snapshots/candidate-CLAUDE.md" +} +``` + +That matches the new `wmill init` overrides: + +```bash +wmill init --use-default --ai-skills-source ./ai_evals/variants/cli/snapshots/candidate-skills +wmill init --use-default --ai-skills-source ./ai_evals/variants/cli/snapshots/candidate-skills --ai-agents-source ./my-candidate-AGENTS.md +``` + ## Next Steps Later iterations should add: diff --git a/cli/README.md b/cli/README.md index ee3b68c35f..0a476da9e5 100644 --- a/cli/README.md +++ b/cli/README.md @@ -110,6 +110,27 @@ source <(wmill completions zsh) ## Development +### AI Guidance Variants + +`wmill init` can now materialize alternate AI guidance bundles without changing +the generated defaults in the repo. + +Examples: + +```bash +wmill init --use-default --ai-skills-source /path/to/custom/skills +wmill init --use-default --ai-skills-source /path/to/custom/skills --ai-agents-source /path/to/AGENTS.md +wmill init --use-default --ai-skills-source /path/to/custom/skills --ai-claude-source /path/to/CLAUDE.md +``` + +This is the same guidance-writing path used by the benchmark CLI under +`ai_evals/`, so the benchmark harness and `wmill init` now generate the same +project guidance shape: + +- `AGENTS.md` +- `CLAUDE.md` +- `.claude/skills/*` + ### Testing with a local `windmill-yaml-validator` To test local changes to the validator before publishing, use `npm link`: diff --git a/cli/src/commands/init/init.ts b/cli/src/commands/init/init.ts index db4950575b..49606479c8 100644 --- a/cli/src/commands/init/init.ts +++ b/cli/src/commands/init/init.ts @@ -1,4 +1,4 @@ -import { stat, writeFile, rm, mkdir } from "node:fs/promises"; +import { stat, writeFile, rm } from "node:fs/promises"; import { colors } from "@cliffy/ansi/colors"; import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; @@ -8,23 +8,9 @@ import { GlobalOptions } from "../../types.ts"; import { readLockfile } from "../../utils/metadata.ts"; import { getActiveWorkspaceOrFallback } from "../workspace/workspace.ts"; import { generateRTNamespace } from "../resource-type/resource-type.ts"; -import { SKILLS, SKILL_CONTENT, SCHEMAS, SCHEMA_MAPPINGS } from "../../guidance/skills.ts"; -import { generateAgentsMdContent } from "../../guidance/core.ts"; +import { writeAiGuidanceFiles } from "../../guidance/writer.ts"; import { generateCommentedTemplate } from "./template.ts"; -/** - * Format a YAML schema for inclusion in skill markdown files. - */ -function formatSchemaForMarkdown(schemaYaml: string, schemaName: string, filePattern: string): string { - return `## ${schemaName} (\`${filePattern}\`) - -Must be a YAML file that adheres to the following schema: - -\`\`\`yaml -${schemaYaml.trim()} -\`\`\``; -} - export interface InitOptions { useDefault?: boolean; useBackend?: boolean; @@ -36,6 +22,9 @@ export interface InitOptions { baseUrl?: string; configDir?: string; bindProfile?: boolean; + aiSkillsSource?: string; + aiAgentsSource?: string; + aiClaudeSource?: string; } /** @@ -235,88 +224,24 @@ async function initAction(opts: InitOptions) { // Create guidance files (AGENTS.md, CLAUDE.md, and Claude skills) try { - // Generate skills reference section for AGENTS.md - const skills_base_dir = ".claude/skills"; - const skillsReference = SKILLS.map( - (s) => `- \`${skills_base_dir}/${s.name}/SKILL.md\` - ${s.description}` - ).join("\n"); + const guidanceResult = await writeAiGuidanceFiles({ + targetDir: ".", + nonDottedPaths, + overwriteProjectGuidance: false, + skillsSourcePath: opts.aiSkillsSource, + agentsSourcePath: opts.aiAgentsSource, + claudeSourcePath: opts.aiClaudeSource, + }); - // Create AGENTS.md file with minimal instructions - if (!(await stat("AGENTS.md").catch(() => null))) { - await writeFile( - "AGENTS.md", - generateAgentsMdContent(skillsReference), "utf-8" - ); + if (guidanceResult.agentsWritten) { log.info(colors.green("Created AGENTS.md")); } - - // Create CLAUDE.md file, referencing AGENTS.md - if (!(await stat("CLAUDE.md").catch(() => null))) { - await writeFile( - "CLAUDE.md", - `Instructions are in @AGENTS.md -`, "utf-8" - ); + if (guidanceResult.claudeWritten) { log.info(colors.green("Created CLAUDE.md")); } - - // Create .claude/skills/ directory and skill files - try { - await mkdir(".claude/skills", { recursive: true }); - - await Promise.all( - SKILLS.map(async (skill) => { - const skillDir = `.claude/skills/${skill.name}`; - await mkdir(skillDir, { recursive: true }); - - let skillContent = SKILL_CONTENT[skill.name]; - if (skillContent) { - // Replace placeholders with actual suffixes based on nonDottedPaths - if (nonDottedPaths) { - skillContent = skillContent - .replaceAll("{{FLOW_SUFFIX}}", "__flow") - .replaceAll("{{APP_SUFFIX}}", "__app") - .replaceAll("{{RAW_APP_SUFFIX}}", "__raw_app") - .replaceAll("{{INLINE_SCRIPT_NAMING}}", "Inline script files should NOT include `.inline_script.` in their names (e.g. use `a.ts`, not `a.inline_script.ts`)."); - } else { - skillContent = skillContent - .replaceAll("{{FLOW_SUFFIX}}", ".flow") - .replaceAll("{{APP_SUFFIX}}", ".app") - .replaceAll("{{RAW_APP_SUFFIX}}", ".raw_app") - .replaceAll("{{INLINE_SCRIPT_NAMING}}", "Inline script files use the `.inline_script.` naming convention (e.g. `a.inline_script.ts`)."); - } - // Check if this skill has schemas that need to be appended - const schemaMappings = SCHEMA_MAPPINGS[skill.name]; - if (schemaMappings && schemaMappings.length > 0) { - // Combine base content with schemas - const schemaDocs = schemaMappings - .map((mapping) => { - const schemaYaml = SCHEMAS[mapping.schemaKey]; - if (schemaYaml) { - return formatSchemaForMarkdown(schemaYaml, mapping.name, mapping.filePattern); - } - return null; - }) - .filter((doc): doc is string => doc !== null); - - if (schemaDocs.length > 0) { - skillContent = skillContent + "\n\n" + schemaDocs.join("\n\n"); - } - } - - await writeFile(`${skillDir}/SKILL.md`, skillContent, "utf-8"); - } - }) - ); - - log.info(colors.green(`Created .claude/skills/ with ${SKILLS.length} skills`)); - } catch (skillError) { - if (skillError instanceof Error) { - log.warn(`Could not create skills: ${skillError.message}`); - } else { - log.warn(`Could not create skills: ${skillError}`); - } - } + log.info( + colors.green(`Created .claude/skills/ with ${guidanceResult.skillCount} skills`) + ); } catch (error) { if (error instanceof Error) { log.warn(`Could not create guidance files: ${error.message}`); @@ -341,6 +266,18 @@ const command = new Command() .description("Bootstrap a windmill project with a wmill.yaml file") .option("--use-default", "Use default settings without checking backend") .option("--use-backend", "Use backend git-sync settings if available") + .option( + "--ai-skills-source ", + "Use a custom skills directory instead of the generated .claude/skills bundle" + ) + .option( + "--ai-agents-source ", + "Use a custom AGENTS.md file for AI guidance" + ) + .option( + "--ai-claude-source ", + "Use a custom CLAUDE.md file for AI guidance" + ) .option( "--repository ", "Specify repository path (e.g., u/user/repo) when using backend settings" diff --git a/cli/src/guidance/writer.ts b/cli/src/guidance/writer.ts new file mode 100644 index 0000000000..dd75e83013 --- /dev/null +++ b/cli/src/guidance/writer.ts @@ -0,0 +1,229 @@ +import { cp, mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { generateAgentsMdContent } from "./core.ts"; +import { + SCHEMAS, + SCHEMA_MAPPINGS, + SKILLS, + SKILL_CONTENT, + type SkillMetadata, +} from "./skills.ts"; + +export interface WriteAiGuidanceOptions { + targetDir: string; + nonDottedPaths?: boolean; + overwriteProjectGuidance?: boolean; + skillsSourcePath?: string; + agentsSourcePath?: string; + claudeSourcePath?: string; +} + +export interface WriteAiGuidanceResult { + agentsWritten: boolean; + claudeWritten: boolean; + skillsWritten: boolean; + skillCount: number; +} + +const CLAUDE_MD_DEFAULT = "Instructions are in @AGENTS.md\n"; + +export async function writeAiGuidanceFiles( + options: WriteAiGuidanceOptions +): Promise { + const nonDottedPaths = options.nonDottedPaths ?? true; + const skillMetadata = options.skillsSourcePath + ? await copySkillsFromSource(options.targetDir, options.skillsSourcePath) + : await writeGeneratedSkills(options.targetDir, nonDottedPaths); + + const agentsWritten = await writeProjectGuidanceFile({ + targetPath: join(options.targetDir, "AGENTS.md"), + overwrite: options.overwriteProjectGuidance ?? false, + content: + options.agentsSourcePath != null + ? await readFile(options.agentsSourcePath, "utf8") + : generateAgentsMdContent(buildSkillsReference(skillMetadata)), + }); + + const claudeWritten = await writeProjectGuidanceFile({ + targetPath: join(options.targetDir, "CLAUDE.md"), + overwrite: options.overwriteProjectGuidance ?? false, + content: + options.claudeSourcePath != null + ? await readFile(options.claudeSourcePath, "utf8") + : CLAUDE_MD_DEFAULT, + }); + + return { + agentsWritten, + claudeWritten, + skillsWritten: true, + skillCount: skillMetadata.length, + }; +} + +function buildSkillsReference(skills: Pick[]): string { + return skills + .map((skill) => `- \`.claude/skills/${skill.name}/SKILL.md\` - ${skill.description}`) + .join("\n"); +} + +async function copySkillsFromSource( + targetDir: string, + skillsSourcePath: string +): Promise { + const skillsDir = join(targetDir, ".claude", "skills"); + await mkdir(join(targetDir, ".claude"), { recursive: true }); + await rm(skillsDir, { recursive: true, force: true }); + await cp(skillsSourcePath, skillsDir, { recursive: true, force: true }); + return await readSkillMetadataFromDirectory(skillsDir); +} + +async function writeGeneratedSkills( + targetDir: string, + nonDottedPaths: boolean +): Promise { + const skillsDir = join(targetDir, ".claude", "skills"); + await rm(skillsDir, { recursive: true, force: true }); + await mkdir(skillsDir, { recursive: true }); + + await Promise.all( + SKILLS.map(async (skill) => { + const skillDir = join(skillsDir, skill.name); + await mkdir(skillDir, { recursive: true }); + await writeFile( + join(skillDir, "SKILL.md"), + renderGeneratedSkillContent(skill.name, nonDottedPaths), + "utf8" + ); + }) + ); + + return SKILLS; +} + +function renderGeneratedSkillContent(skillName: string, nonDottedPaths: boolean): string { + let skillContent = SKILL_CONTENT[skillName]; + if (!skillContent) { + throw new Error(`Missing generated skill content for ${skillName}`); + } + + if (nonDottedPaths) { + skillContent = skillContent + .replaceAll("{{FLOW_SUFFIX}}", "__flow") + .replaceAll("{{APP_SUFFIX}}", "__app") + .replaceAll("{{RAW_APP_SUFFIX}}", "__raw_app") + .replaceAll( + "{{INLINE_SCRIPT_NAMING}}", + "Inline script files should NOT include `.inline_script.` in their names (e.g. use `a.ts`, not `a.inline_script.ts`)." + ); + } else { + skillContent = skillContent + .replaceAll("{{FLOW_SUFFIX}}", ".flow") + .replaceAll("{{APP_SUFFIX}}", ".app") + .replaceAll("{{RAW_APP_SUFFIX}}", ".raw_app") + .replaceAll( + "{{INLINE_SCRIPT_NAMING}}", + "Inline script files use the `.inline_script.` naming convention (e.g. `a.inline_script.ts`)." + ); + } + + const schemaMappings = SCHEMA_MAPPINGS[skillName]; + if (!schemaMappings || schemaMappings.length === 0) { + return skillContent; + } + + const schemaDocs = schemaMappings + .map((mapping) => { + const schemaYaml = SCHEMAS[mapping.schemaKey]; + if (!schemaYaml) { + return null; + } + return formatSchemaForMarkdown(schemaYaml, mapping.name, mapping.filePattern); + }) + .filter((entry): entry is string => entry !== null); + + if (schemaDocs.length === 0) { + return skillContent; + } + + return `${skillContent}\n\n${schemaDocs.join("\n\n")}`; +} + +async function readSkillMetadataFromDirectory(skillsDir: string): Promise { + const entries = await readdir(skillsDir, { withFileTypes: true }); + const skills: SkillMetadata[] = []; + + for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { + if (!entry.isDirectory()) { + continue; + } + + const skillPath = join(skillsDir, entry.name, "SKILL.md"); + if (!(await stat(skillPath).catch(() => null))) { + continue; + } + + const content = await readFile(skillPath, "utf8"); + skills.push(parseSkillMetadata(content, entry.name)); + } + + return skills; +} + +function parseSkillMetadata(content: string, fallbackName: string): SkillMetadata { + const frontMatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/); + if (!frontMatterMatch) { + return { + name: fallbackName, + description: `Skill loaded from ${fallbackName}`, + }; + } + + let name = fallbackName; + let description = `Skill loaded from ${fallbackName}`; + + for (const line of frontMatterMatch[1].split("\n")) { + const separatorIndex = line.indexOf(":"); + if (separatorIndex === -1) { + continue; + } + + const key = line.slice(0, separatorIndex).trim(); + const value = line.slice(separatorIndex + 1).trim(); + + if (key === "name" && value) { + name = value; + } else if (key === "description" && value) { + description = value; + } + } + + return { name, description }; +} + +async function writeProjectGuidanceFile(options: { + targetPath: string; + content: string; + overwrite: boolean; +}): Promise { + if (!options.overwrite && (await stat(options.targetPath).catch(() => null))) { + return false; + } + + await writeFile(options.targetPath, options.content, "utf8"); + return true; +} + +function formatSchemaForMarkdown( + schemaYaml: string, + schemaName: string, + filePattern: string +): string { + return `## ${schemaName} (\`${filePattern}\`) + +Must be a YAML file that adheres to the following schema: + +\`\`\`yaml +${schemaYaml.trim()} +\`\`\``; +}