test: auto-populate cli skill fixtures

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

cli/test-skills/README.md

@@ -4,33 +4,20 @@ Test suite for verifying that Claude Code correctly invokes Windmill auto-genera
 
 ## Overview
 
-This framework tests skill invocation behavior by sending prompts through the Claude Agent SDK and verifying that the expected skills are invoked. Users must provide their own `.claude/skills` folder containing auto-generated Windmill skills.
+This framework tests skill invocation behavior by sending prompts through the Claude Agent SDK and verifying that the expected skills are invoked. The suite mirrors the repo's generated Windmill skills into its test workspace before each run.
 
 ## Prerequisites
 
 - [Bun](https://bun.sh/) installed
 - `ANTHROPIC_API_KEY` environment variable set
-- Auto-generated Windmill skills placed in `.claude/skills/`
+- Auto-generated Windmill skills available under `system_prompts/auto-generated/skills/`
 
 ## User Setup
 
-1. Create a `test-folder` directory inside `cli/test-skills/` and copy your auto-generated Windmill skills into it:
+1. Generate the latest system prompts and CLI skills in the repo:
 
-```
-cli/test-skills/
-└── test-folder/
-    └── .claude/
-        └── skills/
-            ├── write-flow/
-            │   └── SKILL.md
-            ├── write-script-python3/
-            │   └── SKILL.md
-            ├── write-script-bun/
-            │   └── SKILL.md
-            ├── schedules/
-            │   └── SKILL.md
-            └── triggers/
-                └── SKILL.md
+```bash
+python3 system_prompts/generate.py
 ```
 
 2. Set your API key:
@@ -99,5 +86,5 @@ The `src/test-utils.ts` module provides:
 - Tests have extended timeouts (120 seconds) due to API latency
 - Tests run against the actual Claude API, so they consume API credits
 - Tests verify skill invocation, not skill execution
-- The working directory for tests is `test-folder/` (where `.claude/skills` should be placed)
-- Tests will fail with a clear error if `test-folder/` or `test-folder/.claude/skills/` don't exist
+- The working directory for tests is `test-folder/`
+- The suite refreshes `test-folder/.claude/skills/` from `system_prompts/auto-generated/skills/` before running

cli/test-skills/src/test-utils.ts

@@ -1,5 +1,5 @@
 import { query, type Options } from "@anthropic-ai/claude-agent-sdk";
-import { existsSync } from "fs";
+import { cpSync, existsSync, mkdirSync, rmSync } from "fs";
 import { join } from "path";
 
 export interface ToolInvocation {
@@ -29,26 +29,30 @@ export function getTestFolder(): string {
 }
 
 /**
- * Validate that test-folder exists and has .claude/skills
- * Throws an error if validation fails
+ * Get the generated skills directory from the repo root.
+ */
+export function getGeneratedSkillsSource(): string {
+  return join(getTestSkillsDir(), "..", "..", "system_prompts", "auto-generated", "skills");
+}
+
+/**
+ * Ensure test-folder exists and mirrors the repo's generated skills.
  */
 export function validateTestFolder(): void {
   const testFolder = getTestFolder();
   const skillsFolder = join(testFolder, ".claude", "skills");
+  const generatedSkillsSource = getGeneratedSkillsSource();
 
-  if (!existsSync(testFolder)) {
+  if (!existsSync(generatedSkillsSource)) {
     throw new Error(
-      `test-folder does not exist at: ${testFolder}\n` +
-      `Please create it and add your .claude/skills directory inside.`
+      `Generated skills directory not found at: ${generatedSkillsSource}\n` +
+      `Run system prompt generation first so cli/test-skills can mirror the current repo skill bundle.`
     );
   }
 
-  if (!existsSync(skillsFolder)) {
-    throw new Error(
-      `.claude/skills directory not found in test-folder at: ${skillsFolder}\n` +
-      `Please add your auto-generated Windmill skills to test-folder/.claude/skills/`
-    );
-  }
+  mkdirSync(join(testFolder, ".claude"), { recursive: true });
+  rmSync(skillsFolder, { recursive: true, force: true });
+  cpSync(generatedSkillsSource, skillsFolder, { recursive: true });
 }
 
 /**