tests: add cli eval behavior checks (#8899)

* feat: add cli eval behavior checks

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: harden cli eval command parsing

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

ai_evals/README.md

@@ -198,7 +198,7 @@ Typical artifacts by mode:
 - `flow`: `flow.json`
 - `script`: `script.json` plus the generated script file
 - `app`: `app.json` plus frontend/backend files
-- `cli`: `assistant-output.txt` plus generated workspace files
+- `cli`: `assistant-output.txt`, `trace.json`, `wmill-invocations.jsonl`, plus generated workspace files
 - backend-validated attempts also include `backend-preview.json`
 
 ## Layout
@@ -214,5 +214,6 @@ Typical artifacts by mode:
 
 - Frontend modes reuse the production frontend chat code through the Vitest bridge.
 - CLI mode creates an isolated workspace, writes the current checkout guidance into it, and benchmarks the real skills / `AGENTS.md` flow.
+- CLI mode now also records a structured trace of invoked skills, tool calls, proposed `wmill` commands, and any attempted `wmill` executions.
 - Frontend progress streams live while the benchmark is running.
 - Deterministic validators should stay focused on real correctness constraints, not one exact implementation shape.

ai_evals/adapters/cli/runtime.test.ts

@@ -2,6 +2,8 @@ import { describe, expect, it } from "bun:test";
 import {
   anthropicUsageToBenchmarkTokenUsage,
   extractCliResultTokenUsage,
+  extractProposedWmillCommands,
+  parseWmillInvocationLog,
 } from "./runtime";
 
 describe("anthropicUsageToBenchmarkTokenUsage", () => {
@@ -70,3 +72,78 @@ describe("extractCliResultTokenUsage", () => {
     });
   });
 });
+
+describe("extractProposedWmillCommands", () => {
+  it("extracts proposed commands from bullets, code blocks, and inline code", () => {
+    expect(
+      extractProposedWmillCommands(`
+Next:
+- \`wmill generate-metadata --yes\`
+- wmill sync push
+
+You can inspect failures with \`wmill job logs 123\`.
+`)
+    ).toEqual([
+      "wmill generate-metadata --yes",
+      "wmill sync push",
+      "wmill job logs 123",
+    ]);
+  });
+
+  it("extracts inline prose commands that are not wrapped in backticks", () => {
+    expect(
+      extractProposedWmillCommands(
+        "The first command is wmill sync pull before you edit locally."
+      )
+    ).toEqual(["wmill sync pull"]);
+  });
+
+  it("extracts multiple inline prose commands from a single sentence", () => {
+    expect(
+      extractProposedWmillCommands(
+        "Run wmill generate-metadata and then wmill sync push when you are ready."
+      )
+    ).toEqual(["wmill generate-metadata", "wmill sync push"]);
+  });
+
+  it("ignores negated command mentions", () => {
+    expect(
+      extractProposedWmillCommands(
+        "Do not run `wmill sync push`. Instead run `wmill sync pull` first."
+      )
+    ).toEqual(["wmill sync pull"]);
+  });
+});
+
+describe("parseWmillInvocationLog", () => {
+  it("parses stubbed wmill invocations into structured records", () => {
+    expect(
+      parseWmillInvocationLog(`noise
+__WMILL_BENCHMARK__
+2026-04-21T12:00:00+00:00
+/tmp/workspace
+2
+generate-metadata
+--yes
+__WMILL_BENCHMARK__
+2026-04-21T12:00:05+00:00
+/tmp/workspace
+3
+sync
+push
+--dry-run
+`)
+    ).toEqual([
+      {
+        argv: ["generate-metadata", "--yes"],
+        cwd: "/tmp/workspace",
+        timestamp: "2026-04-21T12:00:00+00:00",
+      },
+      {
+        argv: ["sync", "push", "--dry-run"],
+        cwd: "/tmp/workspace",
+        timestamp: "2026-04-21T12:00:05+00:00",
+      },
+    ]);
+  });
+});

ai_evals/adapters/cli/runtime.ts

@@ -1,22 +1,22 @@
 import { query, type Options } from "@anthropic-ai/claude-agent-sdk";
-import { join } from "path";
+import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
+import { delimiter, join } from "path";
 import { fileURLToPath } from "url";
 import { getCliEvalModel, resolveEvalModel, type CliEvalModelConfig } from "../../core/models";
-import type { BenchmarkTokenUsage } from "../../core/types";
+import type {
+  BenchmarkTokenUsage,
+  CliToolInvocation,
+  CliTrace,
+  CliWmillInvocation,
+} from "../../core/types";
 
-export interface ToolInvocation {
-  tool: string;
-  input: Record<string, unknown>;
-  timestamp: number;
-}
+export type ToolInvocation = CliToolInvocation;
 
 export interface PromptRunResult {
-  toolsUsed: ToolInvocation[];
-  skillsInvoked: string[];
   output: string;
   durationMs: number;
-  assistantMessageCount: number;
   tokenUsage: BenchmarkTokenUsage | null;
+  trace: CliTrace;
 }
 
 interface AnthropicUsageLike {
@@ -41,6 +41,25 @@ interface CliResultMessageLike {
 
 const REPO_ROOT = fileURLToPath(new URL("../../../", import.meta.url));
 export const DEFAULT_CLI_EVAL_MODEL: CliEvalModelConfig = getCliEvalModel(resolveEvalModel("cli"));
+const WMILL_STUB_DIR_NAME = ".wmill-benchmark-bin";
+const WMILL_LOG_FILE_NAME = ".wmill-benchmark-wmill-invocations.log";
+const WMILL_LOG_MARKER = "__WMILL_BENCHMARK__";
+const NEGATED_COMMAND_PREFIX = /(?:^|\b)(?:do not|don't|dont|never|instead of)\s+(?:run|use)?\s*$/i;
+const COMMAND_STOP_WORDS = new Set([
+  "and",
+  "before",
+  "after",
+  "then",
+  "instead",
+  "otherwise",
+  "because",
+  "so",
+  "if",
+  "when",
+  "while",
+  "once",
+]);
+const COMMAND_STOP_TOKENS = new Set(["-", "–", "—", "|"]);
 
 export function getGeneratedSkillsSource(): string {
   return join(REPO_ROOT, "system_prompts", "auto-generated", "skills");
@@ -121,19 +140,29 @@ export async function runPromptAndCapture(
 ): Promise<PromptRunResult> {
   const toolsUsed: ToolInvocation[] = [];
   const skillsInvoked: string[] = [];
+  const bashCommands: string[] = [];
   let output = "";
   let assistantMessageCount = 0;
   let tokenUsage: BenchmarkTokenUsage | null = null;
   const startedAt = Date.now();
+  const stubBinDir = join(cwd, WMILL_STUB_DIR_NAME);
+  const wmillLogPath = join(cwd, WMILL_LOG_FILE_NAME);
 
   const options: Options = {
     cwd,
     model: modelConfig.model,
     maxTurns,
     settingSources: ["project"],
-    allowedTools: ["Skill", "Read", "Glob", "Grep", "Bash", "Write", "Edit"]
+    allowedTools: ["Skill", "Read", "Glob", "Grep", "Bash", "Write", "Edit"],
+    env: {
+      ...getQueryEnv(),
+      PATH: process.env.PATH ? `${stubBinDir}${delimiter}${process.env.PATH}` : stubBinDir,
+      WMILL_BENCHMARK_LOG_PATH: wmillLogPath,
+    },
   };
 
+  await installWmillStub(stubBinDir);
+
   for await (const message of query({ prompt, options })) {
     if (message.type === "assistant") {
       assistantMessageCount += 1;
@@ -141,16 +170,23 @@ export async function runPromptAndCapture(
       if (Array.isArray(content)) {
         for (const block of content) {
           if (block.type === "tool_use") {
+            const input = normalizeToolInput(block.input);
             toolsUsed.push({
               tool: block.name,
-              input: block.input as Record<string, unknown>,
+              input,
               timestamp: Date.now()
             });
 
-            if (block.name === "Skill" && typeof block.input === "object" && block.input !== null) {
-              const skillInput = block.input as { skill?: string };
+            if (block.name === "Skill") {
+              const skillInput = input as { skill?: string };
               if (skillInput.skill) {
-                skillsInvoked.push(skillInput.skill);
+                pushUnique(skillsInvoked, skillInput.skill);
+              }
+            }
+
+            if (block.name === "Bash") {
+              for (const command of extractBashCommands(input)) {
+                pushUnique(bashCommands, command);
               }
             }
           } else if (block.type === "text") {
@@ -167,22 +203,32 @@ export async function runPromptAndCapture(
     }
   }
 
+  const proposedCommands = extractProposedWmillCommands(output);
+  const wmillInvocations = await readWmillInvocationLog(wmillLogPath);
+
   return {
-    toolsUsed,
-    skillsInvoked,
     output,
     durationMs: Date.now() - startedAt,
-    assistantMessageCount,
     tokenUsage,
+    trace: {
+      toolsUsed,
+      skillsInvoked,
+      assistantMessageCount,
+      bashCommands,
+      proposedCommands,
+      executedWmillCommands: wmillInvocations.map(formatExecutedWmillCommand),
+      wmillInvocations,
+      firstMutationToolIndex: getFirstMutationToolIndex(toolsUsed),
+    },
   };
 }
 
 export function wasSkillInvoked(result: PromptRunResult, skillName: string): boolean {
-  return result.skillsInvoked.some((skill) => skill === skillName || skill.includes(skillName));
+  return result.trace.skillsInvoked.some((skill) => skill === skillName);
 }
 
 export function wasToolUsed(result: PromptRunResult, toolName: string): boolean {
-  return result.toolsUsed.some((tool) => tool.tool === toolName);
+  return result.trace.toolsUsed.some((tool) => tool.tool === toolName);
 }
 
 export function formatCliRunModelLabel(modelConfig: CliEvalModelConfig): string {
@@ -193,7 +239,294 @@ export function getToolInputs(
   result: PromptRunResult,
   toolName: string
 ): Record<string, unknown>[] {
-  return result.toolsUsed
+  return result.trace.toolsUsed
     .filter((tool) => tool.tool === toolName)
     .map((tool) => tool.input);
 }
+
+export function extractProposedWmillCommands(output: string): string[] {
+  const commands: string[] = [];
+
+  for (const line of output.split(/\r?\n/)) {
+    for (const command of extractInlineBacktickCommands(line)) {
+      pushUnique(commands, command);
+    }
+
+    for (const command of extractInlineProseCommands(line.replace(/^\s*(?:[-*]|\d+\.)\s*/, ""))) {
+      pushUnique(commands, command);
+    }
+  }
+
+  return commands;
+}
+
+export function parseWmillInvocationLog(raw: string): CliWmillInvocation[] {
+  const entries: CliWmillInvocation[] = [];
+  const lines = raw.split(/\r?\n/);
+
+  for (let index = 0; index < lines.length; index += 1) {
+    if (lines[index] !== WMILL_LOG_MARKER) {
+      continue;
+    }
+
+    const timestamp = lines[index + 1] ?? "";
+    const cwd = lines[index + 2] ?? "";
+    const argCount = Number.parseInt(lines[index + 3] ?? "", 10);
+    if (!Number.isFinite(argCount) || argCount < 0) {
+      continue;
+    }
+
+    const start = index + 4;
+    const argv = lines.slice(start, start + argCount);
+    entries.push({ argv, cwd, timestamp });
+    index = start + argCount - 1;
+  }
+
+  return entries;
+}
+
+async function installWmillStub(binDir: string): Promise<void> {
+  await mkdir(binDir, { recursive: true });
+
+  const stubPath = join(binDir, "wmill");
+  const script = `#!/usr/bin/env bash
+set -euo pipefail
+{
+  printf '${WMILL_LOG_MARKER}\\n'
+  date -u +"%Y-%m-%dT%H:%M:%SZ"
+  printf '%s\\n' "$PWD"
+  printf '%s\\n' "$#"
+  printf '%s\\n' "$@"
+} >> "\${WMILL_BENCHMARK_LOG_PATH:?}"
+printf 'wmill benchmark stub: do not execute Windmill CLI commands during ai_evals; describe them in the final response instead.\\n' >&2
+exit 97
+`;
+
+  await writeFile(stubPath, script, "utf8");
+  await chmod(stubPath, 0o755);
+}
+
+async function readWmillInvocationLog(logPath: string): Promise<CliWmillInvocation[]> {
+  const raw = await readFile(logPath, "utf8").catch(() => null);
+  if (!raw) {
+    return [];
+  }
+  return parseWmillInvocationLog(raw);
+}
+
+function getQueryEnv(): Record<string, string> {
+  return Object.fromEntries(
+    Object.entries(process.env).flatMap(([key, value]) =>
+      typeof value === "string" ? [[key, value]] : []
+    )
+  );
+}
+
+function normalizeToolInput(input: unknown): Record<string, unknown> {
+  if (input && typeof input === "object" && !Array.isArray(input)) {
+    return input as Record<string, unknown>;
+  }
+
+  if (typeof input === "string") {
+    return { raw: input };
+  }
+
+  return {};
+}
+
+function extractBashCommands(input: Record<string, unknown>): string[] {
+  const commands: string[] = [];
+
+  for (const key of ["command", "cmd", "script", "raw"]) {
+    const value = input[key];
+    if (typeof value === "string") {
+      for (const line of value.split(/\r?\n/)) {
+        const command = normalizeCommandCandidate(line);
+        if (command) {
+          pushUnique(commands, command);
+        }
+      }
+    }
+  }
+
+  return commands;
+}
+
+function extractInlineBacktickCommands(line: string): string[] {
+  const commands: string[] = [];
+  const regex = /`(wmill [^`\n]+)`/g;
+  let match: RegExpExecArray | null = null;
+
+  while ((match = regex.exec(line)) !== null) {
+    if (hasNegatedCommandPrefix(line.slice(0, match.index))) {
+      continue;
+    }
+
+    const command = normalizeCommandCandidate(match[1]);
+    if (command) {
+      pushUnique(commands, command);
+    }
+  }
+
+  return commands;
+}
+
+function extractInlineProseCommands(line: string): string[] {
+  const commands: string[] = [];
+  let searchFrom = 0;
+
+  while (true) {
+    const inlineIndex = line.toLowerCase().indexOf("wmill ", searchFrom);
+    if (inlineIndex === -1) {
+      return commands;
+    }
+
+    if (!hasNegatedCommandPrefix(line.slice(0, inlineIndex))) {
+      const command = extractInlineProseCommandAt(line, inlineIndex);
+      if (command) {
+        pushUnique(commands, command);
+      }
+    }
+
+    searchFrom = inlineIndex + "wmill ".length;
+  }
+}
+
+function extractInlineProseCommandAt(line: string, startIndex: number): string | null {
+  const tokens = ["wmill"];
+  let cursor = startIndex + "wmill".length;
+
+  while (cursor < line.length) {
+    while (cursor < line.length && /\s/.test(line[cursor]!)) {
+      cursor += 1;
+    }
+
+    if (cursor >= line.length) {
+      break;
+    }
+
+    const current = line[cursor]!;
+    if ("`.,;:()[]{}".includes(current)) {
+      break;
+    }
+
+    const token = readCommandToken(line, cursor);
+    if (!token) {
+      break;
+    }
+
+    if (COMMAND_STOP_WORDS.has(token.value.toLowerCase())) {
+      break;
+    }
+
+    if (COMMAND_STOP_TOKENS.has(token.value)) {
+      break;
+    }
+
+    tokens.push(token.value);
+    cursor = token.nextIndex;
+  }
+
+  if (tokens.length <= 1) {
+    return null;
+  }
+
+  return normalizeCommandCandidate(tokens.join(" "));
+}
+
+function readCommandToken(
+  line: string,
+  startIndex: number
+): { value: string; nextIndex: number } | null {
+  const firstChar = line[startIndex]!;
+
+  if (firstChar === `"` || firstChar === `'`) {
+    const endIndex = line.indexOf(firstChar, startIndex + 1);
+    const nextIndex = endIndex === -1 ? line.length : endIndex + 1;
+    return {
+      value: line.slice(startIndex, nextIndex),
+      nextIndex,
+    };
+  }
+
+  if (firstChar === "<") {
+    const endIndex = line.indexOf(">", startIndex + 1);
+    const nextIndex = endIndex === -1 ? line.length : endIndex + 1;
+    return {
+      value: line.slice(startIndex, nextIndex),
+      nextIndex,
+    };
+  }
+
+  let endIndex = startIndex;
+  while (endIndex < line.length && !/[\s`.,;:()[\]{}#]/.test(line[endIndex]!)) {
+    endIndex += 1;
+  }
+
+  if (endIndex === startIndex) {
+    return null;
+  }
+
+  return {
+    value: line.slice(startIndex, endIndex),
+    nextIndex: endIndex,
+  };
+}
+
+function hasNegatedCommandPrefix(prefix: string): boolean {
+  const normalizedPrefix = prefix
+    .toLowerCase()
+    .replace(/[`"'“”‘’]/g, " ")
+    .replace(/\s+/g, " ")
+    .trimEnd();
+
+  return NEGATED_COMMAND_PREFIX.test(normalizedPrefix);
+}
+
+function normalizeCommandCandidate(value: string): string | null {
+  const trimmed = value.trim().replace(/^`|`$/g, "");
+  if (!trimmed) {
+    return null;
+  }
+
+  const normalized = trimmed
+    .replace(/\s+/g, " ")
+    .replace(/[`.;:,]+$/g, "")
+    .trim();
+
+  return normalized.length > 0 ? normalized : null;
+}
+
+function formatExecutedWmillCommand(entry: CliWmillInvocation): string {
+  return ["wmill", ...entry.argv].join(" ").trim();
+}
+
+function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null {
+  for (const [index, tool] of toolsUsed.entries()) {
+    if (tool.tool === "Write" || tool.tool === "Edit") {
+      return index;
+    }
+
+    if (tool.tool === "Bash" && extractBashCommands(tool.input).some(isLikelyMutatingBashCommand)) {
+      return index;
+    }
+  }
+
+  return null;
+}
+
+function isLikelyMutatingBashCommand(command: string): boolean {
+  return (
+    /\b(?:mkdir|touch|rm|mv|cp|install|tee)\b/.test(command) ||
+    /\b(?:cat|echo|printf)\b.*(?:>|>>|\|\s*tee\b)/.test(command) ||
+    /\bsed\s+-i\b/.test(command) ||
+    /\bperl\s+-pi\b/.test(command) ||
+    /\bwmill\b/.test(command)
+  );
+}
+
+function pushUnique(values: string[], value: string): void {
+  if (!values.includes(value)) {
+    values.push(value);
+  }
+}

ai_evals/cases/cli.yaml

@@ -3,6 +3,23 @@
     Create a Windmill Bun script at `f/evals/hello.ts`.
     It should take a `name` input and return a greeting object like `{ greeting: "Hello, Alice!" }`.
   expected: ai_evals/fixtures/cli/expected/bun-hello-script
+  cliExpect:
+    requiredSkills:
+      - write-script-bun
+    requiredSkillsBeforeFirstMutation:
+      - write-script-bun
+    forbiddenSkills:
+      - write-script-python3
+      - write-flow
+    orderedAssistantMentions:
+      - wmill generate-metadata
+      - wmill sync push
+    orderedProposedCommands:
+      - wmill generate-metadata
+      - wmill sync push
+    forbiddenExecutedCommands:
+      - ^wmill generate-metadata
+      - ^wmill sync push
   judgeChecklist:
     - creates the requested Bun script at f/evals/hello.ts
     - takes a name input
@@ -14,6 +31,22 @@
     It should take a `name` input and return a greeting object like `{ greeting: "Hello, Alice!" }`.
     Put the step code in `hello.ts`.
   expected: ai_evals/fixtures/cli/expected/bun-hello-flow
+  cliExpect:
+    requiredSkills:
+      - write-flow
+    requiredSkillsBeforeFirstMutation:
+      - write-flow
+    forbiddenSkills:
+      - write-script-python3
+    orderedAssistantMentions:
+      - wmill generate-metadata
+      - wmill sync push
+    orderedProposedCommands:
+      - wmill generate-metadata
+      - wmill sync push
+    forbiddenExecutedCommands:
+      - ^wmill generate-metadata
+      - ^wmill sync push
   judgeChecklist:
     - creates the requested flow folder with flow.yaml and hello.ts
     - wires the name input into the flow step
@@ -24,6 +57,23 @@
     Add a Windmill Python script at `f/evals/add_numbers.py`.
     It should take `a` and `b` as inputs and return `{ "total": a + b }`.
   expected: ai_evals/fixtures/cli/expected/python-add-numbers-script
+  cliExpect:
+    requiredSkills:
+      - write-script-python3
+    requiredSkillsBeforeFirstMutation:
+      - write-script-python3
+    forbiddenSkills:
+      - write-script-bun
+      - write-flow
+    orderedAssistantMentions:
+      - wmill generate-metadata
+      - wmill sync push
+    orderedProposedCommands:
+      - wmill generate-metadata
+      - wmill sync push
+    forbiddenExecutedCommands:
+      - ^wmill generate-metadata
+      - ^wmill sync push
   judgeChecklist:
     - creates the requested Python script at f/evals/add_numbers.py
     - takes `a` and `b` as inputs
@@ -59,8 +109,91 @@
     Create a flow at `f/evals/reuse_greeting__flow` that takes a `name` input and reuses that existing script instead of duplicating the logic inline.
   initial: ai_evals/fixtures/cli/initial/flow-reuse-existing-script
   expected: ai_evals/fixtures/cli/expected/flow-reuse-existing-script
+  cliExpect:
+    requiredSkills:
+      - write-flow
+    requiredSkillsBeforeFirstMutation:
+      - write-flow
+    orderedAssistantMentions:
+      - wmill generate-metadata
+      - wmill sync push
+    orderedProposedCommands:
+      - wmill generate-metadata
+      - wmill sync push
+    forbiddenExecutedCommands:
+      - ^wmill generate-metadata
+      - ^wmill sync push
   judgeChecklist:
     - creates the requested flow at f/evals/reuse_greeting__flow
     - reuses the existing script from f/lib by path
     - does not duplicate the greeting logic in a new inline script
     - wires the name input into the reused script
+
+- id: cli-job-debug-guidance
+  prompt: |-
+    A Windmill job failed.
+    Tell me exactly which `wmill` commands to run to inspect the job details, logs, and final result for job ID `123`.
+    Do not modify any files.
+  cliExpect:
+    requiredSkills:
+      - cli-commands
+    workspaceUnchanged: true
+    orderedProposedCommands:
+      - wmill job get 123
+      - wmill job logs 123
+      - wmill job result 123
+    forbiddenProposedCommands:
+      - wmill sync push
+    forbiddenExecutedCommands:
+      - ^wmill job get
+      - ^wmill job logs
+      - ^wmill job result
+  judgeChecklist:
+    - does not modify the workspace
+    - recommends commands to inspect the job details
+    - recommends commands to inspect the job logs
+    - recommends commands to inspect the final result
+
+- id: cli-sync-pull-guidance
+  prompt: |-
+    I want to review remote workspace changes before editing locally.
+    Tell me the first `wmill` command I should run.
+    Do not modify any files.
+  cliExpect:
+    requiredSkills:
+      - cli-commands
+    workspaceUnchanged: true
+    requiredProposedCommands:
+      - wmill sync pull
+    forbiddenProposedCommands:
+      - wmill sync push
+    forbiddenExecutedCommands:
+      - ^wmill sync pull
+      - ^wmill sync push
+  judgeChecklist:
+    - does not modify the workspace
+    - recommends using sync pull before making local edits
+    - does not recommend pushing first
+
+- id: cli-script-deploy-guidance
+  prompt: |-
+    I already modified a Windmill script locally and now want the next CLI commands to prepare it and deploy it.
+    Tell me the commands to run, in order.
+    Do not modify any files.
+  cliExpect:
+    requiredSkills:
+      - cli-commands
+    workspaceUnchanged: true
+    orderedAssistantMentions:
+      - wmill generate-metadata
+      - wmill sync push
+    orderedProposedCommands:
+      - wmill generate-metadata
+      - wmill sync push
+    forbiddenExecutedCommands:
+      - ^wmill generate-metadata
+      - ^wmill sync push
+  judgeChecklist:
+    - does not modify the workspace
+    - recommends generate-metadata before sync push
+    - presents the commands in order

ai_evals/core/cases.test.ts

@@ -101,4 +101,18 @@ describe("loadCases", () => {
       requiredBackendRunnableTypes: [{ key: "a", type: "inline" }],
     });
   });
+
+  it("loads CLI behavior expectations for deploy-guidance cases", async () => {
+    const cliCases = await loadCases("cli");
+    const caseEntry = cliCases.find((entry) => entry.id === "bun-hello-script");
+
+    expect(caseEntry?.cliExpect).toEqual({
+      requiredSkills: ["write-script-bun"],
+      requiredSkillsBeforeFirstMutation: ["write-script-bun"],
+      forbiddenSkills: ["write-script-python3", "write-flow"],
+      orderedAssistantMentions: ["wmill generate-metadata", "wmill sync push"],
+      orderedProposedCommands: ["wmill generate-metadata", "wmill sync push"],
+      forbiddenExecutedCommands: ["^wmill generate-metadata", "^wmill sync push"],
+    });
+  });
 });

ai_evals/core/cases.ts

@@ -2,7 +2,13 @@ import { readFile } from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { parse } from "yaml";
-import type { EvalCase, EvalCaseRuntimeSpec, EvalMode, EvalValidationSpec } from "./types";
+import type {
+  CliValidationSpec,
+  EvalCase,
+  EvalCaseRuntimeSpec,
+  EvalMode,
+  EvalValidationSpec,
+} from "./types";
 
 const REPO_ROOT = fileURLToPath(new URL("../../", import.meta.url));
 const CASES_DIR = path.join(REPO_ROOT, "ai_evals", "cases");
@@ -13,6 +19,7 @@ interface RawEvalCase {
   initial?: string;
   expected?: string;
   validate?: EvalValidationSpec;
+  cliExpect?: CliValidationSpec;
   judgeChecklist?: string[];
   runtime?: EvalCaseRuntimeSpec;
 }
@@ -39,6 +46,7 @@ export async function loadCases(mode: EvalMode): Promise<EvalCase[]> {
     initialPath: resolveFixturePath(entry.initial),
     expectedPath: resolveFixturePath(entry.expected),
     validate: entry.validate,
+    cliExpect: entry.cliExpect,
     judgeChecklist: entry.judgeChecklist,
     runtime: entry.runtime,
   }));

ai_evals/core/types.ts

@@ -77,6 +77,20 @@ export interface AppValidationSpec {
   }>;
 }
 
+export interface CliValidationSpec {
+  requiredSkills?: string[];
+  forbiddenSkills?: string[];
+  requiredSkillsBeforeFirstMutation?: string[];
+  requiredAssistantMentions?: string[];
+  forbiddenAssistantMentions?: string[];
+  orderedAssistantMentions?: string[];
+  requiredProposedCommands?: string[];
+  forbiddenProposedCommands?: string[];
+  orderedProposedCommands?: string[];
+  forbiddenExecutedCommands?: string[];
+  workspaceUnchanged?: boolean;
+}
+
 export type EvalValidationSpec = FlowValidationSpec | AppValidationSpec;
 
 export interface EvalCase {
@@ -85,6 +99,7 @@ export interface EvalCase {
   initialPath?: string;
   expectedPath?: string;
   validate?: EvalValidationSpec;
+  cliExpect?: CliValidationSpec;
   judgeChecklist?: string[];
   runtime?: EvalCaseRuntimeSpec;
 }
@@ -118,6 +133,29 @@ export interface BenchmarkTokenUsage {
   total: number;
 }
 
+export interface CliToolInvocation {
+  tool: string;
+  input: Record<string, unknown>;
+  timestamp: number;
+}
+
+export interface CliWmillInvocation {
+  argv: string[];
+  cwd: string;
+  timestamp: string;
+}
+
+export interface CliTrace {
+  toolsUsed: CliToolInvocation[];
+  skillsInvoked: string[];
+  assistantMessageCount: number;
+  bashCommands: string[];
+  proposedCommands: string[];
+  executedWmillCommands: string[];
+  wmillInvocations: CliWmillInvocation[];
+  firstMutationToolIndex: number | null;
+}
+
 export interface ModeRunOutput<TActual> {
   success: boolean;
   actual: TActual;

ai_evals/core/validators.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "bun:test";
-import { validateAppState, validateScriptState } from "./validators";
+import { validateAppState, validateCliWorkspace, validateScriptState } from "./validators";
 
 describe("validateScriptState", () => {
   it("accepts semantically equivalent script implementations", () => {
@@ -153,3 +153,184 @@ describe("validateAppState", () => {
     });
   });
 });
+
+describe("validateCliWorkspace", () => {
+  it("accepts required CLI skills and proposed commands without execution", () => {
+    const checks = validateCliWorkspace({
+      actualFiles: {
+        "f/evals/hello.ts": "export async function main(name: string) { return { greeting: `Hello, ${name}!` } }\n",
+      },
+      expectedFiles: {
+        "f/evals/hello.ts": "export async function main(name: string)\nreturn { greeting: `Hello, ${name}!` }",
+      },
+      assistantOutput:
+        "Created the script. Next run `wmill generate-metadata --yes` and then `wmill sync push`.",
+      trace: {
+        toolsUsed: [
+          { tool: "Skill", input: { skill: "write-script-bun" }, timestamp: 1 },
+          { tool: "Write", input: { file_path: "f/evals/hello.ts" }, timestamp: 2 },
+        ],
+        skillsInvoked: ["write-script-bun"],
+        assistantMessageCount: 1,
+        bashCommands: [],
+        proposedCommands: ["wmill generate-metadata --yes", "wmill sync push"],
+        executedWmillCommands: [],
+        wmillInvocations: [],
+        firstMutationToolIndex: 1,
+      },
+      cliExpect: {
+        requiredSkills: ["write-script-bun"],
+        requiredSkillsBeforeFirstMutation: ["write-script-bun"],
+        orderedAssistantMentions: ["wmill generate-metadata", "wmill sync push"],
+        orderedProposedCommands: ["wmill generate-metadata", "wmill sync push"],
+        forbiddenExecutedCommands: ["^wmill generate-metadata", "^wmill sync push"],
+      },
+    });
+
+    expect(checks.every((check) => check.passed)).toBe(true);
+  });
+
+  it("fails when a forbidden wmill command is executed", () => {
+    const checks = validateCliWorkspace({
+      actualFiles: {},
+      assistantOutput: "Run `wmill sync push` when ready.",
+      trace: {
+        toolsUsed: [{ tool: "Bash", input: { command: "wmill sync push" }, timestamp: 1 }],
+        skillsInvoked: [],
+        assistantMessageCount: 1,
+        bashCommands: ["wmill sync push"],
+        proposedCommands: ["wmill sync push"],
+        executedWmillCommands: ["wmill sync push"],
+        wmillInvocations: [
+          {
+            argv: ["sync", "push"],
+            cwd: "/tmp/workspace",
+            timestamp: "2026-04-21T12:00:00+00:00",
+          },
+        ],
+        firstMutationToolIndex: 0,
+      },
+      cliExpect: {
+        forbiddenExecutedCommands: ["^wmill sync push"],
+      },
+    });
+
+    expect(checks).toContainEqual({
+      name: "does not execute ^wmill sync push",
+      passed: false,
+      details: "executed=wmill sync push",
+    });
+  });
+
+  it("supports read-only guidance cases that must keep the workspace unchanged", () => {
+    const checks = validateCliWorkspace({
+      actualFiles: {},
+      assistantOutput:
+        "Use `wmill job get 123`, then `wmill job logs 123`, then `wmill job result 123`.",
+      trace: {
+        toolsUsed: [{ tool: "Skill", input: { skill: "cli-commands" }, timestamp: 1 }],
+        skillsInvoked: ["cli-commands"],
+        assistantMessageCount: 1,
+        bashCommands: [],
+        proposedCommands: ["wmill job get 123", "wmill job logs 123", "wmill job result 123"],
+        executedWmillCommands: [],
+        wmillInvocations: [],
+        firstMutationToolIndex: null,
+      },
+      cliExpect: {
+        requiredSkills: ["cli-commands"],
+        workspaceUnchanged: true,
+        orderedProposedCommands: [
+          "wmill job get 123",
+          "wmill job logs 123",
+          "wmill job result 123",
+        ],
+        forbiddenProposedCommands: ["wmill sync push"],
+      },
+    });
+
+    expect(checks.every((check) => check.passed)).toBe(true);
+  });
+
+  it("matches skills by exact name instead of substring", () => {
+    const checks = validateCliWorkspace({
+      actualFiles: {},
+      assistantOutput: "No workspace changes needed.",
+      trace: {
+        toolsUsed: [{ tool: "Skill", input: { skill: "write-flow-helper" }, timestamp: 1 }],
+        skillsInvoked: ["write-flow-helper"],
+        assistantMessageCount: 1,
+        bashCommands: [],
+        proposedCommands: [],
+        executedWmillCommands: [],
+        wmillInvocations: [],
+        firstMutationToolIndex: null,
+      },
+      cliExpect: {
+        requiredSkills: ["write-flow"],
+        forbiddenSkills: ["write-flow"],
+      },
+    });
+
+    expect(checks).toContainEqual({
+      name: "invokes skill write-flow",
+      passed: false,
+      details: "skills=write-flow-helper",
+    });
+    expect(checks).toContainEqual({
+      name: "does not invoke skill write-flow",
+      passed: true,
+    });
+  });
+
+  it("accepts ordered proposed commands when they appear in one concatenated entry", () => {
+    const checks = validateCliWorkspace({
+      actualFiles: {},
+      assistantOutput: "Run wmill generate-metadata and then wmill sync push.",
+      trace: {
+        toolsUsed: [{ tool: "Skill", input: { skill: "cli-commands" }, timestamp: 1 }],
+        skillsInvoked: ["cli-commands"],
+        assistantMessageCount: 1,
+        bashCommands: [],
+        proposedCommands: ["wmill generate-metadata and then wmill sync push"],
+        executedWmillCommands: [],
+        wmillInvocations: [],
+        firstMutationToolIndex: null,
+      },
+      cliExpect: {
+        orderedProposedCommands: ["wmill generate-metadata", "wmill sync push"],
+      },
+    });
+
+    expect(checks).toContainEqual({
+      name: "assistant proposes expected commands in order",
+      passed: true,
+    });
+  });
+
+  it("fails skill-before-mutation checks cleanly when no mutation happened", () => {
+    const checks = validateCliWorkspace({
+      actualFiles: {},
+      assistantOutput: "Run `wmill sync pull` first.",
+      trace: {
+        toolsUsed: [{ tool: "Skill", input: { skill: "cli-commands" }, timestamp: 1 }],
+        skillsInvoked: ["cli-commands"],
+        assistantMessageCount: 1,
+        bashCommands: [],
+        proposedCommands: ["wmill sync pull"],
+        executedWmillCommands: [],
+        wmillInvocations: [],
+        firstMutationToolIndex: null,
+      },
+      cliExpect: {
+        requiredSkillsBeforeFirstMutation: ["cli-commands"],
+      },
+    });
+
+    expect(checks).toContainEqual({
+      name: "invokes skill cli-commands before first mutation",
+      passed: false,
+      details: "firstSkillIndex=0; firstMutationIndex=none",
+    });
+  });
+});

ai_evals/core/validators.ts

@@ -1,6 +1,12 @@
 import path from "node:path";
 import ts from "typescript";
-import type { AppValidationSpec, BenchmarkCheck, FlowValidationSpec } from "./types";
+import type {
+  AppValidationSpec,
+  BenchmarkCheck,
+  CliTrace,
+  CliValidationSpec,
+  FlowValidationSpec,
+} from "./types";
 
 export interface ScriptState {
   path: string;
@@ -249,6 +255,9 @@ export function validateCliWorkspace(input: {
   actualFiles: Record<string, string>;
   expectedFiles?: Record<string, string>;
   initialFiles?: Record<string, string>;
+  assistantOutput?: string;
+  trace?: CliTrace;
+  cliExpect?: CliValidationSpec;
 }): BenchmarkCheck[] {
   const checks: BenchmarkCheck[] = [];
 
@@ -277,10 +286,25 @@ export function validateCliWorkspace(input: {
     );
   }
 
-  if (input.initialFiles) {
+  if (input.cliExpect?.workspaceUnchanged) {
+    const baselineFiles = input.initialFiles ?? {};
+    checks.push(
+      check(
+        "workspace remains unchanged",
+        fileMapsEqual(input.actualFiles, baselineFiles),
+        summarizeWorkspaceDiff(input.actualFiles, baselineFiles)
+      )
+    );
+  } else if (input.initialFiles) {
     checks.push(check("workspace differs from initial", !fileMapsEqual(input.actualFiles, input.initialFiles)));
   }
 
+  if (input.cliExpect) {
+    checks.push(
+      ...validateCliExpectations(input.assistantOutput ?? "", input.trace, input.cliExpect)
+    );
+  }
+
   return checks;
 }
 
@@ -324,6 +348,197 @@ function summarizeProblems(problems: string[], limit = 5): string | undefined {
   return `${problems.slice(0, limit).join("; ")}; ...and ${problems.length - limit} more`;
 }
 
+function validateCliExpectations(
+  assistantOutput: string,
+  trace: CliTrace | undefined,
+  cliExpect: CliValidationSpec
+): BenchmarkCheck[] {
+  const checks: BenchmarkCheck[] = [];
+
+  if (!trace) {
+    checks.push(check("cli trace is available", false));
+    return checks;
+  }
+
+  for (const skillName of cliExpect.requiredSkills ?? []) {
+    checks.push(
+      check(
+        `invokes skill ${skillName}`,
+        cliSkillWasInvoked(trace, skillName),
+        `skills=${trace.skillsInvoked.join(", ") || "(none)"}`
+      )
+    );
+  }
+
+  for (const skillName of cliExpect.forbiddenSkills ?? []) {
+    checks.push(
+      check(
+        `does not invoke skill ${skillName}`,
+        !cliSkillWasInvoked(trace, skillName),
+        `skills=${trace.skillsInvoked.join(", ") || "(none)"}`
+      )
+    );
+  }
+
+  for (const skillName of cliExpect.requiredSkillsBeforeFirstMutation ?? []) {
+    const firstSkillIndex = getFirstSkillToolIndex(trace, skillName);
+    const firstMutationIndex = trace.firstMutationToolIndex;
+    checks.push(
+      check(
+        `invokes skill ${skillName} before first mutation`,
+        firstSkillIndex !== null &&
+          firstMutationIndex !== null &&
+          firstSkillIndex < firstMutationIndex,
+        `firstSkillIndex=${firstSkillIndex ?? "none"}; firstMutationIndex=${firstMutationIndex ?? "none"}`
+      )
+    );
+  }
+
+  for (const phrase of cliExpect.requiredAssistantMentions ?? []) {
+    checks.push(
+      check(
+        `assistant mentions ${phrase}`,
+        assistantMentions(assistantOutput, phrase)
+      )
+    );
+  }
+
+  for (const phrase of cliExpect.forbiddenAssistantMentions ?? []) {
+    checks.push(
+      check(
+        `assistant does not mention ${phrase}`,
+        !assistantMentions(assistantOutput, phrase)
+      )
+    );
+  }
+
+  if ((cliExpect.orderedAssistantMentions?.length ?? 0) > 0) {
+    checks.push(
+      check(
+        "assistant mentions expected items in order",
+        stringsAppearInOrder(assistantOutput, cliExpect.orderedAssistantMentions!),
+        `ordered=${cliExpect.orderedAssistantMentions!.join(" -> ")}`
+      )
+    );
+  }
+
+  for (const command of cliExpect.requiredProposedCommands ?? []) {
+    checks.push(
+      check(
+        `assistant proposes ${command}`,
+        commandListContains(trace.proposedCommands, command),
+        `proposed=${trace.proposedCommands.join("; ") || "(none)"}`
+      )
+    );
+  }
+
+  for (const command of cliExpect.forbiddenProposedCommands ?? []) {
+    checks.push(
+      check(
+        `assistant does not propose ${command}`,
+        !commandListContains(trace.proposedCommands, command),
+        `proposed=${trace.proposedCommands.join("; ") || "(none)"}`
+      )
+    );
+  }
+
+  if ((cliExpect.orderedProposedCommands?.length ?? 0) > 0) {
+    checks.push(
+      check(
+        "assistant proposes expected commands in order",
+        stringsAppearInOrder(trace.proposedCommands.join("\n"), cliExpect.orderedProposedCommands!),
+        `ordered=${cliExpect.orderedProposedCommands!.join(" -> ")}; proposed=${trace.proposedCommands.join("; ") || "(none)"}`
+      )
+    );
+  }
+
+  for (const pattern of cliExpect.forbiddenExecutedCommands ?? []) {
+    checks.push(
+      check(
+        `does not execute ${pattern}`,
+        !trace.executedWmillCommands.some((command) => matchesCommandPattern(command, pattern)),
+        `executed=${trace.executedWmillCommands.join("; ") || "(none)"}`
+      )
+    );
+  }
+
+  return checks;
+}
+
+function cliSkillWasInvoked(trace: CliTrace, skillName: string): boolean {
+  return trace.skillsInvoked.some((skill) => skill === skillName);
+}
+
+function getFirstSkillToolIndex(trace: CliTrace, skillName: string): number | null {
+  for (const [index, tool] of trace.toolsUsed.entries()) {
+    if (tool.tool !== "Skill") {
+      continue;
+    }
+
+    const inputSkill = typeof tool.input.skill === "string" ? tool.input.skill : null;
+    if (inputSkill === skillName) {
+      return index;
+    }
+  }
+
+  return null;
+}
+
+function assistantMentions(output: string, phrase: string): boolean {
+  return output.toLowerCase().includes(phrase.toLowerCase());
+}
+
+function stringsAppearInOrder(output: string, phrases: string[]): boolean {
+  const normalizedOutput = output.toLowerCase();
+  let startIndex = 0;
+
+  for (const phrase of phrases) {
+    const index = normalizedOutput.indexOf(phrase.toLowerCase(), startIndex);
+    if (index === -1) {
+      return false;
+    }
+    startIndex = index + phrase.length;
+  }
+
+  return true;
+}
+
+function commandListContains(commands: string[], command: string): boolean {
+  const normalizedNeedle = command.toLowerCase();
+  return commands.some((entry) => entry.toLowerCase().includes(normalizedNeedle));
+}
+
+function matchesCommandPattern(command: string, pattern: string): boolean {
+  try {
+    return new RegExp(pattern, "i").test(command);
+  } catch {
+    return command.toLowerCase().includes(pattern.toLowerCase());
+  }
+}
+
+function summarizeWorkspaceDiff(
+  actualFiles: Record<string, string>,
+  baselineFiles: Record<string, string>
+): string | undefined {
+  const changes: string[] = [];
+
+  for (const filePath of Object.keys(actualFiles)) {
+    if (!(filePath in baselineFiles)) {
+      changes.push(`added ${filePath}`);
+    } else if (actualFiles[filePath] !== baselineFiles[filePath]) {
+      changes.push(`changed ${filePath}`);
+    }
+  }
+
+  for (const filePath of Object.keys(baselineFiles)) {
+    if (!(filePath in actualFiles)) {
+      changes.push(`removed ${filePath}`);
+    }
+  }
+
+  return summarizeProblems(changes);
+}
+
 function hasSupportedEntrypoint(code: string): boolean {
   return (
     /export\s+(async\s+)?function\s+main\s*\(/.test(code) ||

ai_evals/modes/cli.ts

@@ -13,9 +13,16 @@ import {
 } from "../adapters/cli/runtime";
 import { copyDirectory, readDirectoryFiles } from "../core/files";
 import { validateCliWorkspace } from "../core/validators";
-import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
+import type { BenchmarkArtifactFile, CliTrace, ModeRunner } from "../core/types";
 
-const IGNORE_WORKSPACE_FILES = new Set([".claude", "AGENTS.md", "CLAUDE.md", "rt.d.ts"]);
+const IGNORE_WORKSPACE_FILES = new Set([
+  ".claude",
+  "AGENTS.md",
+  "CLAUDE.md",
+  "rt.d.ts",
+  ".wmill-benchmark-bin",
+  ".wmill-benchmark-wmill-invocations.log",
+]);
 
 interface CliWorkspaceFixture {
   sourceDir: string;
@@ -25,6 +32,7 @@ interface CliWorkspaceFixture {
 interface CliRunActual {
   assistantOutput: string;
   workspaceFiles: Record<string, string>;
+  trace: CliTrace;
 }
 
 const CLAUDE_PROJECT_PREAMBLE = [
@@ -62,7 +70,7 @@ export function createCliModeRunner(
           }
         : undefined;
     },
-    async run(prompt, initial, _context) {
+    async run(prompt, initial, context) {
       const workspaceDir = await mkdtemp(join(tmpdir(), "wmill-cli-benchmark-"));
 
       try {
@@ -78,7 +86,12 @@ export function createCliModeRunner(
         await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8");
 
         const renderedPrompt = await renderPrompt(prompt, workspaceDir);
-        const run = await runPromptAndCapture(renderedPrompt, workspaceDir, 6, modelConfig);
+        const run = await runPromptAndCapture(
+          renderedPrompt,
+          workspaceDir,
+          context.evalCase?.runtime?.maxTurns ?? 6,
+          modelConfig
+        );
         const workspaceFiles = await readDirectoryFiles(workspaceDir, { ignore: IGNORE_WORKSPACE_FILES });
 
         return {
@@ -86,11 +99,12 @@ export function createCliModeRunner(
           actual: {
             assistantOutput: run.output,
             workspaceFiles,
+            trace: run.trace,
           },
-          assistantMessageCount: run.assistantMessageCount,
-          toolCallCount: run.toolsUsed.length,
-          toolsUsed: run.toolsUsed.map((entry) => entry.tool),
-          skillsInvoked: run.skillsInvoked,
+          assistantMessageCount: run.trace.assistantMessageCount,
+          toolCallCount: run.trace.toolsUsed.length,
+          toolsUsed: run.trace.toolsUsed.map((entry) => entry.tool),
+          skillsInvoked: run.trace.skillsInvoked,
           tokenUsage: run.tokenUsage ?? null,
         };
       } catch (error) {
@@ -100,6 +114,7 @@ export function createCliModeRunner(
           actual: {
             assistantOutput: "",
             workspaceFiles: {},
+            trace: emptyCliTrace(),
           },
           error: message,
           assistantMessageCount: 0,
@@ -112,11 +127,14 @@ export function createCliModeRunner(
         await rm(workspaceDir, { recursive: true, force: true });
       }
     },
-    validate({ actual, initial, expected }) {
+    validate({ evalCase, actual, initial, expected }) {
       return validateCliWorkspace({
         actualFiles: actual.workspaceFiles,
         expectedFiles: expected?.files,
         initialFiles: initial?.files,
+        assistantOutput: actual.assistantOutput,
+        trace: actual.trace,
+        cliExpect: evalCase.cliExpect,
       });
     },
     buildArtifacts(actual): BenchmarkArtifactFile[] {
@@ -125,6 +143,17 @@ export function createCliModeRunner(
           path: "assistant-output.txt",
           content: `${actual.assistantOutput}\n`,
         },
+        {
+          path: "trace.json",
+          content: JSON.stringify(actual.trace, null, 2) + "\n",
+        },
+        {
+          path: "wmill-invocations.jsonl",
+          content:
+            actual.trace.wmillInvocations
+              .map((entry) => JSON.stringify(entry))
+              .join("\n") + (actual.trace.wmillInvocations.length > 0 ? "\n" : ""),
+        },
       ];
 
       for (const [filePath, content] of Object.entries(actual.workspaceFiles)) {
@@ -139,6 +168,19 @@ export function createCliModeRunner(
   };
 }
 
+function emptyCliTrace(): CliTrace {
+  return {
+    toolsUsed: [],
+    skillsInvoked: [],
+    assistantMessageCount: 0,
+    bashCommands: [],
+    proposedCommands: [],
+    executedWmillCommands: [],
+    wmillInvocations: [],
+    firstMutationToolIndex: null,
+  };
+}
+
 export function getCliRunModelLabel(
   modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL
 ): string {

docs/system-prompt-testing-status.md

@@ -75,16 +75,27 @@ Each case is intentionally small:
 - optional `initial`
 - optional `expected`
 - optional `validate`
+- optional `cliExpect`
 
 `validate` is mainly used for stronger deterministic checks where exact fixture
 matching would be too strict, especially for `flow` creation cases.
 
+`cliExpect` is used by CLI-mode cases to assert agent behavior deterministically,
+including:
+
+- required or forbidden skills
+- skills invoked before the first file mutation
+- ordered `wmill` command proposals in the assistant response
+- forbidden attempted `wmill` executions
+- read-only guidance cases where the workspace must stay unchanged
+
 Examples of current deterministic checks:
 
 - schema contains one of several accepted input shapes
 - `results.*` references resolve
 - required code/input characteristics exist in some module
 - expected workspace files are created in `cli` mode
+- expected CLI skills and proposed `wmill` commands are observed in `cli` mode
 
 ## Model Selection