From 3426e01584aceef5223204e0575a3270ca8a5af1 Mon Sep 17 00:00:00 2001 From: centdix Date: Tue, 31 Mar 2026 18:45:06 +0200 Subject: [PATCH] test: add cli flow benchmark case Co-Authored-By: Claude Opus 4.5 --- ai_evals/cases/cli/flow.json | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 ai_evals/cases/cli/flow.json diff --git a/ai_evals/cases/cli/flow.json b/ai_evals/cases/cli/flow.json new file mode 100644 index 0000000000..6b72a8a0cb --- /dev/null +++ b/ai_evals/cases/cli/flow.json @@ -0,0 +1,29 @@ +[ + { + "id": "bun-hello-flow", + "description": "Create a minimal Bun flow in a fresh CLI workspace.", + "prompt": "This is a benchmark harness. Create exactly one Windmill flow folder at {{workspace_root}}/f/evals/hello__flow. The flow must contain flow.yaml and one inline Bun script file named hello.ts. The flow should accept a name string input and return an object { greeting: `Hello, ${name}!` }. Use a single rawscript step wired to that input. Keep it minimal. Do not create any other flows or scripts. Do not run any CLI commands. After writing the files, tell me exactly which wmill commands I should run next.", + "maxTurns": 8, + "expectedSkill": "write-flow", + "expectedOutputSubstrings": [ + "wmill flow generate-locks", + "wmill sync push" + ], + "expectedFiles": [ + { + "path": "f/evals/hello__flow/flow.yaml", + "mustContain": [ + "steps:", + "name:" + ] + }, + { + "path": "f/evals/hello__flow/hello.ts", + "mustContain": [ + "export async function main(name: string)", + "greeting: `Hello, ${name}!`" + ] + } + ] + } +]