test: add cli flow benchmark case

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
centdix
2026-03-31 18:45:06 +02:00
co-authored by Claude Opus 4.5
parent df01f33dfa
commit 3426e01584
+29
View File
@@ -0,0 +1,29 @@
[
{
"id": "bun-hello-flow",
"description": "Create a minimal Bun flow in a fresh CLI workspace.",
"prompt": "This is a benchmark harness. Create exactly one Windmill flow folder at {{workspace_root}}/f/evals/hello__flow. The flow must contain flow.yaml and one inline Bun script file named hello.ts. The flow should accept a name string input and return an object { greeting: `Hello, ${name}!` }. Use a single rawscript step wired to that input. Keep it minimal. Do not create any other flows or scripts. Do not run any CLI commands. After writing the files, tell me exactly which wmill commands I should run next.",
"maxTurns": 8,
"expectedSkill": "write-flow",
"expectedOutputSubstrings": [
"wmill flow generate-locks",
"wmill sync push"
],
"expectedFiles": [
{
"path": "f/evals/hello__flow/flow.yaml",
"mustContain": [
"steps:",
"name:"
]
},
{
"path": "f/evals/hello__flow/hello.ts",
"mustContain": [
"export async function main(name: string)",
"greeting: `Hello, ${name}!`"
]
}
]
}
]