diff --git a/ai_evals/cases/flow.yaml b/ai_evals/cases/flow.yaml index 5f4abafc48..a21ae81f17 100644 --- a/ai_evals/cases/flow.yaml +++ b/ai_evals/cases/flow.yaml @@ -359,6 +359,8 @@ - request_approval - finalize_purchase topLevelStepTypes: + - id: request_approval + type: [rawscript, script] - id: finalize_purchase type: rawscript schemaRequiredPaths: @@ -373,6 +375,7 @@ judgeChecklist: - "the flow includes an approval step named `request_approval`" - "`request_approval` pauses the flow and asks the approver for a comment" + - "`request_approval` is a real script step that generates approval/resume URLs (e.g. via `getResumeUrls`) so approvers receive an actionable link, not a no-op passthrough (identity) step" - one approval is enough to continue - "the flow includes a final step named `finalize_purchase`" - "`finalize_purchase` returns an approved status object after approval" diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 52667fa321..f142bc8d36 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -47,7 +47,7 @@ export interface FlowValidationSpec { }>; topLevelStepTypes?: Array<{ id: string; - type: string; + type: string | string[]; }>; moduleRules?: Array<{ id: string; diff --git a/ai_evals/core/validators.ts b/ai_evals/core/validators.ts index 23a6709f9b..7570ccd12d 100644 --- a/ai_evals/core/validators.ts +++ b/ai_evals/core/validators.ts @@ -1378,11 +1378,14 @@ function validateFlowRequirements( continue; } + const allowedTypes = Array.isArray(requiredStep.type) + ? requiredStep.type + : [requiredStep.type]; checks.push( check( `${requiredStep.id} type matches required`, - getModuleType(module) === requiredStep.type, - `expected ${requiredStep.type}, got ${getModuleType(module) ?? "(missing)"}` + allowedTypes.includes(getModuleType(module) ?? ""), + `expected ${allowedTypes.join(" or ")}, got ${getModuleType(module) ?? "(missing)"}` ) ); } diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 03a5c09eb7..298db50ca4 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -5151,8 +5151,11 @@ input_transforms: ## Approval / Suspend Structure +An approval step is a normal **script** step (\`type: rawscript\` or \`type: script\`) that is turned into an approval by adding a module-level \`suspend\`. Its script calls \`wmill.getResumeUrls(approver)\` to generate the secret resume/cancel URLs and returns them so they can be sent to the approver(s) (Slack, email, etc.) or approved from the run page. + - \`suspend\` belongs on the flow module object itself, as a sibling of \`id\` and \`value\` - Never put \`suspend\` inside \`value\` +- Do NOT use \`type: identity\` for an approval step. An identity step suspends but never produces the resume URLs, so approvers have no link to act on — it is not a functional approval. Correct shape: @@ -5168,10 +5171,23 @@ Correct shape: type: string required: [comment] value: - type: identity + type: rawscript + language: bun + input_transforms: + approver: + type: static + value: '' + content: | + import * as wmill from "windmill-client" + + export async function main(approver?: string) { + const urls = await wmill.getResumeUrls(approver) + // send urls.resume / urls.cancel to the approver(s), e.g. via Slack or email + return urls + } \`\`\` -Incorrect shape: +Incorrect shape (suspend misplaced inside \`value\`): \`\`\`yaml - id: request_approval @@ -5181,6 +5197,16 @@ Incorrect shape: required_events: 1 \`\`\` +Incorrect shape (identity has no resume URLs — not a real approval): + +\`\`\`yaml +- id: request_approval + suspend: + required_events: 1 + value: + type: identity +\`\`\` + ## Branch Result Scope Rules - Inside a branch, you may reference earlier outer steps and earlier steps in the same branch diff --git a/system_prompts/auto-generated/flow.md b/system_prompts/auto-generated/flow.md index 70bec07fe6..08bcc2d641 100644 --- a/system_prompts/auto-generated/flow.md +++ b/system_prompts/auto-generated/flow.md @@ -139,8 +139,11 @@ input_transforms: ## Approval / Suspend Structure +An approval step is a normal **script** step (`type: rawscript` or `type: script`) that is turned into an approval by adding a module-level `suspend`. Its script calls `wmill.getResumeUrls(approver)` to generate the secret resume/cancel URLs and returns them so they can be sent to the approver(s) (Slack, email, etc.) or approved from the run page. + - `suspend` belongs on the flow module object itself, as a sibling of `id` and `value` - Never put `suspend` inside `value` +- Do NOT use `type: identity` for an approval step. An identity step suspends but never produces the resume URLs, so approvers have no link to act on — it is not a functional approval. Correct shape: @@ -156,10 +159,23 @@ Correct shape: type: string required: [comment] value: - type: identity + type: rawscript + language: bun + input_transforms: + approver: + type: static + value: '' + content: | + import * as wmill from "windmill-client" + + export async function main(approver?: string) { + const urls = await wmill.getResumeUrls(approver) + // send urls.resume / urls.cancel to the approver(s), e.g. via Slack or email + return urls + } ``` -Incorrect shape: +Incorrect shape (suspend misplaced inside `value`): ```yaml - id: request_approval @@ -169,6 +185,16 @@ Incorrect shape: required_events: 1 ``` +Incorrect shape (identity has no resume URLs — not a real approval): + +```yaml +- id: request_approval + suspend: + required_events: 1 + value: + type: identity +``` + ## Branch Result Scope Rules - Inside a branch, you may reference earlier outer steps and earlier steps in the same branch diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 58ac1ac69d..16ee0616bc 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -170,8 +170,11 @@ input_transforms: ## Approval / Suspend Structure +An approval step is a normal **script** step (\`type: rawscript\` or \`type: script\`) that is turned into an approval by adding a module-level \`suspend\`. Its script calls \`wmill.getResumeUrls(approver)\` to generate the secret resume/cancel URLs and returns them so they can be sent to the approver(s) (Slack, email, etc.) or approved from the run page. + - \`suspend\` belongs on the flow module object itself, as a sibling of \`id\` and \`value\` - Never put \`suspend\` inside \`value\` +- Do NOT use \`type: identity\` for an approval step. An identity step suspends but never produces the resume URLs, so approvers have no link to act on — it is not a functional approval. Correct shape: @@ -187,10 +190,23 @@ Correct shape: type: string required: [comment] value: - type: identity + type: rawscript + language: bun + input_transforms: + approver: + type: static + value: '' + content: | + import * as wmill from "windmill-client" + + export async function main(approver?: string) { + const urls = await wmill.getResumeUrls(approver) + // send urls.resume / urls.cancel to the approver(s), e.g. via Slack or email + return urls + } \`\`\` -Incorrect shape: +Incorrect shape (suspend misplaced inside \`value\`): \`\`\`yaml - id: request_approval @@ -200,6 +216,16 @@ Incorrect shape: required_events: 1 \`\`\` +Incorrect shape (identity has no resume URLs — not a real approval): + +\`\`\`yaml +- id: request_approval + suspend: + required_events: 1 + value: + type: identity +\`\`\` + ## Branch Result Scope Rules - Inside a branch, you may reference earlier outer steps and earlier steps in the same branch diff --git a/system_prompts/auto-generated/skills/write-flow/SKILL.md b/system_prompts/auto-generated/skills/write-flow/SKILL.md index a2a5d294e7..360b0d1ec9 100644 --- a/system_prompts/auto-generated/skills/write-flow/SKILL.md +++ b/system_prompts/auto-generated/skills/write-flow/SKILL.md @@ -225,8 +225,11 @@ input_transforms: ## Approval / Suspend Structure +An approval step is a normal **script** step (`type: rawscript` or `type: script`) that is turned into an approval by adding a module-level `suspend`. Its script calls `wmill.getResumeUrls(approver)` to generate the secret resume/cancel URLs and returns them so they can be sent to the approver(s) (Slack, email, etc.) or approved from the run page. + - `suspend` belongs on the flow module object itself, as a sibling of `id` and `value` - Never put `suspend` inside `value` +- Do NOT use `type: identity` for an approval step. An identity step suspends but never produces the resume URLs, so approvers have no link to act on — it is not a functional approval. Correct shape: @@ -242,10 +245,23 @@ Correct shape: type: string required: [comment] value: - type: identity + type: rawscript + language: bun + input_transforms: + approver: + type: static + value: '' + content: | + import * as wmill from "windmill-client" + + export async function main(approver?: string) { + const urls = await wmill.getResumeUrls(approver) + // send urls.resume / urls.cancel to the approver(s), e.g. via Slack or email + return urls + } ``` -Incorrect shape: +Incorrect shape (suspend misplaced inside `value`): ```yaml - id: request_approval @@ -255,6 +271,16 @@ Incorrect shape: required_events: 1 ``` +Incorrect shape (identity has no resume URLs — not a real approval): + +```yaml +- id: request_approval + suspend: + required_events: 1 + value: + type: identity +``` + ## Branch Result Scope Rules - Inside a branch, you may reference earlier outer steps and earlier steps in the same branch diff --git a/system_prompts/base/flow-base.md b/system_prompts/base/flow-base.md index 17f46333f3..def01a3535 100644 --- a/system_prompts/base/flow-base.md +++ b/system_prompts/base/flow-base.md @@ -139,8 +139,11 @@ input_transforms: ## Approval / Suspend Structure +An approval step is a normal **script** step (`type: rawscript` or `type: script`) that is turned into an approval by adding a module-level `suspend`. Its script calls `wmill.getResumeUrls(approver)` to generate the secret resume/cancel URLs and returns them so they can be sent to the approver(s) (Slack, email, etc.) or approved from the run page. + - `suspend` belongs on the flow module object itself, as a sibling of `id` and `value` - Never put `suspend` inside `value` +- Do NOT use `type: identity` for an approval step. An identity step suspends but never produces the resume URLs, so approvers have no link to act on — it is not a functional approval. Correct shape: @@ -156,10 +159,23 @@ Correct shape: type: string required: [comment] value: - type: identity + type: rawscript + language: bun + input_transforms: + approver: + type: static + value: '' + content: | + import * as wmill from "windmill-client" + + export async function main(approver?: string) { + const urls = await wmill.getResumeUrls(approver) + // send urls.resume / urls.cancel to the approver(s), e.g. via Slack or email + return urls + } ``` -Incorrect shape: +Incorrect shape (suspend misplaced inside `value`): ```yaml - id: request_approval @@ -169,6 +185,16 @@ Incorrect shape: required_events: 1 ``` +Incorrect shape (identity has no resume URLs — not a real approval): + +```yaml +- id: request_approval + suspend: + required_events: 1 + value: + type: identity +``` + ## Branch Result Scope Rules - Inside a branch, you may reference earlier outer steps and earlier steps in the same branch