mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-10-13 08:02:47 +00:00
Ruben FiszelandClaude Opus 5.5
818ddd5d31
* feat: default new raw apps to react 19 and ask for the data setup Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * fix: say what app new --schema writes and soften the already-set-up wording Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * fix: resolve bare init_app table names in the app schema, fix --schema help Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2927 lines
108 KiB
YAML
2927 lines
108 KiB
YAML
- id: global-test1-script-create
|
|
prompt: |-
|
|
Create a draft Bun script at `f/evals/global/greet_user`.
|
|
It should take a string `name` input and return `Hello, ${name}!`.
|
|
Leave it as an AI draft only; do not deploy or save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
path: f/evals/global/greet_user
|
|
language: bun
|
|
valueIncludes:
|
|
- name
|
|
- Hello
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_script
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
- creates a Bun script draft at f/evals/global/greet_user
|
|
- the script accepts a name input
|
|
- the script returns a greeting containing Hello, the provided name, and an exclamation mark
|
|
- the result stays as an AI draft and is not deployed or saved to the workspace
|
|
|
|
- id: global-test2-script-edit-existing
|
|
prompt: |-
|
|
Update the existing workspace script at `f/evals/global/format_greeting`.
|
|
Keep it as a Bun script, but change the greeting so the provided name is uppercased and the returned message ends with an exclamation mark.
|
|
Leave the result as an AI draft only; do not deploy or save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
path: f/evals/global/format_greeting
|
|
language: bun
|
|
valueIncludes:
|
|
- toUpperCase
|
|
- "!"
|
|
toolExpect:
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
- creates an AI draft for the existing f/evals/global/format_greeting script
|
|
- preserves the script as Bun
|
|
- uppercases the provided name in the greeting
|
|
- returns a message ending with an exclamation mark
|
|
- does not deploy or save the draft to the workspace
|
|
|
|
- id: global-test3-flow-create
|
|
prompt: |-
|
|
Create a draft flow at `f/evals/global/sum_numbers`.
|
|
It should take two numeric inputs, `a` and `b`, and return their sum.
|
|
Leave it as an AI draft only; do not deploy or save it.
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: flow
|
|
path: f/evals/global/sum_numbers
|
|
valueIncludes:
|
|
- modules
|
|
- rawscript
|
|
- flow_input.a
|
|
- flow_input.b
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_flow
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: write_flow
|
|
field: modules
|
|
stringStartsWithAnyOf:
|
|
- "["
|
|
judgeChecklist:
|
|
- creates a flow draft at f/evals/global/sum_numbers
|
|
- the flow accepts numeric inputs a and b
|
|
- the flow returns the sum of a and b
|
|
- the result stays as an AI draft and is not deployed or saved to the workspace
|
|
|
|
- id: global-test4-multi-artifact-notification-job
|
|
prompt: |-
|
|
Set up a draft stale-trial notification job.
|
|
Create a Bun script at `f/evals/global/check_stale_trials` that accepts `max_age_days`, uses mocked inline trial account data, and returns the stale trial account IDs.
|
|
Also create a weekday 09:00 UTC schedule at `f/evals/global/check_stale_trials_weekday` for that script with `max_age_days` set to 14.
|
|
Add an HTTP POST trigger at `f/evals/global/check_stale_trials_manual` with route path `evals/check-stale-trials` that runs the same script manually.
|
|
Leave everything as AI drafts only; do not deploy or save anything to the workspace.
|
|
runtime:
|
|
maxTurns: 12
|
|
validate:
|
|
draftCountExactly: 3
|
|
requiredDrafts:
|
|
- type: script
|
|
path: f/evals/global/check_stale_trials
|
|
language: bun
|
|
valueIncludes:
|
|
- max_age_days
|
|
- trial
|
|
- type: schedule
|
|
path: f/evals/global/check_stale_trials_weekday
|
|
valueIncludes:
|
|
- f/evals/global/check_stale_trials
|
|
- UTC
|
|
- "14"
|
|
- type: trigger
|
|
triggerKind: http
|
|
path: f/evals/global/check_stale_trials_manual
|
|
valueIncludes:
|
|
- evals/check-stale-trials
|
|
- f/evals/global/check_stale_trials
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_script
|
|
- write_schedule
|
|
- write_trigger
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
- creates a Bun script draft for stale trial accounts
|
|
- creates a weekday 09:00 UTC schedule draft for the script with max_age_days set to 14
|
|
- creates an HTTP POST trigger draft with route path evals/check-stale-trials for the same script
|
|
- leaves all artifacts as drafts only and does not deploy
|
|
|
|
- id: global-test5-existing-flow-inline-code-edit
|
|
prompt: |-
|
|
Update the existing flow at `f/evals/global/process_invoice`.
|
|
Only change the `calculate_total` inline code so it applies 8% tax and returns an object containing `subtotal`, `tax`, and `total`.
|
|
Leave the updated flow as an AI draft only; do not deploy or save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: flow
|
|
path: f/evals/global/process_invoice
|
|
valueIncludes:
|
|
- calculate_total
|
|
- tax
|
|
- total
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- read_workspace_item
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
- reads the existing process_invoice flow before editing it
|
|
- updates the calculate_total inline code to apply 8% tax
|
|
- returns subtotal, tax, and total from the updated flow logic
|
|
- leaves the result as an AI draft only
|
|
|
|
- id: global-test6-secret-variable-draft
|
|
prompt: |-
|
|
Create a secret variable draft at `f/evals/global/slack_bot_token`.
|
|
Use the placeholder value `xoxb-redacted-test-token` and description `Slack bot token for eval notifications`.
|
|
Do not create any resource or deploy anything.
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: variable
|
|
path: f/evals/global/slack_bot_token
|
|
valueIncludes:
|
|
- Slack bot token
|
|
- "true"
|
|
forbiddenDrafts:
|
|
- type: resource
|
|
path: f/evals/global/slack_bot_token
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_variable
|
|
forbiddenToolsUsed:
|
|
- write_resource
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: write_variable
|
|
field: value
|
|
stringStartsWithAnyOf:
|
|
- xoxb-redacted-test-token
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- creates exactly one secret variable draft at f/evals/global/slack_bot_token
|
|
- uses the requested placeholder value and description
|
|
- does not create a resource or deploy anything
|
|
|
|
# A new app that has to store data, with nothing said about where: the data setup is the
|
|
# user's call, so it is asked before anything is created. The question is dismissed here,
|
|
# which leaves nothing the model may build on.
|
|
- id: global-test7-ambiguous-app-asks-question
|
|
prompt: |-
|
|
Create a new raw app for triaging support tickets.
|
|
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
|
|
runtime:
|
|
maxTurns: 6
|
|
userAnswers: []
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- list_datatables
|
|
- askUserQuestion
|
|
forbiddenToolsUsed:
|
|
- init_app
|
|
- exec_datatable_sql
|
|
- write_app_file
|
|
- write_app_runnable
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: askUserQuestion
|
|
field: question
|
|
stringIncludesAnyOf:
|
|
- data table
|
|
- datatable
|
|
- schema
|
|
- tables
|
|
- store
|
|
skipJudge: true
|
|
|
|
# The same request with the question answered: the app is built on what the user chose.
|
|
- id: global-new-app-data-setup-answer-applied
|
|
prompt: |-
|
|
Create a new raw app for triaging support tickets.
|
|
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
|
|
runtime:
|
|
maxTurns: 32
|
|
userAnswers:
|
|
- A new schema named helpdesk
|
|
- Create new tables
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: app
|
|
valueIncludes:
|
|
- helpdesk
|
|
- react-dom/client
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- askUserQuestion
|
|
- init_app
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
toolCallArgs:
|
|
- tool: init_app
|
|
field: framework
|
|
stringEqualsAnyOf:
|
|
- react19
|
|
- tool: exec_datatable_sql
|
|
field: sql
|
|
stringIncludesAnyOf:
|
|
- helpdesk
|
|
skipJudge: true
|
|
|
|
# No framework named, no app to follow, nothing to store: the modal's recommended
|
|
# framework is used without a question.
|
|
- id: global-new-app-defaults-to-react19
|
|
prompt: |-
|
|
Build me a little app that converts temperatures between Celsius and Fahrenheit.
|
|
runtime:
|
|
maxTurns: 12
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: app
|
|
valueIncludes:
|
|
- react-dom/client
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- init_app
|
|
forbiddenToolsUsed:
|
|
- askUserQuestion
|
|
- exec_datatable_sql
|
|
- deploy_workspace_item
|
|
toolCallArgs:
|
|
- tool: init_app
|
|
field: framework
|
|
stringEqualsAnyOf:
|
|
- react19
|
|
skipJudge: true
|
|
|
|
# The user said where the data goes, so there is nothing left to ask.
|
|
- id: global-new-app-storage-specified-no-question
|
|
prompt: |-
|
|
Build a todo list app in React. Keep the todos in the main data table, in a new
|
|
schema called todo_app.
|
|
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
|
|
runtime:
|
|
maxTurns: 20
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: app
|
|
valueIncludes:
|
|
- todo_app
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- init_app
|
|
- write_app_runnable
|
|
forbiddenToolsUsed:
|
|
- askUserQuestion
|
|
- deploy_workspace_item
|
|
toolCallArgs:
|
|
- tool: exec_datatable_sql
|
|
field: sql
|
|
stringIncludesAnyOf:
|
|
- todo_app
|
|
skipJudge: true
|
|
|
|
- id: global-test8-human-script-infer-path-language
|
|
prompt: |-
|
|
I need a small helper that formats a customer-facing welcome line.
|
|
It should take a person's name and return "Welcome aboard, <name>!".
|
|
Please just stage it as a draft for now.
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
valueIncludes:
|
|
- Welcome aboard
|
|
- name
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_script
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
- creates a single script draft for a welcome-line helper
|
|
- accepts a person's name as input
|
|
- returns a message containing Welcome aboard, the provided name, and an exclamation mark
|
|
- chooses a reasonable workspace path and script language without needing the user to specify them
|
|
- leaves the result as an AI draft only
|
|
|
|
- id: global-test9-human-weekday-trial-job
|
|
prompt: |-
|
|
Can you set up a draft daily job that checks a few hard-coded trial accounts and returns the ones whose trial has ended?
|
|
It should run every weekday morning around 9 in UTC with a 30 day cutoff.
|
|
Keep it as draft work only.
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 2
|
|
requiredDrafts:
|
|
- type: script
|
|
pathIncludes:
|
|
- trial
|
|
valueIncludes:
|
|
- trial
|
|
- "30"
|
|
- type: schedule
|
|
pathIncludes:
|
|
- trial
|
|
valueIncludes:
|
|
- UTC
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_script
|
|
- write_schedule
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
- creates a script draft that checks hard-coded trial accounts
|
|
- returns the accounts whose trial has ended based on a 30 day cutoff
|
|
- creates a schedule draft for weekday mornings around 09:00 UTC
|
|
- links the schedule to the generated script
|
|
- leaves both artifacts as drafts only
|
|
|
|
- id: global-test10-human-secret-variable
|
|
prompt: |-
|
|
I need a placeholder Slack bot token stored securely for future notification work.
|
|
Use xoxb-redacted-test-token and note that it is for eval notifications.
|
|
Only prepare a draft.
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: variable
|
|
pathIncludes:
|
|
- slack
|
|
valueIncludes:
|
|
- eval notifications
|
|
- "true"
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_variable
|
|
forbiddenToolsUsed:
|
|
- write_resource
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: write_variable
|
|
field: value
|
|
stringStartsWithAnyOf:
|
|
- xoxb-redacted-test-token
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- creates a single secret variable draft for the Slack bot token placeholder
|
|
- uses the requested placeholder value
|
|
- includes a note or description that it is for eval notifications
|
|
- does not create a resource or deploy anything
|
|
|
|
- id: global-test11-human-existing-flow-informal-edit
|
|
prompt: |-
|
|
There is an invoice processing flow in this workspace.
|
|
Can you adjust its total calculation so it adds 8% tax and returns subtotal, tax, and total?
|
|
Keep the change as a draft.
|
|
initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: flow
|
|
pathIncludes:
|
|
- invoice
|
|
valueIncludes:
|
|
- calculate_total
|
|
- tax
|
|
- total
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- read_workspace_item
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
- finds and edits the existing invoice processing flow without the user providing its exact path
|
|
- updates the total calculation to apply 8% tax
|
|
- returns subtotal, tax, and total from the updated flow logic
|
|
- leaves the result as an AI draft only
|
|
|
|
- id: global-test12-current-live-script-edit
|
|
prompt: |-
|
|
The script I have open formats greetings.
|
|
Can you update this script so it uppercases the name before greeting them and ends with an exclamation mark?
|
|
Keep it as draft work.
|
|
initial: ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
path: f/evals/global/current_greeting
|
|
language: bun
|
|
valueIncludes:
|
|
- toUpperCase
|
|
- "!"
|
|
forbiddenDrafts:
|
|
- type: script
|
|
path: f/evals/global/format_greeting
|
|
- type: script
|
|
path: f/evals/global/format_greeting_archive
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- read_workspace_item
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
- resolves "this script" to the active live editor script instead of another similarly named workspace script
|
|
- updates the greeting logic to uppercase the provided name
|
|
- returns a greeting ending with an exclamation mark
|
|
- leaves the result as a draft only
|
|
|
|
- id: global-test13-current-live-flow-edit
|
|
prompt: |-
|
|
I have the invoice flow open.
|
|
In the current flow, update the total calculation to add 8% tax and return subtotal, tax, and total.
|
|
Keep the change as a draft.
|
|
initial: ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: flow
|
|
path: f/evals/global/current_invoice_flow
|
|
valueIncludes:
|
|
- calculate_total
|
|
- tax
|
|
- total
|
|
forbiddenDrafts:
|
|
- type: flow
|
|
path: f/evals/global/process_invoice
|
|
- type: flow
|
|
path: f/evals/global/process_refund
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- read_workspace_item
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
- resolves "current flow" to the active live editor flow
|
|
- does not edit the similarly named deployed invoice or refund flows
|
|
- updates the calculate_total logic to apply 8% tax
|
|
- returns subtotal, tax, and total from the updated flow logic
|
|
- leaves the result as a draft only
|
|
|
|
- id: global-test14-current-without-live-editor-asks-question
|
|
prompt: |-
|
|
Please update this script so it returns `ok`.
|
|
Keep it as a draft.
|
|
runtime:
|
|
maxTurns: 4
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- edit_script
|
|
- write_flow
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- asks which script to update when the user refers to "this script" without selected or active editor context
|
|
- does not guess a path or create a new script draft
|
|
|
|
- id: global-test15-human-postgres-resource
|
|
prompt: |-
|
|
I'm wiring the eval reporting database into this workspace.
|
|
Can you stage a Postgres connection for it in the shared evals/global folder?
|
|
Use host `reports-db.internal`, port 5432, database `evals_reporting`, user `report_reader`, and password `pg-redacted-reporting-password`.
|
|
Keep the credentials safe.
|
|
This is just draft work for now.
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 2
|
|
requiredDrafts:
|
|
- type: variable
|
|
pathStartsWith: f/evals/global/
|
|
pathIncludes:
|
|
- evals
|
|
- global
|
|
- report
|
|
- password
|
|
valueIncludes:
|
|
- "true"
|
|
- report
|
|
- type: resource
|
|
pathStartsWith: f/evals/global/
|
|
pathIncludes:
|
|
- evals
|
|
- global
|
|
- report
|
|
valueIncludes:
|
|
- postgres
|
|
- reports-db.internal
|
|
- "5432"
|
|
- evals_reporting
|
|
- report_reader
|
|
- "$var:"
|
|
valueExcludes:
|
|
- pg-redacted-reporting-password
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_variable
|
|
- search_resource_types
|
|
- write_resource
|
|
forbiddenToolsUsed:
|
|
- write_schedule
|
|
- write_trigger
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: write_variable
|
|
field: value
|
|
stringStartsWithAnyOf:
|
|
- pg-redacted-reporting-password
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- creates a Postgres resource draft for the eval reporting database
|
|
- creates a secret variable draft for the database password
|
|
- puts the drafts in sensible eval/global reporting-related paths
|
|
- uses the requested host, port, database, and user
|
|
- references the secret variable from the resource instead of embedding the password
|
|
- leaves the work as a draft only
|
|
|
|
- id: global-test16-human-visible-variable
|
|
prompt: |-
|
|
We keep reusing a 30 day trial cutoff in eval notification jobs.
|
|
Can you stage that as a normal workspace variable in the shared evals/global folder, with a short description so people know what it controls?
|
|
It is not a secret.
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: variable
|
|
pathStartsWith: f/evals/global/
|
|
pathIncludes:
|
|
- evals
|
|
- global
|
|
- trial
|
|
valueIncludes:
|
|
- "30"
|
|
- "false"
|
|
- trial
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_variable
|
|
forbiddenToolsUsed:
|
|
- write_resource
|
|
- write_schedule
|
|
- write_trigger
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
- creates exactly one non-secret variable draft for the trial cutoff
|
|
- stores the value 30
|
|
- chooses a sensible eval/global path related to trials or notifications
|
|
- includes a useful description of what the value controls
|
|
- does not create resources, schedules, triggers, or deployed workspace changes
|
|
|
|
- id: global-test17-human-schedule-existing-helper
|
|
prompt: |-
|
|
The workspace already has a report digest helper.
|
|
Can you stage a weekday 8:30 AM UTC run for it with `dry_run` turned on?
|
|
I only want the schedule draft for review.
|
|
initial: ai_evals/fixtures/frontend/global/initial/report_digest_script.json
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: schedule
|
|
pathIncludes:
|
|
- digest
|
|
valueIncludes:
|
|
- f/evals/global/send_report_digest
|
|
- UTC
|
|
- dry_run
|
|
- "true"
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- list_workspace_items
|
|
- write_schedule
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- write_flow
|
|
- write_resource
|
|
- write_variable
|
|
- write_trigger
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
- finds the existing report digest helper rather than creating a new script or flow
|
|
- creates one schedule draft for that helper
|
|
- schedules it for weekdays around 08:30 UTC
|
|
- passes dry_run as true
|
|
- leaves only the schedule draft for review
|
|
|
|
- id: global-test29-schedule-with-retry-and-error-handler
|
|
prompt: |-
|
|
The workspace already has a report digest helper.
|
|
Schedule it every morning at 6am UTC with `dry_run` on, and have it retry twice
|
|
if it fails. Run it on our `nightly` worker group.
|
|
Draft only, I'll review before deploying.
|
|
initial: ai_evals/fixtures/frontend/global/initial/report_digest_script.json
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: schedule
|
|
pathIncludes:
|
|
- digest
|
|
valueIncludes:
|
|
- f/evals/global/send_report_digest
|
|
- attempts
|
|
- nightly
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_schedule
|
|
- get_schedule_schema
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- write_flow
|
|
- deploy_workspace_item
|
|
judgeChecklist:
|
|
- finds the existing report digest helper rather than creating a new script or flow
|
|
- creates one schedule draft for that helper running daily around 06:00 UTC
|
|
- configures a retry policy with two attempts
|
|
- routes the schedule to the nightly worker tag
|
|
- leaves the schedule as a draft without deploying
|
|
|
|
- id: global-test18-human-slack-resource-with-secret
|
|
prompt: |-
|
|
I'm preparing Slack notifications for eval failures.
|
|
Can you stage a Slack connection in the shared evals/global folder?
|
|
The bot token is `xoxb-redacted-test-token`; keep it safe.
|
|
Don't deploy anything yet.
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 2
|
|
requiredDrafts:
|
|
- type: variable
|
|
pathStartsWith: f/evals/global/
|
|
pathIncludes:
|
|
- evals
|
|
- global
|
|
- slack
|
|
- token
|
|
valueIncludes:
|
|
- "true"
|
|
- type: resource
|
|
pathStartsWith: f/evals/global/
|
|
pathIncludes:
|
|
- evals
|
|
- global
|
|
- slack
|
|
valueIncludes:
|
|
- slack
|
|
- "$var:"
|
|
valueExcludes:
|
|
- xoxb-redacted-test-token
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_variable
|
|
- search_resource_types
|
|
- write_resource
|
|
forbiddenToolsUsed:
|
|
- write_schedule
|
|
- write_trigger
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: write_variable
|
|
field: value
|
|
stringStartsWithAnyOf:
|
|
- xoxb-redacted-test-token
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- creates a secret variable draft for the Slack bot token placeholder
|
|
- creates a Slack resource draft that references the secret variable instead of embedding the token
|
|
- keeps both drafts under a sensible eval/global Slack-related path
|
|
- does not create schedules, triggers, or deployed workspace changes
|
|
|
|
- id: global-test19-datatable-not-configured-asks-to-set-up
|
|
prompt: |-
|
|
Here are two newsletter signups: alice@example.com and bob@example.com.
|
|
Save them into a workspace data table for me.
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- list_datatables
|
|
forbiddenToolsUsed:
|
|
- exec_datatable_sql
|
|
- write_script
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- checks which data tables exist in the workspace before acting
|
|
- recognizes that no data table is configured in this workspace
|
|
- explains that a data table must first be set up by the user in the workspace settings (Data Tables) and is not created via SQL
|
|
- does not run SQL, write a script, or invent a data table to work around the missing configuration
|
|
- tells the user to configure a data table and then try again
|
|
|
|
- id: global-test20-datatable-no-hallucinated-main
|
|
prompt: |-
|
|
Pull the latest rows from the orders table in our data table so I can see recent orders.
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- list_datatables
|
|
forbiddenToolsUsed:
|
|
- exec_datatable_sql
|
|
- write_script
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- checks which data tables exist in the workspace before querying
|
|
- recognizes that no data table is configured in this workspace
|
|
- does not assume a data table named "main" (or any other name) exists
|
|
- does not run SQL against a guessed data table or fabricate order rows
|
|
- tells the user they need to set up a data table in the workspace settings first
|
|
|
|
- id: global-test21-datatable-list-summarize
|
|
prompt: |-
|
|
What tables do we have in our workspace data table? Just give me the list.
|
|
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- list_datatables
|
|
forbiddenToolsUsed:
|
|
- get_datatable_table_schema
|
|
- exec_datatable_sql
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- lists the tables available in the workspace data table (orders and customers)
|
|
- answers from the data table listing rather than fabricating table names
|
|
- does not fetch column details or run SQL just to produce a table list
|
|
|
|
- id: global-test22-datatable-inspect-columns
|
|
prompt: |-
|
|
What columns does the orders table have in our workspace data table?
|
|
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- get_datatable_table_schema
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- inspects the orders table schema in the workspace data table
|
|
- reports the orders columns (such as id, customer_id, total, status, created_at)
|
|
- answers from the retrieved schema rather than guessing the columns
|
|
|
|
- id: global-test23-datatable-query-select
|
|
prompt: |-
|
|
Show me the orders in our workspace data table, including their status and total.
|
|
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- exec_datatable_sql
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: exec_datatable_sql
|
|
field: sql
|
|
stringIncludesAnyOf:
|
|
- select
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- runs a SELECT query against the orders table in the workspace data table
|
|
- reports the orders returned by the query back to the user instead of fabricating data
|
|
- does not tell the user to set up a data table, since one already exists
|
|
|
|
- id: global-test24-datatable-create-table
|
|
prompt: |-
|
|
Add a new table called refunds to our workspace data table, with an order id and a refund amount.
|
|
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- exec_datatable_sql
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: exec_datatable_sql
|
|
field: sql
|
|
stringIncludesAnyOf:
|
|
- create table
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- creates the refunds table with a plain CREATE TABLE statement on the data table
|
|
- includes an order id and a refund amount column
|
|
- treats creating the table as a normal SQL statement and does not claim a separate registration step is needed
|
|
- does not write a script to create the table
|
|
|
|
- id: global-test25-datatable-mutate-rows
|
|
prompt: |-
|
|
Mark order number 2 as shipped in our workspace data table.
|
|
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
|
|
runtime:
|
|
# Headroom for inspect-schema -> UPDATE -> verify; the in-memory engine now
|
|
# persists the write, so verification confirms on the first try (no loop).
|
|
maxTurns: 12
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- exec_datatable_sql
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: exec_datatable_sql
|
|
field: sql
|
|
stringIncludesAnyOf:
|
|
- update
|
|
- insert into
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- runs an UPDATE on the orders table setting the status of order id 2 to shipped
|
|
- targets only order number 2 rather than rewriting the whole table
|
|
- confirms the change back to the user
|
|
|
|
- id: global-test26-datatable-script-sdk
|
|
prompt: |-
|
|
Write a script that reads our workspace data table and returns the total revenue across all orders.
|
|
Leave it as an AI draft only; do not deploy or save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
valueIncludes:
|
|
- wmill.datatable(
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- get_instructions
|
|
- write_script
|
|
forbiddenToolsUsed:
|
|
- exec_datatable_sql
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
# The judge has no datatable SDK reference and wrongly penalizes correct
|
|
# wmill.datatable() tagged-template usage, so rely on the deterministic checks:
|
|
# required get_instructions + write_script, forbidden exec_datatable_sql, and a
|
|
# draft that contains wmill.datatable(.
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- writes a script (not a chat-time SQL execution) that reads the workspace data table at runtime
|
|
- uses the wmill.datatable() SDK to query the orders table and sum the order totals
|
|
- returns the total revenue from the script
|
|
- leaves the result as an AI draft and does not deploy or save it
|
|
|
|
- id: global-test27-list-recent-runs
|
|
prompt: |-
|
|
What are the most recent runs in this workspace? Give me a quick rundown of what ran and whether it succeeded.
|
|
initial: ai_evals/fixtures/frontend/global/initial/jobs_seed.json
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- list_runs
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
- write_script
|
|
# Read-only job inspection produces no draft, so the global judge (which only
|
|
# sees the drafts artifact) would score it empty — validate via tool use.
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- lists the recent runs from the workspace rather than fabricating them
|
|
- summarizes each run's path and success/failure from the returned data
|
|
|
|
- id: global-test28-fetch-failed-run-logs
|
|
prompt: |-
|
|
The run with id 01920000-0000-7000-8000-0000000000f1 failed. Pull its logs and tell me what went wrong.
|
|
initial: ai_evals/fixtures/frontend/global/initial/jobs_seed.json
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- get_run
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
- write_script
|
|
toolCallArgs:
|
|
- tool: get_run
|
|
field: id
|
|
stringIncludesAnyOf:
|
|
- 01920000-0000-7000-8000-0000000000f1
|
|
# Same as above — no draft is produced, so rely on the deterministic tool-use
|
|
# and argument checks rather than the judge.
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- fetches the logs for the requested job id
|
|
- explains the failure from the returned logs (connection refused to the upstream API)
|
|
|
|
- id: global-run-args-and-result
|
|
prompt: |-
|
|
What was the run 01920000-0000-7000-8000-0000000000f2 called with, and what did it return?
|
|
initial: ai_evals/fixtures/frontend/global/initial/jobs_seed.json
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- get_run
|
|
forbiddenToolsUsed:
|
|
- test_run_script
|
|
- run_script
|
|
- deploy_workspace_item
|
|
toolCallArgs:
|
|
- tool: get_run
|
|
field: id
|
|
stringIncludesAnyOf:
|
|
- 01920000-0000-7000-8000-0000000000f2
|
|
# Read-only, so no draft for the global judge to score — validated on tool use
|
|
# and the deterministic argument check, like the neighbouring run cases.
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- reports the arguments the run was called with (region emea, 12 recipients)
|
|
- reports what the run returned (12 sent, 3 skipped)
|
|
- does not start a new run to find out
|
|
|
|
# --- Page navigation (open_page) ---
|
|
# The assistant should take the user to a Windmill page (Runs/Schedules) with the
|
|
# right filters via open_page, rather than describing where to click or dumping the
|
|
# data. No draft is produced, so the global judge is skipped and we validate the
|
|
# tool call and its arguments.
|
|
|
|
- id: global-openpage1-runs-failed-of-script
|
|
prompt: |-
|
|
Take me to the failed runs of the script at f/evals/global/greet_user so I can see what's going wrong.
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- open_page
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: open_page
|
|
field: page
|
|
stringIncludesAnyOf:
|
|
- runs
|
|
- tool: open_page
|
|
field: status
|
|
stringIncludesAnyOf:
|
|
- failure
|
|
- tool: open_page
|
|
field: path
|
|
stringIncludesAnyOf:
|
|
- f/evals/global/greet_user
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- opens the Runs page filtered to the failed runs of f/evals/global/greet_user
|
|
- applies both the failure status and the script path as filters
|
|
- does not write, deploy, or delete anything
|
|
|
|
- id: global-openpage2-runs-of-schedule
|
|
prompt: |-
|
|
Open the runs page filtered to the jobs triggered by the schedule f/evals/global/nightly_digest.
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- open_page
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: open_page
|
|
field: page
|
|
stringIncludesAnyOf:
|
|
- runs
|
|
- tool: open_page
|
|
field: schedule_path
|
|
stringIncludesAnyOf:
|
|
- f/evals/global/nightly_digest
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- opens the Runs page filtered to jobs triggered by the f/evals/global/nightly_digest schedule
|
|
- passes the schedule path as the filter
|
|
- does not write, deploy, or delete anything
|
|
|
|
- id: global-openpage3-open-schedule
|
|
prompt: |-
|
|
Open the schedule f/evals/global/nightly_digest so I can review and edit it.
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- open_page
|
|
forbiddenToolsUsed:
|
|
- write_schedule
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: open_page
|
|
field: page
|
|
stringIncludesAnyOf:
|
|
- schedules
|
|
- tool: open_page
|
|
field: open
|
|
stringIncludesAnyOf:
|
|
- f/evals/global/nightly_digest
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- opens the Schedules page and targets the f/evals/global/nightly_digest schedule for editing
|
|
- passes the schedule path so the editor opens on it
|
|
- does not write, deploy, or delete anything
|
|
|
|
- id: global-openpage4-workspace-settings-tab
|
|
prompt: |-
|
|
Take me to the Git sync configuration for this workspace.
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- open_page
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: open_page
|
|
field: page
|
|
stringIncludesAnyOf:
|
|
- workspace_settings
|
|
- tool: open_page
|
|
field: tab
|
|
stringIncludesAnyOf:
|
|
- git_sync
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- opens the Workspace settings page on the git_sync tab
|
|
- does not write, deploy, or delete anything
|
|
|
|
- id: global-openpage5-audit-logs-user
|
|
prompt: |-
|
|
Open the audit logs filtered to actions performed by the user admin.
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- open_page
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: open_page
|
|
field: page
|
|
stringIncludesAnyOf:
|
|
- audit_logs
|
|
- tool: open_page
|
|
field: username
|
|
stringIncludesAnyOf:
|
|
- admin
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- opens the Audit logs page filtered to the admin user
|
|
- does not write, deploy, or delete anything
|
|
|
|
- id: global-openpage6-triggers-kind
|
|
prompt: |-
|
|
Take me to the Kafka triggers for this workspace.
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- open_page
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: open_page
|
|
field: page
|
|
stringIncludesAnyOf:
|
|
- triggers
|
|
- tool: open_page
|
|
field: trigger_kind
|
|
stringIncludesAnyOf:
|
|
- kafka
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- opens the Kafka triggers page
|
|
- does not write, deploy, or delete anything
|
|
|
|
- id: global-draft-does-not-open-compare
|
|
prompt: |-
|
|
Create a TypeScript script draft at f/evals/global/compare_review_demo that returns the string "ok" (no need to test it). I want to look over the change before it gets deployed.
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 1
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_script
|
|
forbiddenToolsUsed:
|
|
- open_page
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- creates the script draft without opening any workspace page
|
|
- does not deploy or delete anything
|
|
|
|
- id: global-openpage7-compare-fork-deploy
|
|
prompt: |-
|
|
I'm done with my changes in this fork and want to ship them to the parent workspace myself. Take me there.
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- open_page
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: open_page
|
|
field: page
|
|
stringIncludesAnyOf:
|
|
- compare
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- opens Compare & Deploy only because the user explicitly wants to deploy the fork into its parent
|
|
- does not deploy or delete anything
|
|
|
|
- id: global-openpage8-runs-label-and-worker
|
|
prompt: |-
|
|
Show me the runs carrying the label nightly-digest that ran on the worker wk-eval-1.
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- open_page
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
# One page carrying both filters — two pages each carrying one is not the ask.
|
|
toolCallArgsSameCall:
|
|
- tool: open_page
|
|
args:
|
|
- field: page
|
|
stringIncludesAnyOf:
|
|
- runs
|
|
- field: label
|
|
stringIncludesAnyOf:
|
|
- nightly-digest
|
|
- field: worker
|
|
stringIncludesAnyOf:
|
|
- wk-eval-1
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- opens the Runs page filtered to the nightly-digest label on worker wk-eval-1
|
|
- does not write, deploy, or delete anything
|
|
|
|
# Exclusion is the filter shape most easily lost in translation: the page encodes it as
|
|
# a `!`-prefixed value, and a model that only knows the positive form silently opens a
|
|
# page showing exactly what the user asked to hide.
|
|
- id: global-openpage9-runs-exclude-schedules
|
|
prompt: |-
|
|
Open the runs page but hide everything a schedule kicked off — I only care about the rest.
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- open_page
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgsSameCall:
|
|
- tool: open_page
|
|
args:
|
|
- field: page
|
|
stringIncludesAnyOf:
|
|
- runs
|
|
- field: job_trigger_kind
|
|
stringIncludesAnyOf:
|
|
- '!schedule'
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- opens the Runs page with schedule-triggered jobs excluded
|
|
- does not write, deploy, or delete anything
|
|
|
|
- id: global-closepage1-close-runs-tab
|
|
prompt: |-
|
|
You just opened the runs page for me in the side panel. Close that tab, I'm done looking at it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/preview_runs_tab.json
|
|
runtime:
|
|
maxTurns: 6
|
|
sessionChat: true
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- close_page
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: close_page
|
|
field: match
|
|
stringIncludesAnyOf:
|
|
- runs
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- closes the runs preview tab in the side panel
|
|
- does not write, deploy, or delete anything
|
|
|
|
# --- Artifact version history ---
|
|
# Every content change to an artifact is snapshotted, and update_artifact requires a
|
|
# change_note that the user reads in the version picker. A blank note makes the history
|
|
# unreadable, so pin that the model fills it on every edit.
|
|
|
|
- id: global-artifact-note-on-each-edit
|
|
prompt: |-
|
|
Write up a short rollout plan for me as a doc I can come back to, covering a staged
|
|
rollout in three phases. Then add a rollback section to it, and after that tighten
|
|
the wording of phase 2.
|
|
runtime:
|
|
maxTurns: 12
|
|
sessionChat: true
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- create_artifact
|
|
- update_artifact
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
# The note is what the version picker shows; a blank one makes history unreadable.
|
|
- tool: update_artifact
|
|
field: change_note
|
|
nonEmpty: true
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- creates one artifact and revises it rather than creating a second artifact
|
|
- each revision carries a short description of what changed
|
|
|
|
# A reader who pins an older version in the artifact's picker is looking at something the
|
|
# artifact tools never report: an artifact tab carries no ACTIVE PREVIEW section, so the pin
|
|
# reaches the chat through get_preview_status alone. Asked what is on screen, the model has
|
|
# to read the panel instead of answering from the artifact's own history.
|
|
|
|
- id: global-artifact-pinned-version-question
|
|
prompt: |-
|
|
Which version of the onboarding plan am I looking at right now?
|
|
initial: ai_evals/fixtures/frontend/global/initial/artifact_onboarding_plan_pinned_v2.json
|
|
runtime:
|
|
maxTurns: 6
|
|
sessionChat: true
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- get_preview_status
|
|
forbiddenToolsUsed:
|
|
- create_artifact
|
|
- update_artifact
|
|
- deploy_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- answers that the panel is showing version 2, not the latest version 5
|
|
- does not edit or re-create the artifact
|
|
|
|
# --- Documentation search (search_docs) ---
|
|
# Pure product-knowledge questions: the assistant should consult the docs via
|
|
# search_docs and answer conversationally, not draft or mutate anything. No
|
|
# draft is produced, so the global judge is skipped and we validate tool use.
|
|
|
|
- id: global-docs-ai-agent-step
|
|
prompt: |-
|
|
Does Windmill support a flow step where an LLM decides which of my scripts to call based on the input?
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- search_docs
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- write_flow
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
|
|
- id: global-docs-retry-step
|
|
prompt: |-
|
|
How does automatic retry work for a flow step that calls a flaky API?
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- search_docs
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- write_flow
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
|
|
- id: global-docs-key-value-store
|
|
prompt: |-
|
|
Can I use a Redis-style key-value store from my Windmill scripts, and how?
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- search_docs
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- write_flow
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
|
|
- id: global-docs-cron-schedule-format
|
|
prompt: |-
|
|
How do Windmill's cron schedules work, and what format does the schedule expression use?
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- search_docs
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- write_flow
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
|
|
# --- Raw app on a large project (context-usage benchmark) ---
|
|
# These cases run against the deliberately large `analytics_dashboard` raw-app
|
|
# fixture (~20 frontend files incl. a 5k-line data module, plus backend runnables).
|
|
# They exist to measure how much context the global chat consumes when working in a
|
|
# big raw app: test29 is a read-heavy debugging hunt, test30 is a small edit baseline.
|
|
# tokenUsage is recorded per run, so the same cases re-run after a read-tool change
|
|
# (the read_app_file cap + offset/limit paging) quantify the optimization. skipJudge:
|
|
# the judge only sees the drafts artifact and cannot run the app, so we validate
|
|
# deterministically.
|
|
|
|
- id: global-test29-raw-app-debug-large
|
|
prompt: |-
|
|
The analytics dashboard app at `f/evals/global/analytics_dashboard` has a bug:
|
|
the Revenue Summary tile shows a total that is lower than the per-order line
|
|
totals and the per-region breakdown. Track down what is computing revenue
|
|
incorrectly and fix it. Keep the change as an AI draft only; do not deploy or
|
|
save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard
|
|
runtime:
|
|
maxTurns: 20
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: app
|
|
path: f/evals/global/analytics_dashboard
|
|
valueIncludes:
|
|
- "return order.unitPrice * order.quantity"
|
|
toolExpect:
|
|
requiredToolsAnyOf:
|
|
# Inspecting the app's files is satisfied by either reading them directly
|
|
# or grepping for the revenue calculation.
|
|
- [read_app_file, search_app]
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- inspects the dashboard app's files to locate the revenue calculation
|
|
- fixes the per-order revenue so it multiplies unit price by quantity
|
|
- leaves the result as an AI draft and does not deploy or save it
|
|
|
|
- id: global-test30-raw-app-small-edit-large
|
|
prompt: |-
|
|
In the dashboard app at `f/evals/global/analytics_dashboard`, change the main
|
|
page heading from "Operations Console" to "Revenue Overview". Leave everything
|
|
else unchanged. Keep it as an AI draft only; do not deploy or save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: app
|
|
path: f/evals/global/analytics_dashboard
|
|
valueIncludes:
|
|
- "Revenue Overview"
|
|
toolExpect:
|
|
requiredToolsAnyOf:
|
|
# Inspecting the app's files is satisfied by reading them directly or
|
|
# grepping for the target with search_app.
|
|
- [read_app_file, search_app]
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- renames the main page heading to Revenue Overview
|
|
- does not change other dashboard behavior
|
|
- leaves the result as an AI draft only
|
|
|
|
- id: global-test31-raw-app-debug-inspect-data
|
|
prompt: |-
|
|
The raw app dashboard at `f/evals/global/analytics_dashboard` is reporting
|
|
revenue totals that look too low. Inspect the app's files — both the sample
|
|
order data module and the revenue calculation — to work out whether the bug is
|
|
in the data or in the calculation, then fix the actual cause. Keep the change as
|
|
an AI draft only; do not deploy or save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard
|
|
runtime:
|
|
maxTurns: 22
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: app
|
|
path: f/evals/global/analytics_dashboard
|
|
valueIncludes:
|
|
- "return order.unitPrice * order.quantity"
|
|
toolExpect:
|
|
requiredToolsAnyOf:
|
|
# Inspecting the app's files is satisfied by reading them directly or
|
|
# grepping for the target with search_app.
|
|
- [read_app_file, search_app]
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- inspects both the sample order data module and the revenue aggregation logic
|
|
- identifies the per-order revenue bug and fixes it to multiply unit price by quantity
|
|
- leaves the result as an AI draft only
|
|
|
|
- id: global-test32-raw-app-cross-file-consistency
|
|
prompt: |-
|
|
The raw app dashboard at `f/evals/global/analytics_dashboard` shows revenue
|
|
totals that disagree between the Revenue Summary tile, the orders table, and the
|
|
regional breakdown. Investigate how each of those computes revenue, work out
|
|
which calculation is wrong, and fix it. Keep the change as an AI draft only; do
|
|
not deploy or save it.
|
|
# Cross-file investigation: forces the model through several overlapping files
|
|
# (the summary's aggregation helper, the orders table, the regional breakdown) —
|
|
# a realistic multi-file read load that exercises the read_app_file cap.
|
|
initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard
|
|
runtime:
|
|
maxTurns: 24
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: app
|
|
path: f/evals/global/analytics_dashboard
|
|
valueIncludes:
|
|
- "return order.unitPrice * order.quantity"
|
|
toolExpect:
|
|
requiredToolsAnyOf:
|
|
# Inspecting the app's files is satisfied by reading them directly or
|
|
# grepping for the target with search_app.
|
|
- [read_app_file, search_app]
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- inspects the revenue calculation behind the summary tile, the orders table, and the regional breakdown
|
|
- identifies that the per-order revenue helper omits quantity and fixes it to multiply unit price by quantity
|
|
- leaves the result as an AI draft only
|
|
|
|
- id: global-test33-raw-app-rename-across-files
|
|
prompt: |-
|
|
In the dashboard app at `f/evals/global/analytics_dashboard`, rename the
|
|
`formatCurrency` helper to `formatMoney` everywhere it is defined, imported, and
|
|
called. Leave the separate `formatCurrencyPrecise` helper exactly as it is. Keep
|
|
the change as an AI draft only; do not deploy or save it.
|
|
# Find-all-usages rename: formatCurrency is defined once and called in 6 places
|
|
# spread across 4 component files (and imported in 4). Locating every usage is the
|
|
# exact task search_app is meant to make cheap — one grep returns all file:line
|
|
# rows instead of reading each component whole. valueExcludes "formatCurrency("
|
|
# asserts the definition and all call sites were renamed while tolerating the
|
|
# preserved formatCurrencyPrecise (which is never followed by "(").
|
|
initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard
|
|
runtime:
|
|
maxTurns: 22
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: app
|
|
path: f/evals/global/analytics_dashboard
|
|
valueIncludes:
|
|
- "export function formatMoney"
|
|
- "formatMoney("
|
|
valueExcludes:
|
|
- "formatCurrency("
|
|
toolExpect:
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- renames the formatCurrency definition, imports, and all call sites to formatMoney
|
|
- leaves the unrelated formatCurrencyPrecise helper unchanged
|
|
- leaves the result as an AI draft only
|
|
|
|
# --- Path selection (u/<user> vs f/<folder>) ---
|
|
# These cases assert how the assistant picks a workspace path when the user gives
|
|
# none: a bare name defaults to the personal scope `u/<user>/`, an existing folder
|
|
# whose purpose matches is used, a non-admin targets a writable folder and never a
|
|
# read-only one, and shared intent with no matching folder asks rather than invents.
|
|
# Each depends on the seeded `user` fixture (username / is_admin / folders /
|
|
# folders_read) so the prompt's folder guidance and `u/{username}` are well-formed.
|
|
|
|
- id: global-path1-bare-name-defaults-to-personal
|
|
prompt: |-
|
|
Stage a quick draft helper that takes a string and returns it trimmed of
|
|
leading and trailing whitespace. Just keep it as a draft.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
pathStartsWith: u/admin/
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_script
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- stages a single script draft for a trim helper
|
|
- defaults the path to the current user's personal scope (u/admin/...) since no path or folder was given
|
|
- does not invent an f/<folder> path
|
|
- leaves the result as a draft only
|
|
|
|
- id: global-path2-match-existing-folder
|
|
prompt: |-
|
|
Draft a flow for the marketing team's weekly campaign report.
|
|
It should take a week number and return a short summary string.
|
|
Keep it as a draft only.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_folders.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: flow
|
|
pathStartsWith: f/marketing/
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_flow
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- drafts a flow for the marketing campaign report
|
|
- places it in the existing marketing folder (f/marketing/...) rather than the personal scope or an invented folder
|
|
- leaves the result as a draft only
|
|
|
|
- id: global-path3-shared-intent-unknown-folder-asks
|
|
prompt: |-
|
|
Put together a draft onboarding checklist flow for the People Ops team to use
|
|
when a new hire joins. Keep it as a draft.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_folders.json
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- askUserQuestion
|
|
forbiddenToolsUsed:
|
|
- write_flow
|
|
- write_script
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- recognizes the request implies shared/team work but names no existing folder (none of marketing/data_engineering/shared_utils fit People Ops)
|
|
- asks which folder to use instead of guessing or inventing one
|
|
- does not create a draft until the folder is known
|
|
|
|
- id: global-path4-nonadmin-avoids-readonly-folder
|
|
prompt: |-
|
|
Draft a small flow that returns today's date as an ISO string, and stage it in
|
|
one of our shared team folders. Keep it as a draft.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_bob_nonadmin_teams.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: flow
|
|
pathStartsWith: f/team_a/
|
|
forbiddenDrafts:
|
|
- type: flow
|
|
pathStartsWith: f/team_b/
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_flow
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- drafts a flow that returns the current date as an ISO string
|
|
- places it in team_a (writable by this non-admin user) and not team_b (read-only)
|
|
- leaves the result as a draft only
|
|
|
|
- id: global-test-pipeline-create-node
|
|
prompt: |-
|
|
Set up the first step of a data pipeline at `f/evals/global/orders_ingest`.
|
|
On a schedule, it should pull raw orders and land them in a managed DuckLake
|
|
table so later steps can build on it. Keep it as an AI draft only — don't
|
|
deploy or save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
path: f/evals/global/orders_ingest
|
|
valueIncludes:
|
|
- pipeline
|
|
forbiddenDrafts:
|
|
- type: flow
|
|
pathStartsWith: f/evals/global/
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_script
|
|
forbiddenToolsUsed:
|
|
- write_flow
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
# A pipeline node is DECLARATIVE: triggers are declared by `-- on <ref>`
|
|
# annotations (the trigger row is created separately) and a `-- materialize`
|
|
# output is a MANAGED write where the body is a bare SELECT that the runtime
|
|
# wraps in the create/replace. Do not expect a separate trigger config or a
|
|
# hand-written CREATE TABLE / INSERT — those would be wrong for a materialize node.
|
|
- builds a data pipeline node as a script (not a flow)
|
|
- marks the script as a pipeline member with the pipeline annotation in the script's comment syntax (`-- pipeline` for a DuckDB/SQL node, not `// pipeline`)
|
|
- declares the schedule trigger with the `-- on schedule` annotation comment (this annotation is the correct and complete way a pipeline node binds a schedule; no separate trigger configuration is expected)
|
|
- declares the managed DuckLake output with `-- materialize ducklake://<table>` and writes the body as a bare SELECT (materialize is a managed write, so the node correctly does NOT hand-write its own CREATE TABLE / INSERT)
|
|
- leaves the result as an AI draft and does not deploy or save it
|
|
|
|
- id: global-test-pipeline-two-node-chain
|
|
prompt: |-
|
|
Build a small data pipeline in the `f/evals/global` folder: one step that
|
|
ingests orders into a DuckLake table, and a second step that reads that table
|
|
and writes a daily order-count rollup table. Wire the second step to run off
|
|
the first step's output. Keep everything as drafts — don't deploy.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 14
|
|
validate:
|
|
draftCountAtLeast: 2
|
|
requiredDrafts:
|
|
- type: script
|
|
pathStartsWith: f/evals/global/
|
|
valueIncludes:
|
|
- pipeline
|
|
- ducklake
|
|
forbiddenDrafts:
|
|
- type: flow
|
|
pathStartsWith: f/evals/global/
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_script
|
|
forbiddenToolsUsed:
|
|
- write_flow
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
# Pipeline nodes are declarative: `-- on <ref>` binds inputs/triggers and
|
|
# `-- materialize ducklake://<table>` is a managed write whose body is a bare
|
|
# SELECT. Do not expect hand-written CREATE TABLE / INSERT on a materialize node.
|
|
- creates two data pipeline nodes as scripts (not a flow) in f/evals/global
|
|
- both scripts carry the pipeline annotation in their comment syntax (`-- pipeline` for DuckDB/SQL nodes, not `// pipeline`)
|
|
- the first ingests orders into a DuckLake table (a `-- materialize ducklake://<table>` output with a bare SELECT body is correct; no hand-written CREATE TABLE / INSERT is expected)
|
|
- the second reads that same table via `-- on ducklake://<that-table>` and materializes a daily rollup table, wiring it to the first step's output asset
|
|
- leaves both as AI drafts without deploying
|
|
|
|
- id: global-test-pipeline-complex-incremental
|
|
prompt: |-
|
|
Build a data pipeline in the `f/evals/global` folder for our web shop's
|
|
orders. It has three steps:
|
|
1. On a schedule, ingest the raw order CSVs under `s3://raw/orders/` into a
|
|
managed DuckLake table.
|
|
2. An incremental daily rollup: read that raw orders table and, on each run,
|
|
append just the current day's order count and total revenue into a second
|
|
DuckLake table. It should process one day at a time, not rebuild the whole
|
|
table every run.
|
|
3. A final step that reads the daily rollup table and exports the latest data
|
|
as a Parquet file to `s3://reports/` for the BI team.
|
|
Wire each step to the previous step's output so they form one pipeline. Keep
|
|
everything as AI drafts — don't deploy or save.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 18
|
|
validate:
|
|
draftCountAtLeast: 3
|
|
requiredDrafts:
|
|
- type: script
|
|
pathStartsWith: f/evals/global/
|
|
valueIncludes:
|
|
- pipeline
|
|
- ducklake
|
|
forbiddenDrafts:
|
|
- type: flow
|
|
pathStartsWith: f/evals/global/
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_script
|
|
forbiddenToolsUsed:
|
|
- write_flow
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
# Pipeline nodes are declarative: `-- on <ref>` binds inputs/triggers, and a
|
|
# DuckLake `-- materialize` output is a managed write whose body is a bare SELECT
|
|
# (the runtime performs the create/replace/append/merge). Do not expect a
|
|
# separate trigger config or hand-written CREATE TABLE / INSERT on a
|
|
# materialize node. S3/Parquet output is NOT materialize: the body writes it.
|
|
- builds the pipeline as three independent scripts (not a flow) in f/evals/global
|
|
- every node carries the pipeline annotation in its own comment syntax (`-- pipeline` for DuckDB/SQL nodes, not `// pipeline`)
|
|
- step 1 binds a schedule with `-- on schedule` and declares a managed DuckLake output with `-- materialize ducklake://<table>` and a bare SELECT body (no separate trigger config or hand-written CREATE TABLE is expected)
|
|
- "step 2 is incremental: each run adds only that day's rows to a second DuckLake table rather than rebuilding the whole table every run (e.g. an `append` or `key=<col>` merge materialize mode, not a full replace). Selecting the day via the `-- partitioned daily` + `{partition}` / `wm_partition(...)` idiom is the idiomatic form, but an equivalent current-day filter also satisfies this; a full-refresh/replace of the whole table does not"
|
|
- step 2 reads the same DuckLake table step 1 writes (via `-- on ducklake://<that-table>`), wiring it to step 1's output asset
|
|
- step 3 reads the daily rollup table and exports it as a Parquet file to S3
|
|
- does not misuse `-- materialize` for the S3 Parquet export (materialize is DuckLake-only; the S3 output is written by the script body, e.g. a DuckDB COPY or an SDK write)
|
|
- leaves all three nodes as AI drafts without deploying or saving
|
|
|
|
- id: global-path5-create-folder-then-draft
|
|
prompt: |-
|
|
Create a new shared folder called "analytics" for our data work, then draft a
|
|
script in it that returns the current timestamp as an ISO string. Keep the
|
|
script as a draft.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
pathStartsWith: f/analytics/
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- create_folder
|
|
- write_script
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- creates a new shared folder named "analytics" via create_folder
|
|
- drafts a script placed in that folder (f/analytics/...) returning an ISO timestamp
|
|
- leaves the script as a draft only
|
|
|
|
- id: global-artifact-plan-create
|
|
prompt: |-
|
|
I'm about to build a customer onboarding flow, but first I want a short written plan I can review and iterate on before any code.
|
|
Draft a markdown plan with a title, a one-sentence summary, and three or four bullet steps.
|
|
Keep it as something I can reopen and revise later — don't build the flow itself yet.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json
|
|
runtime:
|
|
maxTurns: 6
|
|
sessionChat: true
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- create_artifact
|
|
forbiddenToolsUsed:
|
|
- write_flow
|
|
- write_script
|
|
- deploy_workspace_item
|
|
judgeChecklist:
|
|
- saves the plan as a markdown artifact via create_artifact rather than only replying inline
|
|
- the artifact content has a title, a one-line summary, and three or four bullet steps for onboarding
|
|
- the artifact is registered as the session's plan (role "plan"), not as an ordinary note - the user asked for the plan they will come back to and revise
|
|
- does not create a flow or script draft yet
|
|
|
|
- id: global-planmode1-hands-over-a-plan
|
|
prompt: |-
|
|
Our support inbox is a mess. I want incoming emails triaged by urgency and routed to the
|
|
right team, with anything urgent also posted to Slack.
|
|
Work out how you'd build this in Windmill.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 10
|
|
sessionChat: true
|
|
planMode: true
|
|
# No draft assertion: approving the plan opens the gate mid-run, and building from there is
|
|
# what production asks for, so a draft is not a failure. The gate itself is covered by
|
|
# shared.test.ts; what only a real model can show is whether it researches and hands over a
|
|
# usable plan instead of guessing at one.
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- exit_plan_mode
|
|
# Not "saves the plan as an artifact": exit_plan_mode writes it, so the harness would
|
|
# satisfy that on every run the tool is called at all — it grades itself, not the model.
|
|
judgeChecklist:
|
|
- the plan covers classifying an incoming email by urgency, routing it to a team, and posting urgent ones to Slack
|
|
- the plan is specific about what would be built in Windmill (a flow and its steps, or the scripts involved)
|
|
|
|
- id: global-planmode2-sketches-while-planning
|
|
prompt: |-
|
|
We're moving our nightly CSV export off a schedule and onto a webhook the vendor calls
|
|
when their file is ready. Work out how you'd rebuild it in Windmill — and draw me the
|
|
shape of it before you write anything, I find that easier to react to than prose.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 10
|
|
sessionChat: true
|
|
planMode: true
|
|
# Both tools, because either alone is a different behaviour: create_artifact without
|
|
# exit_plan_mode means the model filed the plan as the drawing, and exit_plan_mode without
|
|
# create_artifact means the posture blocked the drawing it was asked for.
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- create_artifact
|
|
- exit_plan_mode
|
|
judgeChecklist:
|
|
- saves a diagram of the proposed design as an artifact rather than only describing it in chat
|
|
- the diagram covers the webhook that starts the run and the steps that replace the nightly schedule
|
|
- hands the plan over with exit_plan_mode instead of leaving it in the artifact
|
|
- does not register the diagram as the session's plan document
|
|
|
|
- id: global-npm1-script-search-package
|
|
prompt: |-
|
|
Find a good npm package for parsing RSS/Atom feeds and use it to create a draft Bun script
|
|
at `f/evals/global/fetch_feed`. The script should take a string `url` input and return the
|
|
feed title along with the titles of the 5 most recent items. Tell me which package you
|
|
picked and link its documentation. Leave it as an AI draft only; do not deploy.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 12
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
path: f/evals/global/fetch_feed
|
|
language: bun
|
|
valueIncludes:
|
|
- url
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- search_npm_packages
|
|
- write_script
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
- searches npm for a feed-parsing package before writing the code
|
|
- the draft Bun script imports the chosen npm package rather than hand-rolling an XML/RSS parser
|
|
- the script takes a url input and returns the feed title plus the recent item titles
|
|
- the reply names the chosen package and links its documentation
|
|
- the result stays as an AI draft and is not deployed
|
|
|
|
- id: global-dbschema1-postgres-resource-tables
|
|
prompt: |-
|
|
I have a postgres resource at f/data/reports_pg in this workspace.
|
|
What tables does that database have?
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json
|
|
runtime:
|
|
maxTurns: 12
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- get_db_schema
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- write_resource
|
|
- test_run_script
|
|
- deploy_workspace_item
|
|
toolCallArgs:
|
|
- tool: get_db_schema
|
|
field: resourcePath
|
|
stringIncludesAnyOf:
|
|
- f/data/reports_pg
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- fetches the schema through get_db_schema with the resource path f/data/reports_pg
|
|
- when the lookup fails, tells the user instead of inventing table names
|
|
- does not write scripts or resources to answer a read-only question
|
|
|
|
# --- Dedicated tools for workspace, run and instance state ---
|
|
# The harness serves the worker read itself (a benchmark fetch handler in
|
|
# adapters/frontend). The stale `api-catalog` in the id below is kept so
|
|
# results stay comparable across benchmark runs.
|
|
|
|
- id: global-test30-api-catalog-workers
|
|
prompt: |-
|
|
Are any workers online right now? Give me a quick summary of the workers and when they last pinged.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- list_workers
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- deploy_workspace_item
|
|
# Read-only workspace inspection produces no draft; validate via tool use.
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- reads worker state through list_workers instead of guessing or fabricating
|
|
- reports worker status from the returned data
|
|
|
|
- id: global-test37-ducklake-declared-measure
|
|
prompt: |-
|
|
We track orders in the main ducklake. Write me a duckdb script that reports total
|
|
revenue by month. Keep it as a draft, don't deploy it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/ducklake_orders_metrics.json
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 1
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- list_data_metrics
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
# The judge runs: the point is not that the tool was called but that the number it
|
|
# describes is the declared one. `revenue` excludes test rows, so an aggregate that
|
|
# reproduces it without the filter is plausible, runnable and wrong.
|
|
judgeChecklist:
|
|
- totals revenue with the declared sum over the amount column rather than an invented aggregate over a guessed column
|
|
- excludes test orders from the total, as the declared revenue measure does
|
|
- groups by month using the declared order_month expression over order_date
|
|
- does not introduce column names absent from the declarations
|
|
|
|
- id: global-test31-draft-test-run-not-deployed
|
|
prompt: |-
|
|
Update `f/evals/global/format_greeting` so the provided name is uppercased in the greeting, then run it with name "ada" to check it works.
|
|
Keep it as an AI draft only; do not deploy or save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
path: f/evals/global/format_greeting
|
|
language: bun
|
|
valueIncludes:
|
|
- toUpperCase
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- test_run_script
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
# The judge only sees the drafts artifact and cannot observe runs, so it always
|
|
# docks the prompt's "run it" requirement — validate deterministically instead:
|
|
# draft content via valueIncludes, the test run via toolExpect.
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- creates an AI draft of f/evals/global/format_greeting with the name uppercased in the greeting
|
|
- does not deploy or save the draft
|
|
|
|
- id: global-test32-delete-via-dedicated-tool
|
|
prompt: |-
|
|
Delete the deployed script `f/evals/global/send_report_digest` from the workspace — we don't use it anymore.
|
|
initial: ai_evals/fixtures/frontend/global/initial/report_digest_script.json
|
|
runtime:
|
|
maxTurns: 6
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- delete_workspace_item
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
# Deletion produces no draft; validate via tool use.
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- deletes the deployed script via delete_workspace_item
|
|
|
|
- id: global-test33-run-deployed-script-with-form
|
|
prompt: |-
|
|
Run the deployed script `f/evals/global/format_greeting` for me with the name "ada".
|
|
initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json
|
|
runtime:
|
|
maxTurns: 8
|
|
# A session chat is where the run card has a preview pane beside it; run_script
|
|
# itself is offered in every chat.
|
|
sessionChat: true
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- run_script
|
|
# A draft may declare different arguments than the deployed version being run, so
|
|
# the names to prefill have to come from the deployed schema.
|
|
- read_workspace_item
|
|
forbiddenToolsUsed:
|
|
- test_run_script
|
|
- write_script
|
|
- deploy_workspace_item
|
|
# An empty form pushes the work back onto the user, so the prefill is part of
|
|
# what the tool is for.
|
|
toolCallArgs:
|
|
- tool: run_script
|
|
field: args.name
|
|
stringIncludesAnyOf:
|
|
- ada
|
|
# Running produces no draft, and the judge cannot observe runs; validate via tool use.
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- runs the deployed script through run_script rather than a preview test run
|
|
- passes the name "ada" so the confirmation form comes up prefilled
|
|
|
|
- id: global-test34-run-with-secret-from-variable
|
|
prompt: |-
|
|
Run the deployed `f/evals/global/billing_sync` for the account `acme` — use the billing
|
|
API token we already keep in the workspace.
|
|
initial: ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json
|
|
runtime:
|
|
maxTurns: 10
|
|
# A session chat is where the run card has a preview pane beside it; run_script
|
|
# itself is offered in every chat.
|
|
sessionChat: true
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- run_script
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- deploy_workspace_item
|
|
# A secret argument is filled by naming the variable that holds it: the value stays in
|
|
# the variable and only its path travels. A literal reaches the job as a reference too,
|
|
# minted on the way in, but it stays in the tool call the model emitted.
|
|
toolCallArgs:
|
|
- tool: run_script
|
|
field: args.api_token
|
|
stringIncludesAnyOf:
|
|
- "$var:f/evals/global/stripe_api_token"
|
|
- tool: run_script
|
|
field: args.account
|
|
stringIncludesAnyOf:
|
|
- acme
|
|
# Running produces no draft, and the judge cannot observe runs; validate via tool use.
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- fills the secret argument with a reference to the existing workspace variable rather than a literal token
|
|
- passes the account "acme"
|
|
- does not invent or guess the token's value
|
|
|
|
- id: global-test35-run-deployed-flow-with-form
|
|
prompt: |-
|
|
Run the deployed flow `f/evals/global/notify_customer` for me — the customer is `acme`.
|
|
initial: ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json
|
|
runtime:
|
|
maxTurns: 8
|
|
# A session chat is where the run card has a preview pane beside it; run_flow
|
|
# itself is offered in every chat.
|
|
sessionChat: true
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- run_flow
|
|
# A draft may declare different arguments than the deployed version being run, so
|
|
# the names to prefill have to come from the deployed schema.
|
|
- read_workspace_item
|
|
forbiddenToolsUsed:
|
|
- test_run_flow
|
|
- write_flow
|
|
- deploy_workspace_item
|
|
# An empty form pushes the work back onto the user, so the prefill is part of
|
|
# what the tool is for.
|
|
toolCallArgs:
|
|
- tool: run_flow
|
|
field: args.customer
|
|
stringIncludesAnyOf:
|
|
- acme
|
|
# Running produces no draft, and the judge cannot observe runs; validate via tool use.
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- runs the deployed flow through run_flow rather than a preview test run
|
|
- passes the customer "acme" so the confirmation form comes up prefilled
|
|
|
|
- id: global-test36-draft-flow-test-run-not-deployed
|
|
prompt: |-
|
|
Update the `calculate_total` step of `f/evals/global/process_invoice` so it applies 8% tax and
|
|
returns `subtotal`, `tax` and `total`, then run it to check it works.
|
|
Keep it as an AI draft only; do not deploy or save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: flow
|
|
path: f/evals/global/process_invoice
|
|
toolExpect:
|
|
# A one-step flow is as well checked by running the step as the whole flow, so both
|
|
# count: what matters is that the run is against the draft.
|
|
requiredToolsAnyOf:
|
|
- [test_run_flow, test_run_step]
|
|
# The draft is what the user asked to check, and run_flow would run the deployed
|
|
# version instead — the edit would not be in what ran.
|
|
forbiddenToolsUsed:
|
|
- run_flow
|
|
- deploy_workspace_item
|
|
# The judge cannot observe runs, and the edit's content is already pinned by
|
|
# global-test5 on this fixture; what this case guards is where the run went.
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- creates an AI draft of f/evals/global/process_invoice applying 8% tax
|
|
- does not deploy or save the draft
|
|
|
|
- id: global-test38-chat-flow-follow-up-same-conversation
|
|
prompt: |-
|
|
I want to check that my support chat flow `f/evals/global/support_chat` remembers what was said.
|
|
Test it: first send "My name is Ada", then send "What is my name?" as a follow-up in the same chat.
|
|
initial: ai_evals/fixtures/frontend/global/initial/support_chat_flow.json
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- test_run_flow
|
|
# A chat flow's memory lives in its conversation, so a follow-up only reaches the first
|
|
# turn's history when both test runs name the same conversation.
|
|
toolCallArgs:
|
|
- tool: test_run_flow
|
|
field: memory_id
|
|
sharedByAtLeast: 2
|
|
forbiddenToolsUsed:
|
|
- run_flow
|
|
- deploy_workspace_item
|
|
# The judge cannot observe runs; what this case guards is the conversation the runs share.
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- test-runs the chat flow twice, the second message as a follow-up in the first run's conversation
|
|
|
|
- id: global-undo-created-draft
|
|
prompt: |-
|
|
Create a draft Postgres resource at `u/admin/scratch_db` for host db.example.com port 5432, database `orders`, user `app`, and tell me what fields it ended up with.
|
|
Once you've shown me that, delete it from the workspace again — I only wanted to see the shape of it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_resource
|
|
- discard_local_draft
|
|
forbiddenToolsUsed:
|
|
- delete_workspace_item
|
|
- deploy_workspace_item
|
|
# Undoing a draft leaves no draft behind; validate via tool use.
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- undoes its own never-deployed resource with discard_local_draft rather than delete_workspace_item
|
|
|
|
- id: global-draft-diff-report
|
|
prompt: |-
|
|
Update the existing workspace script at `f/evals/global/format_greeting` so the returned message ends with an exclamation mark, keeping everything else the same.
|
|
Then show me exactly what your draft changes compared to the deployed version.
|
|
Leave the result as an AI draft only; do not deploy or save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
path: f/evals/global/format_greeting
|
|
language: bun
|
|
valueIncludes:
|
|
- "!"
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- diff
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: diff
|
|
field: path
|
|
stringIncludesAnyOf:
|
|
- f/evals/global/format_greeting
|
|
judgeChecklist:
|
|
- creates an AI draft for the existing f/evals/global/format_greeting script with the exclamation-mark change
|
|
- the draft changes only the returned message's punctuation — summary, language, path, and the rest of the code are untouched
|
|
- does not deploy or save the draft to the workspace
|
|
|
|
- id: global-resource-manual-credentials
|
|
prompt: |-
|
|
Set up a resource for our production Postgres database at `f/evals/global/prod_db` (host db.internal.example.com, port 5432, database `orders`, user `app`).
|
|
I don't want to paste the password into this chat — prepare everything so I can enter it myself.
|
|
Leave it as an AI draft only; do not deploy or save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 12
|
|
validate:
|
|
draftCountAtLeast: 1
|
|
requiredDrafts:
|
|
- type: resource
|
|
path: f/evals/global/prod_db
|
|
valueIncludes:
|
|
- db.internal.example.com
|
|
- orders
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_resource
|
|
- open_page
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
# The model may land the user in the resource's drawer or in the drawer of
|
|
# the secret variable it created for the password — both are correct.
|
|
- tool: open_page
|
|
field: page
|
|
stringIncludesAnyOf:
|
|
- resources
|
|
- variables
|
|
- tool: open_page
|
|
field: open
|
|
stringIncludesAnyOf:
|
|
- prod_db
|
|
- password
|
|
judgeChecklist:
|
|
- creates a postgres resource draft at f/evals/global/prod_db with the provided host, port, database, and user
|
|
- the password is left for the user to provide (empty, a placeholder, or a secret variable reference) — no invented password value presented as real
|
|
- does not deploy or save anything to the workspace
|
|
|
|
- id: global-test29-email-trigger-draft
|
|
prompt: |-
|
|
Set up a draft auto-reply job.
|
|
Create a Bun script at `f/evals/global/email_pong` that returns the string "pong".
|
|
Then set it up so it runs whenever an email is received at the inbox `pong`.
|
|
Leave everything as AI drafts only; do not deploy or save anything to the workspace.
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
requiredDrafts:
|
|
- type: script
|
|
path: f/evals/global/email_pong
|
|
language: bun
|
|
valueIncludes:
|
|
- pong
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_script
|
|
- write_trigger
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
# "Runs when an email is received" must resolve to the native email trigger kind,
|
|
# never a faked HTTP webhook. Assert the recorded tool-call kind (not the draft):
|
|
# it holds even on a CE backend where email trigger routes (smtp+private) 404.
|
|
- tool: write_trigger
|
|
field: kind
|
|
stringIncludesAnyOf:
|
|
- email
|
|
skipJudge: true
|
|
|
|
# `List Pages` takes a cursor without ever following one, so where the next cursor
|
|
# comes from (`_links.next`) is stated in the authored metadata and in no script body.
|
|
# Paginating is therefore the one task where reading scripts is not an equal route.
|
|
- id: global-hub4-confluence-paginate-all-pages
|
|
prompt: |-
|
|
I need a script that returns every page in one of my Confluence spaces, not just the
|
|
first batch — it should keep fetching until there are no more.
|
|
Save it as a draft at `f/evals/global/confluence_all_pages` and leave it as an AI
|
|
draft; do not deploy it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 14
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
path: f/evals/global/confluence_all_pages
|
|
valueIncludes:
|
|
- /wiki/
|
|
- Basic
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_script
|
|
# Either route to the integration's conventions is fine; the checklist is where the
|
|
# difference between them has to show up.
|
|
requiredToolsAnyOf:
|
|
- - search_hub_scripts
|
|
- get_hub_integration
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
judgeChecklist:
|
|
- the script loops until the API stops returning a next link, rather than making a single request
|
|
- the next cursor is read from the response's `_links.next` or the Link header, not from an invented field such as `next_cursor`, `nextPageToken` or `offset`
|
|
- it authenticates with HTTP Basic built from the Confluence resource's email and API token rather than a bearer token
|
|
- the script takes a Confluence resource as an input parameter rather than loose credential strings
|
|
- the result stays an AI draft and is not deployed
|
|
|
|
# Unlike the Confluence case, both facts this needs — the {"result"} envelope and
|
|
# PATCH-not-PUT — are visible in ServiceNow's own shipped scripts. That makes it the
|
|
# control for whether the metadata earns its place when the scripts already answer.
|
|
- id: global-hub5-servicenow-assign-incident
|
|
prompt: |-
|
|
Write me a script that assigns an existing ServiceNow incident to a person, looking
|
|
that person up by their email address.
|
|
Save it as a draft at `f/evals/global/servicenow_assign_incident` and leave it as an
|
|
AI draft; do not deploy it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 14
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
path: f/evals/global/servicenow_assign_incident
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_script
|
|
requiredToolsAnyOf:
|
|
- - search_hub_scripts
|
|
- get_hub_integration
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
judgeChecklist:
|
|
- it looks the person up in the sys_user table by email rather than assuming a sys_id was supplied
|
|
- it reads records out of the `result` envelope the ServiceNow API wraps responses in
|
|
- it sets assigned_to to the user's sys_id rather than to the raw email address
|
|
- it updates the incident with a partial update (PATCH) rather than a full replace that would clear the fields it did not send
|
|
- the result stays an AI draft and is not deployed
|
|
|
|
# Outreach is JSON:API. Its content type and payload envelope are both stated in the
|
|
# metadata and visible in the shipped update scripts, so this is a second control.
|
|
- id: global-hub6-outreach-reschedule-task
|
|
prompt: |-
|
|
Write me a script that moves an existing Outreach task to a new due date.
|
|
Save it as a draft at `f/evals/global/outreach_reschedule_task` and leave it as an AI
|
|
draft; do not deploy it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 14
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
path: f/evals/global/outreach_reschedule_task
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_script
|
|
requiredToolsAnyOf:
|
|
- - search_hub_scripts
|
|
- get_hub_integration
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
judgeChecklist:
|
|
- the request sends the JSON:API content type application/vnd.api+json for its body
|
|
- the payload wraps the change as data.type and data.attributes rather than sending bare top-level fields
|
|
- it identifies the task by id inside the payload rather than only in the URL, and treats the id as a number
|
|
- it updates the existing task with PATCH rather than creating a new one
|
|
- the result stays an AI draft and is not deployed
|
|
|
|
# --- Windmill Hub reuse (search_hub_scripts + read_workspace_item on a hub/ path) ---
|
|
# Holded's API is obscure enough that a model writing from memory cannot reproduce
|
|
# its endpoint and `key` auth header — so the draft's fidelity to the published
|
|
# script is what proves the hub content was actually fetched, not guessed.
|
|
- id: global-hub1-reuse-hub-script
|
|
prompt: |-
|
|
I want to email one of my Holded invoices to a customer from Windmill.
|
|
There is already a script for that on the Windmill hub — reuse it instead of writing
|
|
your own, and save it as a draft script at `f/evals/global/holded_send_document`.
|
|
Leave it as an AI draft; do not deploy it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 12
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: script
|
|
path: f/evals/global/holded_send_document
|
|
language: bun
|
|
valueIncludes:
|
|
- api.holded.com/api/invoicing/v1/documents
|
|
# `mailTemplateId` is an optional field of the published script's body
|
|
# that a model writing from memory does not invent, so it is what
|
|
# separates reusing the hub script from re-deriving one that merely
|
|
# hits the same endpoint.
|
|
- mailTemplateId
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- search_hub_scripts
|
|
- read_workspace_item
|
|
- write_script
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
- tool: read_workspace_item
|
|
field: path
|
|
stringIncludesAnyOf:
|
|
- hub/
|
|
judgeChecklist:
|
|
- the draft sends an existing Holded document by email rather than creating one
|
|
- the request targets Holded's document send endpoint, not an invented URL
|
|
- authentication uses Holded's own key header rather than a bearer token
|
|
- the document type, document id, and recipient emails are inputs to the script
|
|
- the result stays an AI draft and is not deployed
|
|
|
|
# Same task as global-hub1 with the hub never mentioned: this is what tells us the
|
|
# model reaches for the hub on its own rather than only when pointed at it.
|
|
- id: global-hub2-checks-hub-unprompted
|
|
prompt: |-
|
|
I want to email one of my Holded invoices to a customer from Windmill.
|
|
Save it as a draft script at `f/evals/global/holded_send_document` and leave it as
|
|
an AI draft; do not deploy it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 12
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
# No language assertion: the Holded script's body is `//native`, so a faithful
|
|
# fork lands on bun or bunnative depending on how literally it is copied.
|
|
- type: script
|
|
path: f/evals/global/holded_send_document
|
|
valueIncludes:
|
|
- api.holded.com/api/invoicing/v1/documents
|
|
- mailTemplateId
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- search_hub_scripts
|
|
- write_script
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
judgeChecklist:
|
|
- the draft sends an existing Holded document by email rather than creating one
|
|
- the request targets Holded's document send endpoint, not an invented URL
|
|
- authentication uses Holded's own key header rather than a bearer token
|
|
- the result stays an AI draft and is not deployed
|
|
|
|
# The fixture hub carries two Baremetrics scripts, neither for annotations, so nothing
|
|
# does what is asked. Its conventions — an `apiKey` resource, bearer auth, the
|
|
# api.baremetrics.com/v1 base — are only knowable by reading one of those near-misses.
|
|
- id: global-hub3-integration-as-example
|
|
prompt: |-
|
|
Write me a script that adds an annotation to my Baremetrics account.
|
|
Save it as a draft at `f/evals/global/baremetrics_create_annotation` and leave it as
|
|
an AI draft; do not deploy it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 14
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
# No language assertion: matching the hub script's own dialect is the point,
|
|
# and Baremetrics' scripts are `//native`, so both bun and bunnative are right.
|
|
- type: script
|
|
path: f/evals/global/baremetrics_create_annotation
|
|
valueIncludes:
|
|
- api.baremetrics.com/v1
|
|
- apiKey
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- search_hub_scripts
|
|
- write_script
|
|
# Reading a near-miss script and reading the integration's own metadata are two
|
|
# routes to the same conventions, so requiring either keeps the case about the
|
|
# outcome rather than about which route the model took.
|
|
requiredToolsAnyOf:
|
|
- - read_workspace_item
|
|
- get_hub_integration
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
judgeChecklist:
|
|
- the script takes a Baremetrics resource as an input parameter rather than a bare API key string
|
|
- the resource type it declares carries an apiKey field, matching how other Baremetrics hub scripts take credentials
|
|
- the request authenticates with a bearer token built from that resource
|
|
- the script posts an annotation rather than reading data back
|
|
- the result stays an AI draft and is not deployed
|
|
|
|
# The value of a secret variable is unreadable, so a metadata-only edit must leave
|
|
# it untouched: passing any `value` here means inventing one, which silently
|
|
# replaces the real secret at deploy.
|
|
- id: global-secret-variable-description-only-edit
|
|
prompt: |-
|
|
The variable `f/evals/global/stripe_api_token` has a confusing description.
|
|
Change it to say it is the Stripe key used by the nightly billing sync, and leave
|
|
everything else about the variable alone. Keep it as an AI draft; do not deploy it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/secret_api_token_variable.json
|
|
runtime:
|
|
maxTurns: 8
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: variable
|
|
path: f/evals/global/stripe_api_token
|
|
valueIncludes:
|
|
# "billing" is already in the seeded description and labels, so it would pass
|
|
# without any edit; "nightly" can only come from the new description.
|
|
- nightly
|
|
- '"is_secret": true'
|
|
valueExcludes:
|
|
# The draft must not carry an invented value, a self-reference, or the
|
|
# real secret it was never shown.
|
|
- '$var:'
|
|
- sk_live_do_not_leak_me
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_variable
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
toolCallArgs:
|
|
# Restating `is_secret: true` is fine; supplying a value is not.
|
|
- tool: write_variable
|
|
field: value
|
|
fieldMustBeAbsent: true
|
|
judgeChecklist:
|
|
- updates the description of f/evals/global/stripe_api_token to mention Stripe and the nightly billing sync
|
|
- keeps the variable secret
|
|
- does not invent, guess, or restate a value for the variable
|
|
# A secret draft stores "" when it stages no new value, and this case must not stage
|
|
# one — the judge has to be told, or it reads the "" as the value having been cleared.
|
|
- 'the draft''s empty value string is expected and correct: a secret variable draft stores "" when no new value is staged, which is exactly right for a description-only edit, so it does NOT mean the value was cleared or changed'
|
|
- leaves the result as an AI draft and does not deploy it
|
|
|
|
# Reproduces GIT-967: a hello-world flow with a React app in front of it. The agent
|
|
# built a backend runnable that fetched the Windmill REST API directly, guessing at
|
|
# WM_TOKEN / BASE_INTERNAL_URL / WM_BASE_URL until it fell back to localhost:8000, and
|
|
# wired the app to a flow that was never deployed. Both are pinned here.
|
|
- id: global-app-triggers-flow
|
|
prompt: |-
|
|
Create a hello world workflow at `f/evals/global/hello_world_flow` that just returns a
|
|
greeting, and a React app at `f/evals/global/hello_world_app` with a button that runs
|
|
that workflow and shows the result on the page.
|
|
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
|
runtime:
|
|
maxTurns: 24
|
|
validate:
|
|
draftCountAtLeast: 2
|
|
requiredDrafts:
|
|
- type: flow
|
|
path: f/evals/global/hello_world_flow
|
|
- type: app
|
|
path: f/evals/global/hello_world_app
|
|
valueIncludes:
|
|
# The frontend must go through the generated bindings, which is the only
|
|
# credentialed way it can reach anything server-side.
|
|
- './wmill'
|
|
valueExcludes:
|
|
# Every artifact of hand-rolling HTTP against the Windmill API from a
|
|
# runnable instead of using the client or a path runnable.
|
|
- BASE_INTERNAL_URL
|
|
- WM_BASE_URL
|
|
- WM_TOKEN
|
|
- localhost:8000
|
|
- getResultMaybe
|
|
- jobs/list
|
|
- getWorkspaceToken
|
|
- getBaseUrl
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- write_flow
|
|
- init_app
|
|
- write_app_runnable
|
|
judgeChecklist:
|
|
- creates a flow at f/evals/global/hello_world_flow that returns a greeting
|
|
- creates a React raw app at f/evals/global/hello_world_app with a button that runs the flow
|
|
- the app actually invokes the flow rather than reimplementing its logic in an inline runnable
|
|
# Both items staying drafts is the CORRECT outcome — the chat must not deploy without
|
|
# being asked. What is judged is that the flow is named as the one item needing a deploy.
|
|
- 'leaving both the flow and the app as drafts is expected and correct: the chat deploys nothing unless asked. Judge only whether the flow is identified as the single item that will need deploying, and that the app is NOT presented as needing deployment to be tried'
|
|
# The judge has repeatedly flagged a correct app as broken over this.
|
|
- '`/wmill.d.ts` is generated by Windmill from the app''s runnables and is deliberately absent from the app''s files — its absence is correct and is NOT a missing-module bug'
|
|
- the app reaches the flow through a backend runnable, not through hand-written HTTP calls to the Windmill API
|
|
- the app's frontend calls the runnable via the generated ./wmill bindings — backend, or backendAsync together with waitJob/getJob/streamJob, all of which are real exports of that module
|
|
- does not read WM_TOKEN, BASE_INTERNAL_URL or WM_BASE_URL, and does not construct a Windmill API URL anywhere
|
|
- does not call windmill-client functions that do not exist, such as getBaseUrl or getWorkspaceToken
|
|
# The point of the case: a path runnable (and wmill.runFlow*) resolves the deployed
|
|
# item, so a flow left as a draft makes the app dead on arrival and the user has to be
|
|
# told. This can't live in judgeChecklist — the global judge only ever sees the drafts,
|
|
# never what the assistant said.
|
|
assistantExpect:
|
|
# Plain substring test: an alternative must name the FLOW and read as an outstanding
|
|
# obligation. Flow-agnostic wording is satisfied by "the app must be deployed"; tense-neutral
|
|
# wording by a deploy the agent only claims to have made. The mirror expectation "don't ask
|
|
# to deploy the app" can't be a forbiddenMentions entry, since correct answers negate it.
|
|
requiredMentionsAnyOf:
|
|
- - deploy the flow
|
|
- deploy that flow
|
|
- deploy this flow
|
|
- deploy just the flow
|
|
- deploy the workflow
|
|
- deploy that workflow
|
|
- deploy hello_world_flow
|
|
- flow must be deployed
|
|
- flow needs to be deployed
|
|
- flow has to be deployed
|
|
- flow needs deploying
|
|
- flow will need to be deployed
|
|
- flow will have to be deployed
|
|
- once the flow is deployed
|
|
- until the flow is deployed
|
|
- workflow must be deployed
|
|
- workflow needs to be deployed
|
|
- workflow has to be deployed
|
|
|
|
# --- AI agent steps: the provider config must name a model the referenced resource serves ---
|
|
# The workspace AI settings are not a runtime gate, so the model id can only come from the
|
|
# resources themselves. Both cases seed AI provider resources; without them the assistant has
|
|
# nothing to write but a guessed id, which is the defect these pin.
|
|
|
|
- id: global-ai-agent-step-uses-workspace-model
|
|
prompt: |-
|
|
Create a draft flow at `f/evals/global/support_answer` with a single AI agent step that answers
|
|
the user's question. The question comes in as a flow input called `query`. No tools.
|
|
Leave it as an AI draft only; do not deploy or save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/ai_provider_anthropic.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: flow
|
|
path: f/evals/global/support_answer
|
|
valueIncludes:
|
|
- aiagent
|
|
- $res:f/evals/global/anthropic_main
|
|
- claude-sonnet-5
|
|
toolExpect:
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
- creates a draft flow at f/evals/global/support_answer with one AI agent step
|
|
- the step's provider is an object carrying kind, resource and model, not a bare resource string
|
|
- the provider references the workspace's own AI provider resource f/evals/global/anthropic_main
|
|
- the model is one the workspace configured for that resource, not an id invented from memory
|
|
- the user's question reaches the agent from the query flow input
|
|
- the result stays as an AI draft and is not deployed
|
|
|
|
- id: global-ai-agent-step-asks-which-provider
|
|
prompt: |-
|
|
Create a draft flow at `f/evals/global/triage_ticket` with a single AI agent step that summarises
|
|
an incoming ticket. The ticket text comes in as a flow input called `ticket`. No tools.
|
|
Leave it as an AI draft only; do not deploy or save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/ai_providers_two.json
|
|
runtime:
|
|
maxTurns: 10
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- askUserQuestion
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
# Two configured providers make the choice the user's: the step may only be written once the
|
|
# question is answered, and it must use one of the workspace's own resources. Which one depends
|
|
# on the answer, so the assertion covers the shared path prefix rather than a fixed resource.
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: flow
|
|
path: f/evals/global/triage_ticket
|
|
valueIncludes:
|
|
- aiagent
|
|
- $res:f/evals/global/
|
|
# Whether the assistant asked before writing lives in the tool record, not in the draft the
|
|
# global judge sees.
|
|
skipJudge: true
|
|
|
|
# Only satisfiable from the resource's own model listing: the workspace AI settings configure just
|
|
# claude-sonnet-5, so an assistant working off the settings has no Opus id to write.
|
|
- id: global-ai-agent-step-uses-listed-model
|
|
prompt: |-
|
|
Create a draft flow at `f/evals/global/deep_review` with a single AI agent step that reviews a
|
|
pull request diff. The diff comes in as a flow input called `diff`. Use the Opus model — this
|
|
one needs the strongest model available. No tools.
|
|
Leave it as an AI draft only; do not deploy or save it.
|
|
initial: ai_evals/fixtures/frontend/global/initial/ai_provider_anthropic.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: flow
|
|
path: f/evals/global/deep_review
|
|
valueIncludes:
|
|
- aiagent
|
|
- $res:f/evals/global/anthropic_main
|
|
- claude-opus-5
|
|
toolExpect:
|
|
forbiddenToolsUsed:
|
|
- deploy_workspace_item
|
|
- delete_workspace_item
|
|
judgeChecklist:
|
|
- creates a draft flow at f/evals/global/deep_review with one AI agent step
|
|
- the step uses the workspace's anthropic resource f/evals/global/anthropic_main
|
|
- the model is the Opus one the user asked for, taken from the models that resource serves
|
|
- the diff flow input reaches the agent
|
|
|
|
# The failure this pins: passing a resource as `{"$res": "<path>"}` (or as a bare path), which
|
|
# reaches the script unresolved because the backend only substitutes a string value that itself
|
|
# starts with `$res:`. The mock preview echoes args back and reports success, so nothing in the
|
|
# loop corrects a wrong shape — the arg form is the whole test.
|
|
- id: global-run-arg-resource-reference
|
|
prompt: |-
|
|
Run `f/evals/global/github_repo_stats` against the `windmill-labs/windmill` repo, passing our
|
|
GitHub credentials at `f/evals/global/github_main` as its `gh_auth` input, and tell me whether
|
|
it went through.
|
|
initial: ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json
|
|
runtime:
|
|
maxTurns: 10
|
|
validate:
|
|
draftCountExactly: 0
|
|
toolExpect:
|
|
requiredToolsUsed:
|
|
- test_run_script
|
|
forbiddenToolsUsed:
|
|
- write_script
|
|
- deploy_workspace_item
|
|
toolCallArgs:
|
|
# Exact: the mock never resolves the reference, so a near-miss path like
|
|
# `$res:f/evals/global/github_main_backup` would otherwise pass.
|
|
- tool: test_run_script
|
|
field: args.gh_auth
|
|
stringEqualsAnyOf:
|
|
- "$res:f/evals/global/github_main"
|
|
# The judge only sees drafts, and this case makes none — the deliverable is the shape of the
|
|
# run argument, checked deterministically above.
|
|
skipJudge: true
|
|
judgeChecklist:
|
|
- runs the existing script rather than rewriting it
|
|
- passes the GitHub resource as the bare string $res:f/evals/global/github_main
|
|
|
|
- id: global-drag-and-drop-app-not-editable
|
|
prompt: |-
|
|
Add a refresh button to the ops console app, and the same to the sales board app.
|
|
initial: ai_evals/fixtures/frontend/global/initial/apps_code_and_drag_and_drop.json
|
|
runtime:
|
|
maxTurns: 12
|
|
validate:
|
|
# One request, two apps, only one of them editable: the code app must come back with a
|
|
# draft and the drag-and-drop one must not. Refusing both, or editing both, fails here —
|
|
# which is what makes this a test of the distinction rather than of caution.
|
|
draftCountExactly: 1
|
|
requiredDrafts:
|
|
- type: app
|
|
path: f/evals/global/ops_console
|
|
forbiddenDrafts:
|
|
- type: app
|
|
path: f/evals/global/sales_board
|
|
toolExpect:
|
|
# Deliberately not constraining write_app_file/patch_app_file by argument: an entry there
|
|
# fails when its tool was never called, so naming both would fail on whichever the model
|
|
# did not pick. The draft assertions above cover the same ground, tool-agnostically.
|
|
forbiddenToolsUsed:
|
|
- init_app
|
|
- deploy_workspace_item
|
|
- delete_app_file
|
|
- delete_app_runnable
|
|
assistantExpect:
|
|
# A refusal leaves no draft for the judge to read, so the explanation is checked here.
|
|
# Only the app kind: substring tests cannot see paraphrase, and every wording of "I can't
|
|
# edit it" defeats a fixed list.
|
|
requiredMentionsAnyOf:
|
|
- - drag-and-drop
|
|
- drag and drop
|
|
- low-code
|
|
- no-code
|
|
skipJudge: true
|