windmill/ai_evals/cases/global.yaml

- id: global-test1-script-create
  prompt: |-
    Create a draft Bun script at `f/evals/global/greet_user`.
    It should take a string `name` input and return `Hello, ${name}!`.
    Leave it as an AI draft only; do not deploy or save it.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        path: f/evals/global/greet_user
        language: bun
        valueIncludes:
          - name
          - Hello
  toolExpect:
    requiredToolsUsed:
      - write_script
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    - creates a Bun script draft at f/evals/global/greet_user
    - the script accepts a name input
    - the script returns a greeting containing Hello, the provided name, and an exclamation mark
    - the result stays as an AI draft and is not deployed or saved to the workspace

- id: global-test2-script-edit-existing
  prompt: |-
    Update the existing workspace script at `f/evals/global/format_greeting`.
    Keep it as a Bun script, but change the greeting so the provided name is uppercased and the returned message ends with an exclamation mark.
    Leave the result as an AI draft only; do not deploy or save it.
  initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        path: f/evals/global/format_greeting
        language: bun
        valueIncludes:
          - toUpperCase
          - "!"
  toolExpect:
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    - creates an AI draft for the existing f/evals/global/format_greeting script
    - preserves the script as Bun
    - uppercases the provided name in the greeting
    - returns a message ending with an exclamation mark
    - does not deploy or save the draft to the workspace

- id: global-test3-flow-create
  prompt: |-
    Create a draft flow at `f/evals/global/sum_numbers`.
    It should take two numeric inputs, `a` and `b`, and return their sum.
    Leave it as an AI draft only; do not deploy or save it.
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: flow
        path: f/evals/global/sum_numbers
        valueIncludes:
          - modules
          - rawscript
          - flow_input.a
          - flow_input.b
  toolExpect:
    requiredToolsUsed:
      - write_flow
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: write_flow
        field: modules
        stringStartsWithAnyOf:
          - "["
  judgeChecklist:
    - creates a flow draft at f/evals/global/sum_numbers
    - the flow accepts numeric inputs a and b
    - the flow returns the sum of a and b
    - the result stays as an AI draft and is not deployed or saved to the workspace

- id: global-test4-multi-artifact-notification-job
  prompt: |-
    Set up a draft stale-trial notification job.
    Create a Bun script at `f/evals/global/check_stale_trials` that accepts `max_age_days`, uses mocked inline trial account data, and returns the stale trial account IDs.
    Also create a weekday 09:00 UTC schedule at `f/evals/global/check_stale_trials_weekday` for that script with `max_age_days` set to 14.
    Add an HTTP POST trigger at `f/evals/global/check_stale_trials_manual` with route path `evals/check-stale-trials` that runs the same script manually.
    Leave everything as AI drafts only; do not deploy or save anything to the workspace.
  runtime:
    maxTurns: 12
  validate:
    draftCountExactly: 3
    requiredDrafts:
      - type: script
        path: f/evals/global/check_stale_trials
        language: bun
        valueIncludes:
          - max_age_days
          - trial
      - type: schedule
        path: f/evals/global/check_stale_trials_weekday
        valueIncludes:
          - f/evals/global/check_stale_trials
          - UTC
          - "14"
      - type: trigger
        triggerKind: http
        path: f/evals/global/check_stale_trials_manual
        valueIncludes:
          - evals/check-stale-trials
          - f/evals/global/check_stale_trials
  toolExpect:
    requiredToolsUsed:
      - write_script
      - write_schedule
      - write_trigger
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    - creates a Bun script draft for stale trial accounts
    - creates a weekday 09:00 UTC schedule draft for the script with max_age_days set to 14
    - creates an HTTP POST trigger draft with route path evals/check-stale-trials for the same script
    - leaves all artifacts as drafts only and does not deploy

- id: global-test5-existing-flow-inline-code-edit
  prompt: |-
    Update the existing flow at `f/evals/global/process_invoice`.
    Only change the `calculate_total` inline code so it applies 8% tax and returns an object containing `subtotal`, `tax`, and `total`.
    Leave the updated flow as an AI draft only; do not deploy or save it.
  initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: flow
        path: f/evals/global/process_invoice
        valueIncludes:
          - calculate_total
          - tax
          - total
  toolExpect:
    requiredToolsUsed:
      - read_workspace_item
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    - reads the existing process_invoice flow before editing it
    - updates the calculate_total inline code to apply 8% tax
    - returns subtotal, tax, and total from the updated flow logic
    - leaves the result as an AI draft only

- id: global-test6-secret-variable-draft
  prompt: |-
    Create a secret variable draft at `f/evals/global/slack_bot_token`.
    Use the placeholder value `xoxb-redacted-test-token` and description `Slack bot token for eval notifications`.
    Do not create any resource or deploy anything.
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: variable
        path: f/evals/global/slack_bot_token
        valueIncludes:
          - Slack bot token
          - "true"
    forbiddenDrafts:
      - type: resource
        path: f/evals/global/slack_bot_token
  toolExpect:
    requiredToolsUsed:
      - write_variable
    forbiddenToolsUsed:
      - write_resource
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: write_variable
        field: value
        stringStartsWithAnyOf:
          - xoxb-redacted-test-token
  skipJudge: true
  judgeChecklist:
    - creates exactly one secret variable draft at f/evals/global/slack_bot_token
    - uses the requested placeholder value and description
    - does not create a resource or deploy anything

# A new app that has to store data, with nothing said about where: the data setup is the
# user's call, so it is asked before anything is created. The question is dismissed here,
# which leaves nothing the model may build on.
- id: global-test7-ambiguous-app-asks-question
  prompt: |-
    Create a new raw app for triaging support tickets.
  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
  runtime:
    maxTurns: 6
    userAnswers: []
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - list_datatables
      - askUserQuestion
    forbiddenToolsUsed:
      - init_app
      - exec_datatable_sql
      - write_app_file
      - write_app_runnable
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: askUserQuestion
        field: question
        stringIncludesAnyOf:
          - data table
          - datatable
          - schema
          - tables
          - store
  skipJudge: true

# The same request with the question answered: the app is built on what the user chose.
- id: global-new-app-data-setup-answer-applied
  prompt: |-
    Create a new raw app for triaging support tickets.
  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
  runtime:
    maxTurns: 32
    userAnswers:
      - A new schema named helpdesk
      - Create new tables
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: app
        valueIncludes:
          - helpdesk
          - react-dom/client
  toolExpect:
    requiredToolsUsed:
      - askUserQuestion
      - init_app
    forbiddenToolsUsed:
      - deploy_workspace_item
    toolCallArgs:
      - tool: init_app
        field: framework
        stringEqualsAnyOf:
          - react19
      - tool: exec_datatable_sql
        field: sql
        stringIncludesAnyOf:
          - helpdesk
  skipJudge: true

# No framework named, no app to follow, nothing to store: the modal's recommended
# framework is used without a question.
- id: global-new-app-defaults-to-react19
  prompt: |-
    Build me a little app that converts temperatures between Celsius and Fahrenheit.
  runtime:
    maxTurns: 12
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: app
        valueIncludes:
          - react-dom/client
  toolExpect:
    requiredToolsUsed:
      - init_app
    forbiddenToolsUsed:
      - askUserQuestion
      - exec_datatable_sql
      - deploy_workspace_item
    toolCallArgs:
      - tool: init_app
        field: framework
        stringEqualsAnyOf:
          - react19
  skipJudge: true

# The user said where the data goes, so there is nothing left to ask.
- id: global-new-app-storage-specified-no-question
  prompt: |-
    Build a todo list app in React. Keep the todos in the main data table, in a new
    schema called todo_app.
  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
  runtime:
    maxTurns: 20
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: app
        valueIncludes:
          - todo_app
  toolExpect:
    requiredToolsUsed:
      - init_app
      - write_app_runnable
    forbiddenToolsUsed:
      - askUserQuestion
      - deploy_workspace_item
    toolCallArgs:
      - tool: exec_datatable_sql
        field: sql
        stringIncludesAnyOf:
          - todo_app
  skipJudge: true

- id: global-test8-human-script-infer-path-language
  prompt: |-
    I need a small helper that formats a customer-facing welcome line.
    It should take a person's name and return "Welcome aboard, <name>!".
    Please just stage it as a draft for now.
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        valueIncludes:
          - Welcome aboard
          - name
  toolExpect:
    requiredToolsUsed:
      - write_script
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    - creates a single script draft for a welcome-line helper
    - accepts a person's name as input
    - returns a message containing Welcome aboard, the provided name, and an exclamation mark
    - chooses a reasonable workspace path and script language without needing the user to specify them
    - leaves the result as an AI draft only

- id: global-test9-human-weekday-trial-job
  prompt: |-
    Can you set up a draft daily job that checks a few hard-coded trial accounts and returns the ones whose trial has ended?
    It should run every weekday morning around 9 in UTC with a 30 day cutoff.
    Keep it as draft work only.
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 2
    requiredDrafts:
      - type: script
        pathIncludes:
          - trial
        valueIncludes:
          - trial
          - "30"
      - type: schedule
        pathIncludes:
          - trial
        valueIncludes:
          - UTC
  toolExpect:
    requiredToolsUsed:
      - write_script
      - write_schedule
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    - creates a script draft that checks hard-coded trial accounts
    - returns the accounts whose trial has ended based on a 30 day cutoff
    - creates a schedule draft for weekday mornings around 09:00 UTC
    - links the schedule to the generated script
    - leaves both artifacts as drafts only

- id: global-test10-human-secret-variable
  prompt: |-
    I need a placeholder Slack bot token stored securely for future notification work.
    Use xoxb-redacted-test-token and note that it is for eval notifications.
    Only prepare a draft.
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: variable
        pathIncludes:
          - slack
        valueIncludes:
          - eval notifications
          - "true"
  toolExpect:
    requiredToolsUsed:
      - write_variable
    forbiddenToolsUsed:
      - write_resource
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: write_variable
        field: value
        stringStartsWithAnyOf:
          - xoxb-redacted-test-token
  skipJudge: true
  judgeChecklist:
    - creates a single secret variable draft for the Slack bot token placeholder
    - uses the requested placeholder value
    - includes a note or description that it is for eval notifications
    - does not create a resource or deploy anything

- id: global-test11-human-existing-flow-informal-edit
  prompt: |-
    There is an invoice processing flow in this workspace.
    Can you adjust its total calculation so it adds 8% tax and returns subtotal, tax, and total?
    Keep the change as a draft.
  initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: flow
        pathIncludes:
          - invoice
        valueIncludes:
          - calculate_total
          - tax
          - total
  toolExpect:
    requiredToolsUsed:
      - read_workspace_item
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    - finds and edits the existing invoice processing flow without the user providing its exact path
    - updates the total calculation to apply 8% tax
    - returns subtotal, tax, and total from the updated flow logic
    - leaves the result as an AI draft only

- id: global-test12-current-live-script-edit
  prompt: |-
    The script I have open formats greetings.
    Can you update this script so it uppercases the name before greeting them and ends with an exclamation mark?
    Keep it as draft work.
  initial: ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        path: f/evals/global/current_greeting
        language: bun
        valueIncludes:
          - toUpperCase
          - "!"
    forbiddenDrafts:
      - type: script
        path: f/evals/global/format_greeting
      - type: script
        path: f/evals/global/format_greeting_archive
  toolExpect:
    requiredToolsUsed:
      - read_workspace_item
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    - resolves "this script" to the active live editor script instead of another similarly named workspace script
    - updates the greeting logic to uppercase the provided name
    - returns a greeting ending with an exclamation mark
    - leaves the result as a draft only

- id: global-test13-current-live-flow-edit
  prompt: |-
    I have the invoice flow open.
    In the current flow, update the total calculation to add 8% tax and return subtotal, tax, and total.
    Keep the change as a draft.
  initial: ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: flow
        path: f/evals/global/current_invoice_flow
        valueIncludes:
          - calculate_total
          - tax
          - total
    forbiddenDrafts:
      - type: flow
        path: f/evals/global/process_invoice
      - type: flow
        path: f/evals/global/process_refund
  toolExpect:
    requiredToolsUsed:
      - read_workspace_item
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    - resolves "current flow" to the active live editor flow
    - does not edit the similarly named deployed invoice or refund flows
    - updates the calculate_total logic to apply 8% tax
    - returns subtotal, tax, and total from the updated flow logic
    - leaves the result as a draft only

- id: global-test14-current-without-live-editor-asks-question
  prompt: |-
    Please update this script so it returns `ok`.
    Keep it as a draft.
  runtime:
    maxTurns: 4
  validate:
    draftCountExactly: 0
  toolExpect:
    forbiddenToolsUsed:
      - write_script
      - edit_script
      - write_flow
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - asks which script to update when the user refers to "this script" without selected or active editor context
    - does not guess a path or create a new script draft

- id: global-test15-human-postgres-resource
  prompt: |-
    I'm wiring the eval reporting database into this workspace.
    Can you stage a Postgres connection for it in the shared evals/global folder?
    Use host `reports-db.internal`, port 5432, database `evals_reporting`, user `report_reader`, and password `pg-redacted-reporting-password`.
    Keep the credentials safe.
    This is just draft work for now.
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 2
    requiredDrafts:
      - type: variable
        pathStartsWith: f/evals/global/
        pathIncludes:
          - evals
          - global
          - report
          - password
        valueIncludes:
          - "true"
          - report
      - type: resource
        pathStartsWith: f/evals/global/
        pathIncludes:
          - evals
          - global
          - report
        valueIncludes:
          - postgres
          - reports-db.internal
          - "5432"
          - evals_reporting
          - report_reader
          - "$var:"
        valueExcludes:
          - pg-redacted-reporting-password
  toolExpect:
    requiredToolsUsed:
      - write_variable
      - search_resource_types
      - write_resource
    forbiddenToolsUsed:
      - write_schedule
      - write_trigger
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: write_variable
        field: value
        stringStartsWithAnyOf:
          - pg-redacted-reporting-password
  skipJudge: true
  judgeChecklist:
    - creates a Postgres resource draft for the eval reporting database
    - creates a secret variable draft for the database password
    - puts the drafts in sensible eval/global reporting-related paths
    - uses the requested host, port, database, and user
    - references the secret variable from the resource instead of embedding the password
    - leaves the work as a draft only

- id: global-test16-human-visible-variable
  prompt: |-
    We keep reusing a 30 day trial cutoff in eval notification jobs.
    Can you stage that as a normal workspace variable in the shared evals/global folder, with a short description so people know what it controls?
    It is not a secret.
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: variable
        pathStartsWith: f/evals/global/
        pathIncludes:
          - evals
          - global
          - trial
        valueIncludes:
          - "30"
          - "false"
          - trial
  toolExpect:
    requiredToolsUsed:
      - write_variable
    forbiddenToolsUsed:
      - write_resource
      - write_schedule
      - write_trigger
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    - creates exactly one non-secret variable draft for the trial cutoff
    - stores the value 30
    - chooses a sensible eval/global path related to trials or notifications
    - includes a useful description of what the value controls
    - does not create resources, schedules, triggers, or deployed workspace changes

- id: global-test17-human-schedule-existing-helper
  prompt: |-
    The workspace already has a report digest helper.
    Can you stage a weekday 8:30 AM UTC run for it with `dry_run` turned on?
    I only want the schedule draft for review.
  initial: ai_evals/fixtures/frontend/global/initial/report_digest_script.json
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: schedule
        pathIncludes:
          - digest
        valueIncludes:
          - f/evals/global/send_report_digest
          - UTC
          - dry_run
          - "true"
  toolExpect:
    requiredToolsUsed:
      - list_workspace_items
      - write_schedule
    forbiddenToolsUsed:
      - write_script
      - write_flow
      - write_resource
      - write_variable
      - write_trigger
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    - finds the existing report digest helper rather than creating a new script or flow
    - creates one schedule draft for that helper
    - schedules it for weekdays around 08:30 UTC
    - passes dry_run as true
    - leaves only the schedule draft for review

- id: global-test29-schedule-with-retry-and-error-handler
  prompt: |-
    The workspace already has a report digest helper.
    Schedule it every morning at 6am UTC with `dry_run` on, and have it retry twice
    if it fails. Run it on our `nightly` worker group.
    Draft only, I'll review before deploying.
  initial: ai_evals/fixtures/frontend/global/initial/report_digest_script.json
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: schedule
        pathIncludes:
          - digest
        valueIncludes:
          - f/evals/global/send_report_digest
          - attempts
          - nightly
  toolExpect:
    requiredToolsUsed:
      - write_schedule
      - get_schedule_schema
    forbiddenToolsUsed:
      - write_script
      - write_flow
      - deploy_workspace_item
  judgeChecklist:
    - finds the existing report digest helper rather than creating a new script or flow
    - creates one schedule draft for that helper running daily around 06:00 UTC
    - configures a retry policy with two attempts
    - routes the schedule to the nightly worker tag
    - leaves the schedule as a draft without deploying

- id: global-test18-human-slack-resource-with-secret
  prompt: |-
    I'm preparing Slack notifications for eval failures.
    Can you stage a Slack connection in the shared evals/global folder?
    The bot token is `xoxb-redacted-test-token`; keep it safe.
    Don't deploy anything yet.
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 2
    requiredDrafts:
      - type: variable
        pathStartsWith: f/evals/global/
        pathIncludes:
          - evals
          - global
          - slack
          - token
        valueIncludes:
          - "true"
      - type: resource
        pathStartsWith: f/evals/global/
        pathIncludes:
          - evals
          - global
          - slack
        valueIncludes:
          - slack
          - "$var:"
        valueExcludes:
          - xoxb-redacted-test-token
  toolExpect:
    requiredToolsUsed:
      - write_variable
      - search_resource_types
      - write_resource
    forbiddenToolsUsed:
      - write_schedule
      - write_trigger
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: write_variable
        field: value
        stringStartsWithAnyOf:
          - xoxb-redacted-test-token
  skipJudge: true
  judgeChecklist:
    - creates a secret variable draft for the Slack bot token placeholder
    - creates a Slack resource draft that references the secret variable instead of embedding the token
    - keeps both drafts under a sensible eval/global Slack-related path
    - does not create schedules, triggers, or deployed workspace changes

- id: global-test19-datatable-not-configured-asks-to-set-up
  prompt: |-
    Here are two newsletter signups: alice@example.com and bob@example.com.
    Save them into a workspace data table for me.
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - list_datatables
    forbiddenToolsUsed:
      - exec_datatable_sql
      - write_script
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - checks which data tables exist in the workspace before acting
    - recognizes that no data table is configured in this workspace
    - explains that a data table must first be set up by the user in the workspace settings (Data Tables) and is not created via SQL
    - does not run SQL, write a script, or invent a data table to work around the missing configuration
    - tells the user to configure a data table and then try again

- id: global-test20-datatable-no-hallucinated-main
  prompt: |-
    Pull the latest rows from the orders table in our data table so I can see recent orders.
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - list_datatables
    forbiddenToolsUsed:
      - exec_datatable_sql
      - write_script
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - checks which data tables exist in the workspace before querying
    - recognizes that no data table is configured in this workspace
    - does not assume a data table named "main" (or any other name) exists
    - does not run SQL against a guessed data table or fabricate order rows
    - tells the user they need to set up a data table in the workspace settings first

- id: global-test21-datatable-list-summarize
  prompt: |-
    What tables do we have in our workspace data table? Just give me the list.
  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - list_datatables
    forbiddenToolsUsed:
      - get_datatable_table_schema
      - exec_datatable_sql
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - lists the tables available in the workspace data table (orders and customers)
    - answers from the data table listing rather than fabricating table names
    - does not fetch column details or run SQL just to produce a table list

- id: global-test22-datatable-inspect-columns
  prompt: |-
    What columns does the orders table have in our workspace data table?
  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - get_datatable_table_schema
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - inspects the orders table schema in the workspace data table
    - reports the orders columns (such as id, customer_id, total, status, created_at)
    - answers from the retrieved schema rather than guessing the columns

- id: global-test23-datatable-query-select
  prompt: |-
    Show me the orders in our workspace data table, including their status and total.
  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - exec_datatable_sql
    forbiddenToolsUsed:
      - write_script
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: exec_datatable_sql
        field: sql
        stringIncludesAnyOf:
          - select
  skipJudge: true
  judgeChecklist:
    - runs a SELECT query against the orders table in the workspace data table
    - reports the orders returned by the query back to the user instead of fabricating data
    - does not tell the user to set up a data table, since one already exists

- id: global-test24-datatable-create-table
  prompt: |-
    Add a new table called refunds to our workspace data table, with an order id and a refund amount.
  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - exec_datatable_sql
    forbiddenToolsUsed:
      - write_script
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: exec_datatable_sql
        field: sql
        stringIncludesAnyOf:
          - create table
  skipJudge: true
  judgeChecklist:
    - creates the refunds table with a plain CREATE TABLE statement on the data table
    - includes an order id and a refund amount column
    - treats creating the table as a normal SQL statement and does not claim a separate registration step is needed
    - does not write a script to create the table

# The same request on a data table that keeps its schema as migrations: the table is
# created by a migration, since plain SQL would leave it out of the history.
- id: global-datatable-schema-change-uses-migration
  prompt: |-
    Add a new table called refunds to our workspace data table, with an order id and a refund amount.
  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - create_datatable_migration
    forbiddenToolsUsed:
      - write_script
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: create_datatable_migration
        field: code_up
        stringIncludesAnyOf:
          - create table
  skipJudge: true
  judgeChecklist:
    - notices that the data table keeps its schema as migrations
    - creates the refunds table through a migration rather than a plain CREATE TABLE statement
    - includes an order id and a refund amount column
    - gives the migration a down that drops the table

# Row data is not schema: it is written with plain SQL even where migrations are enabled.
- id: global-datatable-row-change-skips-migration
  prompt: |-
    Mark order number 2 as shipped in our workspace data table.
  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json
  runtime:
    maxTurns: 12
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - exec_datatable_sql
    forbiddenToolsUsed:
      - create_datatable_migration
      - write_script
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - runs an UPDATE on the orders table setting the status of order id 2 to shipped
    - does not create a migration for a change to row data

- id: global-test25-datatable-mutate-rows
  prompt: |-
    Mark order number 2 as shipped in our workspace data table.
  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
  runtime:
    # Headroom for inspect-schema -> UPDATE -> verify; the in-memory engine now
    # persists the write, so verification confirms on the first try (no loop).
    maxTurns: 12
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - exec_datatable_sql
    forbiddenToolsUsed:
      - write_script
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: exec_datatable_sql
        field: sql
        stringIncludesAnyOf:
          - update
          - insert into
  skipJudge: true
  judgeChecklist:
    - runs an UPDATE on the orders table setting the status of order id 2 to shipped
    - targets only order number 2 rather than rewriting the whole table
    - confirms the change back to the user

- id: global-test26-datatable-script-sdk
  prompt: |-
    Write a script that reads our workspace data table and returns the total revenue across all orders.
    Leave it as an AI draft only; do not deploy or save it.
  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        valueIncludes:
          - wmill.datatable(
  toolExpect:
    requiredToolsUsed:
      - get_instructions
      - write_script
    forbiddenToolsUsed:
      - exec_datatable_sql
      - deploy_workspace_item
      - delete_workspace_item
  # The judge has no datatable SDK reference and wrongly penalizes correct
  # wmill.datatable() tagged-template usage, so rely on the deterministic checks:
  # required get_instructions + write_script, forbidden exec_datatable_sql, and a
  # draft that contains wmill.datatable(.
  skipJudge: true
  judgeChecklist:
    - writes a script (not a chat-time SQL execution) that reads the workspace data table at runtime
    - uses the wmill.datatable() SDK to query the orders table and sum the order totals
    - returns the total revenue from the script
    - leaves the result as an AI draft and does not deploy or save it

- id: global-test27-list-recent-runs
  prompt: |-
    What are the most recent runs in this workspace? Give me a quick rundown of what ran and whether it succeeded.
  initial: ai_evals/fixtures/frontend/global/initial/jobs_seed.json
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - list_runs
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
      - write_script
  # Read-only job inspection produces no draft, so the global judge (which only
  # sees the drafts artifact) would score it empty — validate via tool use.
  skipJudge: true
  judgeChecklist:
    - lists the recent runs from the workspace rather than fabricating them
    - summarizes each run's path and success/failure from the returned data

- id: global-test28-fetch-failed-run-logs
  prompt: |-
    The run with id 01920000-0000-7000-8000-0000000000f1 failed. Pull its logs and tell me what went wrong.
  initial: ai_evals/fixtures/frontend/global/initial/jobs_seed.json
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - get_run
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
      - write_script
    toolCallArgs:
      - tool: get_run
        field: id
        stringIncludesAnyOf:
          - 01920000-0000-7000-8000-0000000000f1
  # Same as above — no draft is produced, so rely on the deterministic tool-use
  # and argument checks rather than the judge.
  skipJudge: true
  judgeChecklist:
    - fetches the logs for the requested job id
    - explains the failure from the returned logs (connection refused to the upstream API)

- id: global-run-args-and-result
  prompt: |-
    What was the run 01920000-0000-7000-8000-0000000000f2 called with, and what did it return?
  initial: ai_evals/fixtures/frontend/global/initial/jobs_seed.json
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - get_run
    forbiddenToolsUsed:
      - test_run_script
      - run_script
      - deploy_workspace_item
    toolCallArgs:
      - tool: get_run
        field: id
        stringIncludesAnyOf:
          - 01920000-0000-7000-8000-0000000000f2
  # Read-only, so no draft for the global judge to score — validated on tool use
  # and the deterministic argument check, like the neighbouring run cases.
  skipJudge: true
  judgeChecklist:
    - reports the arguments the run was called with (region emea, 12 recipients)
    - reports what the run returned (12 sent, 3 skipped)
    - does not start a new run to find out

# --- Page navigation (open_page) ---
# The assistant should take the user to a Windmill page (Runs/Schedules) with the
# right filters via open_page, rather than describing where to click or dumping the
# data. No draft is produced, so the global judge is skipped and we validate the
# tool call and its arguments.

- id: global-openpage1-runs-failed-of-script
  prompt: |-
    Take me to the failed runs of the script at f/evals/global/greet_user so I can see what's going wrong.
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - open_page
    forbiddenToolsUsed:
      - write_script
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: open_page
        field: page
        stringIncludesAnyOf:
          - runs
      - tool: open_page
        field: status
        stringIncludesAnyOf:
          - failure
      - tool: open_page
        field: path
        stringIncludesAnyOf:
          - f/evals/global/greet_user
  skipJudge: true
  judgeChecklist:
    - opens the Runs page filtered to the failed runs of f/evals/global/greet_user
    - applies both the failure status and the script path as filters
    - does not write, deploy, or delete anything

- id: global-openpage2-runs-of-schedule
  prompt: |-
    Open the runs page filtered to the jobs triggered by the schedule f/evals/global/nightly_digest.
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - open_page
    forbiddenToolsUsed:
      - write_script
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: open_page
        field: page
        stringIncludesAnyOf:
          - runs
      - tool: open_page
        field: schedule_path
        stringIncludesAnyOf:
          - f/evals/global/nightly_digest
  skipJudge: true
  judgeChecklist:
    - opens the Runs page filtered to jobs triggered by the f/evals/global/nightly_digest schedule
    - passes the schedule path as the filter
    - does not write, deploy, or delete anything

- id: global-openpage3-open-schedule
  prompt: |-
    Open the schedule f/evals/global/nightly_digest so I can review and edit it.
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - open_page
    forbiddenToolsUsed:
      - write_schedule
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: open_page
        field: page
        stringIncludesAnyOf:
          - schedules
      - tool: open_page
        field: open
        stringIncludesAnyOf:
          - f/evals/global/nightly_digest
  skipJudge: true
  judgeChecklist:
    - opens the Schedules page and targets the f/evals/global/nightly_digest schedule for editing
    - passes the schedule path so the editor opens on it
    - does not write, deploy, or delete anything

- id: global-openpage4-workspace-settings-tab
  prompt: |-
    Take me to the Git sync configuration for this workspace.
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - open_page
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: open_page
        field: page
        stringIncludesAnyOf:
          - workspace_settings
      - tool: open_page
        field: tab
        stringIncludesAnyOf:
          - git_sync
  skipJudge: true
  judgeChecklist:
    - opens the Workspace settings page on the git_sync tab
    - does not write, deploy, or delete anything

- id: global-openpage5-audit-logs-user
  prompt: |-
    Open the audit logs filtered to actions performed by the user admin.
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - open_page
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: open_page
        field: page
        stringIncludesAnyOf:
          - audit_logs
      - tool: open_page
        field: username
        stringIncludesAnyOf:
          - admin
  skipJudge: true
  judgeChecklist:
    - opens the Audit logs page filtered to the admin user
    - does not write, deploy, or delete anything

- id: global-openpage6-triggers-kind
  prompt: |-
    Take me to the Kafka triggers for this workspace.
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - open_page
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: open_page
        field: page
        stringIncludesAnyOf:
          - triggers
      - tool: open_page
        field: trigger_kind
        stringIncludesAnyOf:
          - kafka
  skipJudge: true
  judgeChecklist:
    - opens the Kafka triggers page
    - does not write, deploy, or delete anything

- id: global-draft-does-not-open-compare
  prompt: |-
    Create a TypeScript script draft at f/evals/global/compare_review_demo that returns the string "ok" (no need to test it). I want to look over the change before it gets deployed.
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 1
  toolExpect:
    requiredToolsUsed:
      - write_script
    forbiddenToolsUsed:
      - open_page
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - creates the script draft without opening any workspace page
    - does not deploy or delete anything

- id: global-openpage7-compare-fork-deploy
  prompt: |-
    I'm done with my changes in this fork and want to ship them to the parent workspace myself. Take me there.
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - open_page
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: open_page
        field: page
        stringIncludesAnyOf:
          - compare
  skipJudge: true
  judgeChecklist:
    - opens Compare & Deploy only because the user explicitly wants to deploy the fork into its parent
    - does not deploy or delete anything

- id: global-openpage8-runs-label-and-worker
  prompt: |-
    Show me the runs carrying the label nightly-digest that ran on the worker wk-eval-1.
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - open_page
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
    # One page carrying both filters — two pages each carrying one is not the ask.
    toolCallArgsSameCall:
      - tool: open_page
        args:
          - field: page
            stringIncludesAnyOf:
              - runs
          - field: label
            stringIncludesAnyOf:
              - nightly-digest
          - field: worker
            stringIncludesAnyOf:
              - wk-eval-1
  skipJudge: true
  judgeChecklist:
    - opens the Runs page filtered to the nightly-digest label on worker wk-eval-1
    - does not write, deploy, or delete anything

# Exclusion is the filter shape most easily lost in translation: the page encodes it as
# a `!`-prefixed value, and a model that only knows the positive form silently opens a
# page showing exactly what the user asked to hide.
- id: global-openpage9-runs-exclude-schedules
  prompt: |-
    Open the runs page but hide everything a schedule kicked off — I only care about the rest.
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - open_page
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgsSameCall:
      - tool: open_page
        args:
          - field: page
            stringIncludesAnyOf:
              - runs
          - field: job_trigger_kind
            stringIncludesAnyOf:
              - '!schedule'
  skipJudge: true
  judgeChecklist:
    - opens the Runs page with schedule-triggered jobs excluded
    - does not write, deploy, or delete anything

- id: global-closepage1-close-runs-tab
  prompt: |-
    You just opened the runs page for me in the side panel. Close that tab, I'm done looking at it.
  initial: ai_evals/fixtures/frontend/global/initial/preview_runs_tab.json
  runtime:
    maxTurns: 6
    sessionChat: true
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - close_page
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: close_page
        field: match
        stringIncludesAnyOf:
          - runs
  skipJudge: true
  judgeChecklist:
    - closes the runs preview tab in the side panel
    - does not write, deploy, or delete anything

# --- Artifact version history ---
# Every content change to an artifact is snapshotted, and update_artifact requires a
# change_note that the user reads in the version picker. A blank note makes the history
# unreadable, so pin that the model fills it on every edit.

- id: global-artifact-note-on-each-edit
  prompt: |-
    Write up a short rollout plan for me as a doc I can come back to, covering a staged
    rollout in three phases. Then add a rollback section to it, and after that tighten
    the wording of phase 2.
  runtime:
    maxTurns: 12
    sessionChat: true
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - create_artifact
      - update_artifact
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      # The note is what the version picker shows; a blank one makes history unreadable.
      - tool: update_artifact
        field: change_note
        nonEmpty: true
  skipJudge: true
  judgeChecklist:
    - creates one artifact and revises it rather than creating a second artifact
    - each revision carries a short description of what changed

# A reader who pins an older version in the artifact's picker is looking at something the
# artifact tools never report: an artifact tab carries no ACTIVE PREVIEW section, so the pin
# reaches the chat through get_preview_status alone. Asked what is on screen, the model has
# to read the panel instead of answering from the artifact's own history.

- id: global-artifact-pinned-version-question
  prompt: |-
    Which version of the onboarding plan am I looking at right now?
  initial: ai_evals/fixtures/frontend/global/initial/artifact_onboarding_plan_pinned_v2.json
  runtime:
    maxTurns: 6
    sessionChat: true
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - get_preview_status
    forbiddenToolsUsed:
      - create_artifact
      - update_artifact
      - deploy_workspace_item
  skipJudge: true
  judgeChecklist:
    - answers that the panel is showing version 2, not the latest version 5
    - does not edit or re-create the artifact

# --- Documentation search (search_docs) ---
# Pure product-knowledge questions: the assistant should consult the docs via
# search_docs and answer conversationally, not draft or mutate anything. No
# draft is produced, so the global judge is skipped and we validate tool use.

- id: global-docs-ai-agent-step
  prompt: |-
    Does Windmill support a flow step where an LLM decides which of my scripts to call based on the input?
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - search_docs
    forbiddenToolsUsed:
      - write_script
      - write_flow
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true

- id: global-docs-retry-step
  prompt: |-
    How does automatic retry work for a flow step that calls a flaky API?
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - search_docs
    forbiddenToolsUsed:
      - write_script
      - write_flow
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true

- id: global-docs-key-value-store
  prompt: |-
    Can I use a Redis-style key-value store from my Windmill scripts, and how?
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - search_docs
    forbiddenToolsUsed:
      - write_script
      - write_flow
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true

- id: global-docs-cron-schedule-format
  prompt: |-
    How do Windmill's cron schedules work, and what format does the schedule expression use?
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - search_docs
    forbiddenToolsUsed:
      - write_script
      - write_flow
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true

# --- Raw app on a large project (context-usage benchmark) ---
# These cases run against the deliberately large `analytics_dashboard` raw-app
# fixture (~20 frontend files incl. a 5k-line data module, plus backend runnables).
# They exist to measure how much context the global chat consumes when working in a
# big raw app: test29 is a read-heavy debugging hunt, test30 is a small edit baseline.
# tokenUsage is recorded per run, so the same cases re-run after a read-tool change
# (the read_app_file cap + offset/limit paging) quantify the optimization. skipJudge:
# the judge only sees the drafts artifact and cannot run the app, so we validate
# deterministically.

- id: global-test29-raw-app-debug-large
  prompt: |-
    The analytics dashboard app at `f/evals/global/analytics_dashboard` has a bug:
    the Revenue Summary tile shows a total that is lower than the per-order line
    totals and the per-region breakdown. Track down what is computing revenue
    incorrectly and fix it. Keep the change as an AI draft only; do not deploy or
    save it.
  initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard
  runtime:
    maxTurns: 20
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: app
        path: f/evals/global/analytics_dashboard
        valueIncludes:
          - "return order.unitPrice * order.quantity"
  toolExpect:
    requiredToolsAnyOf:
      # Inspecting the app's files is satisfied by either reading them directly
      # or grepping for the revenue calculation.
      - [read_app_file, search_app]
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - inspects the dashboard app's files to locate the revenue calculation
    - fixes the per-order revenue so it multiplies unit price by quantity
    - leaves the result as an AI draft and does not deploy or save it

- id: global-test30-raw-app-small-edit-large
  prompt: |-
    In the dashboard app at `f/evals/global/analytics_dashboard`, change the main
    page heading from "Operations Console" to "Revenue Overview". Leave everything
    else unchanged. Keep it as an AI draft only; do not deploy or save it.
  initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: app
        path: f/evals/global/analytics_dashboard
        valueIncludes:
          - "Revenue Overview"
  toolExpect:
    requiredToolsAnyOf:
      # Inspecting the app's files is satisfied by reading them directly or
      # grepping for the target with search_app.
      - [read_app_file, search_app]
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - renames the main page heading to Revenue Overview
    - does not change other dashboard behavior
    - leaves the result as an AI draft only

- id: global-test31-raw-app-debug-inspect-data
  prompt: |-
    The raw app dashboard at `f/evals/global/analytics_dashboard` is reporting
    revenue totals that look too low. Inspect the app's files — both the sample
    order data module and the revenue calculation — to work out whether the bug is
    in the data or in the calculation, then fix the actual cause. Keep the change as
    an AI draft only; do not deploy or save it.
  initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard
  runtime:
    maxTurns: 22
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: app
        path: f/evals/global/analytics_dashboard
        valueIncludes:
          - "return order.unitPrice * order.quantity"
  toolExpect:
    requiredToolsAnyOf:
      # Inspecting the app's files is satisfied by reading them directly or
      # grepping for the target with search_app.
      - [read_app_file, search_app]
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - inspects both the sample order data module and the revenue aggregation logic
    - identifies the per-order revenue bug and fixes it to multiply unit price by quantity
    - leaves the result as an AI draft only

- id: global-test32-raw-app-cross-file-consistency
  prompt: |-
    The raw app dashboard at `f/evals/global/analytics_dashboard` shows revenue
    totals that disagree between the Revenue Summary tile, the orders table, and the
    regional breakdown. Investigate how each of those computes revenue, work out
    which calculation is wrong, and fix it. Keep the change as an AI draft only; do
    not deploy or save it.
  # Cross-file investigation: forces the model through several overlapping files
  # (the summary's aggregation helper, the orders table, the regional breakdown) —
  # a realistic multi-file read load that exercises the read_app_file cap.
  initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard
  runtime:
    maxTurns: 24
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: app
        path: f/evals/global/analytics_dashboard
        valueIncludes:
          - "return order.unitPrice * order.quantity"
  toolExpect:
    requiredToolsAnyOf:
      # Inspecting the app's files is satisfied by reading them directly or
      # grepping for the target with search_app.
      - [read_app_file, search_app]
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - inspects the revenue calculation behind the summary tile, the orders table, and the regional breakdown
    - identifies that the per-order revenue helper omits quantity and fixes it to multiply unit price by quantity
    - leaves the result as an AI draft only

- id: global-test33-raw-app-rename-across-files
  prompt: |-
    In the dashboard app at `f/evals/global/analytics_dashboard`, rename the
    `formatCurrency` helper to `formatMoney` everywhere it is defined, imported, and
    called. Leave the separate `formatCurrencyPrecise` helper exactly as it is. Keep
    the change as an AI draft only; do not deploy or save it.
  # Find-all-usages rename: formatCurrency is defined once and called in 6 places
  # spread across 4 component files (and imported in 4). Locating every usage is the
  # exact task search_app is meant to make cheap — one grep returns all file:line
  # rows instead of reading each component whole. valueExcludes "formatCurrency("
  # asserts the definition and all call sites were renamed while tolerating the
  # preserved formatCurrencyPrecise (which is never followed by "(").
  initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard
  runtime:
    maxTurns: 22
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: app
        path: f/evals/global/analytics_dashboard
        valueIncludes:
          - "export function formatMoney"
          - "formatMoney("
        valueExcludes:
          - "formatCurrency("
  toolExpect:
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - renames the formatCurrency definition, imports, and all call sites to formatMoney
    - leaves the unrelated formatCurrencyPrecise helper unchanged
    - leaves the result as an AI draft only

# --- Path selection (u/<user> vs f/<folder>) ---
# These cases assert how the assistant picks a workspace path when the user gives
# none: a bare name defaults to the personal scope `u/<user>/`, an existing folder
# whose purpose matches is used, a non-admin targets a writable folder and never a
# read-only one, and shared intent with no matching folder asks rather than invents.
# Each depends on the seeded `user` fixture (username / is_admin / folders /
# folders_read) so the prompt's folder guidance and `u/{username}` are well-formed.

- id: global-path1-bare-name-defaults-to-personal
  prompt: |-
    Stage a quick draft helper that takes a string and returns it trimmed of
    leading and trailing whitespace. Just keep it as a draft.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        pathStartsWith: u/admin/
  toolExpect:
    requiredToolsUsed:
      - write_script
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - stages a single script draft for a trim helper
    - defaults the path to the current user's personal scope (u/admin/...) since no path or folder was given
    - does not invent an f/<folder> path
    - leaves the result as a draft only

- id: global-path2-match-existing-folder
  prompt: |-
    Draft a flow for the marketing team's weekly campaign report.
    It should take a week number and return a short summary string.
    Keep it as a draft only.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_folders.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: flow
        pathStartsWith: f/marketing/
  toolExpect:
    requiredToolsUsed:
      - write_flow
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - drafts a flow for the marketing campaign report
    - places it in the existing marketing folder (f/marketing/...) rather than the personal scope or an invented folder
    - leaves the result as a draft only

- id: global-path3-shared-intent-unknown-folder-asks
  prompt: |-
    Put together a draft onboarding checklist flow for the People Ops team to use
    when a new hire joins. Keep it as a draft.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_folders.json
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - askUserQuestion
    forbiddenToolsUsed:
      - write_flow
      - write_script
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - recognizes the request implies shared/team work but names no existing folder (none of marketing/data_engineering/shared_utils fit People Ops)
    - asks which folder to use instead of guessing or inventing one
    - does not create a draft until the folder is known

- id: global-path4-nonadmin-avoids-readonly-folder
  prompt: |-
    Draft a small flow that returns today's date as an ISO string, and stage it in
    one of our shared team folders. Keep it as a draft.
  initial: ai_evals/fixtures/frontend/global/initial/user_bob_nonadmin_teams.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: flow
        pathStartsWith: f/team_a/
    forbiddenDrafts:
      - type: flow
        pathStartsWith: f/team_b/
  toolExpect:
    requiredToolsUsed:
      - write_flow
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - drafts a flow that returns the current date as an ISO string
    - places it in team_a (writable by this non-admin user) and not team_b (read-only)
    - leaves the result as a draft only

- id: global-test-pipeline-create-node
  prompt: |-
    Set up the first step of a data pipeline at `f/evals/global/orders_ingest`.
    On a schedule, it should pull raw orders and land them in a managed DuckLake
    table so later steps can build on it. Keep it as an AI draft only — don't
    deploy or save it.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        path: f/evals/global/orders_ingest
        valueIncludes:
          - pipeline
    forbiddenDrafts:
      - type: flow
        pathStartsWith: f/evals/global/
  toolExpect:
    requiredToolsUsed:
      - write_script
    forbiddenToolsUsed:
      - write_flow
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    # A pipeline node is DECLARATIVE: triggers are declared by `-- on <ref>`
    # annotations (the trigger row is created separately) and a `-- materialize`
    # output is a MANAGED write where the body is a bare SELECT that the runtime
    # wraps in the create/replace. Do not expect a separate trigger config or a
    # hand-written CREATE TABLE / INSERT — those would be wrong for a materialize node.
    - builds a data pipeline node as a script (not a flow)
    - marks the script as a pipeline member with the pipeline annotation in the script's comment syntax (`-- pipeline` for a DuckDB/SQL node, not `// pipeline`)
    - declares the schedule trigger with the `-- on schedule` annotation comment (this annotation is the correct and complete way a pipeline node binds a schedule; no separate trigger configuration is expected)
    - declares the managed DuckLake output with `-- materialize ducklake://<table>` and writes the body as a bare SELECT (materialize is a managed write, so the node correctly does NOT hand-write its own CREATE TABLE / INSERT)
    - leaves the result as an AI draft and does not deploy or save it

- id: global-test-pipeline-two-node-chain
  prompt: |-
    Build a small data pipeline in the `f/evals/global` folder: one step that
    ingests orders into a DuckLake table, and a second step that reads that table
    and writes a daily order-count rollup table. Wire the second step to run off
    the first step's output. Keep everything as drafts — don't deploy.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 14
  validate:
    draftCountAtLeast: 2
    requiredDrafts:
      - type: script
        pathStartsWith: f/evals/global/
        valueIncludes:
          - pipeline
          - ducklake
    forbiddenDrafts:
      - type: flow
        pathStartsWith: f/evals/global/
  toolExpect:
    requiredToolsUsed:
      - write_script
    forbiddenToolsUsed:
      - write_flow
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    # Pipeline nodes are declarative: `-- on <ref>` binds inputs/triggers and
    # `-- materialize ducklake://<table>` is a managed write whose body is a bare
    # SELECT. Do not expect hand-written CREATE TABLE / INSERT on a materialize node.
    - creates two data pipeline nodes as scripts (not a flow) in f/evals/global
    - both scripts carry the pipeline annotation in their comment syntax (`-- pipeline` for DuckDB/SQL nodes, not `// pipeline`)
    - the first ingests orders into a DuckLake table (a `-- materialize ducklake://<table>` output with a bare SELECT body is correct; no hand-written CREATE TABLE / INSERT is expected)
    - the second reads that same table via `-- on ducklake://<that-table>` and materializes a daily rollup table, wiring it to the first step's output asset
    - leaves both as AI drafts without deploying

- id: global-test-pipeline-complex-incremental
  prompt: |-
    Build a data pipeline in the `f/evals/global` folder for our web shop's
    orders. It has three steps:
    1. On a schedule, ingest the raw order CSVs under `s3://raw/orders/` into a
       managed DuckLake table.
    2. An incremental daily rollup: read that raw orders table and, on each run,
       append just the current day's order count and total revenue into a second
       DuckLake table. It should process one day at a time, not rebuild the whole
       table every run.
    3. A final step that reads the daily rollup table and exports the latest data
       as a Parquet file to `s3://reports/` for the BI team.
    Wire each step to the previous step's output so they form one pipeline. Keep
    everything as AI drafts — don't deploy or save.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 18
  validate:
    draftCountAtLeast: 3
    requiredDrafts:
      - type: script
        pathStartsWith: f/evals/global/
        valueIncludes:
          - pipeline
          - ducklake
    forbiddenDrafts:
      - type: flow
        pathStartsWith: f/evals/global/
  toolExpect:
    requiredToolsUsed:
      - write_script
    forbiddenToolsUsed:
      - write_flow
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    # Pipeline nodes are declarative: `-- on <ref>` binds inputs/triggers, and a
    # DuckLake `-- materialize` output is a managed write whose body is a bare SELECT
    # (the runtime performs the create/replace/append/merge). Do not expect a
    # separate trigger config or hand-written CREATE TABLE / INSERT on a
    # materialize node. S3/Parquet output is NOT materialize: the body writes it.
    - builds the pipeline as three independent scripts (not a flow) in f/evals/global
    - every node carries the pipeline annotation in its own comment syntax (`-- pipeline` for DuckDB/SQL nodes, not `// pipeline`)
    - step 1 binds a schedule with `-- on schedule` and declares a managed DuckLake output with `-- materialize ducklake://<table>` and a bare SELECT body (no separate trigger config or hand-written CREATE TABLE is expected)
    - "step 2 is incremental: each run adds only that day's rows to a second DuckLake table rather than rebuilding the whole table every run (e.g. an `append` or `key=<col>` merge materialize mode, not a full replace). Selecting the day via the `-- partitioned daily` + `{partition}` / `wm_partition(...)` idiom is the idiomatic form, but an equivalent current-day filter also satisfies this; a full-refresh/replace of the whole table does not"
    - step 2 reads the same DuckLake table step 1 writes (via `-- on ducklake://<that-table>`), wiring it to step 1's output asset
    - step 3 reads the daily rollup table and exports it as a Parquet file to S3
    - does not misuse `-- materialize` for the S3 Parquet export (materialize is DuckLake-only; the S3 output is written by the script body, e.g. a DuckDB COPY or an SDK write)
    - leaves all three nodes as AI drafts without deploying or saving

- id: global-path5-create-folder-then-draft
  prompt: |-
    Create a new shared folder called "analytics" for our data work, then draft a
    script in it that returns the current timestamp as an ISO string. Keep the
    script as a draft.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        pathStartsWith: f/analytics/
  toolExpect:
    requiredToolsUsed:
      - create_folder
      - write_script
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  skipJudge: true
  judgeChecklist:
    - creates a new shared folder named "analytics" via create_folder
    - drafts a script placed in that folder (f/analytics/...) returning an ISO timestamp
    - leaves the script as a draft only

- id: global-artifact-plan-create
  prompt: |-
    I'm about to build a customer onboarding flow, but first I want a short written plan I can review and iterate on before any code.
    Draft a markdown plan with a title, a one-sentence summary, and three or four bullet steps.
    Keep it as something I can reopen and revise later — don't build the flow itself yet.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json
  runtime:
    maxTurns: 6
    sessionChat: true
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - create_artifact
    forbiddenToolsUsed:
      - write_flow
      - write_script
      - deploy_workspace_item
  judgeChecklist:
    - saves the plan as a markdown artifact via create_artifact rather than only replying inline
    - the artifact content has a title, a one-line summary, and three or four bullet steps for onboarding
    - the artifact is registered as the session's plan (role "plan"), not as an ordinary note - the user asked for the plan they will come back to and revise
    - does not create a flow or script draft yet

- id: global-planmode1-hands-over-a-plan
  prompt: |-
    Our support inbox is a mess. I want incoming emails triaged by urgency and routed to the
    right team, with anything urgent also posted to Slack.
    Work out how you'd build this in Windmill.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 10
    sessionChat: true
    planMode: true
  # No draft assertion: approving the plan opens the gate mid-run, and building from there is
  # what production asks for, so a draft is not a failure. The gate itself is covered by
  # shared.test.ts; what only a real model can show is whether it researches and hands over a
  # usable plan instead of guessing at one.
  toolExpect:
    requiredToolsUsed:
      - exit_plan_mode
  # Not "saves the plan as an artifact": exit_plan_mode writes it, so the harness would
  # satisfy that on every run the tool is called at all — it grades itself, not the model.
  judgeChecklist:
    - the plan covers classifying an incoming email by urgency, routing it to a team, and posting urgent ones to Slack
    - the plan is specific about what would be built in Windmill (a flow and its steps, or the scripts involved)

- id: global-planmode2-sketches-while-planning
  prompt: |-
    We're moving our nightly CSV export off a schedule and onto a webhook the vendor calls
    when their file is ready. Work out how you'd rebuild it in Windmill — and draw me the
    shape of it before you write anything, I find that easier to react to than prose.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 10
    sessionChat: true
    planMode: true
  # Both tools, because either alone is a different behaviour: create_artifact without
  # exit_plan_mode means the model filed the plan as the drawing, and exit_plan_mode without
  # create_artifact means the posture blocked the drawing it was asked for.
  toolExpect:
    requiredToolsUsed:
      - create_artifact
      - exit_plan_mode
  judgeChecklist:
    - saves a diagram of the proposed design as an artifact rather than only describing it in chat
    - the diagram covers the webhook that starts the run and the steps that replace the nightly schedule
    - hands the plan over with exit_plan_mode instead of leaving it in the artifact
    - does not register the diagram as the session's plan document

- id: global-npm1-script-search-package
  prompt: |-
    Find a good npm package for parsing RSS/Atom feeds and use it to create a draft Bun script
    at `f/evals/global/fetch_feed`. The script should take a string `url` input and return the
    feed title along with the titles of the 5 most recent items. Tell me which package you
    picked and link its documentation. Leave it as an AI draft only; do not deploy.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 12
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        path: f/evals/global/fetch_feed
        language: bun
        valueIncludes:
          - url
  toolExpect:
    requiredToolsUsed:
      - search_npm_packages
      - write_script
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    - searches npm for a feed-parsing package before writing the code
    - the draft Bun script imports the chosen npm package rather than hand-rolling an XML/RSS parser
    - the script takes a url input and returns the feed title plus the recent item titles
    - the reply names the chosen package and links its documentation
    - the result stays as an AI draft and is not deployed

- id: global-dbschema1-postgres-resource-tables
  prompt: |-
    I have a postgres resource at f/data/reports_pg in this workspace.
    What tables does that database have?
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json
  runtime:
    maxTurns: 12
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - get_db_schema
    forbiddenToolsUsed:
      - write_script
      - write_resource
      - test_run_script
      - deploy_workspace_item
    toolCallArgs:
      - tool: get_db_schema
        field: resourcePath
        stringIncludesAnyOf:
          - f/data/reports_pg
  skipJudge: true
  judgeChecklist:
    - fetches the schema through get_db_schema with the resource path f/data/reports_pg
    - when the lookup fails, tells the user instead of inventing table names
    - does not write scripts or resources to answer a read-only question

# --- Dedicated tools for workspace, run and instance state ---
# The harness serves the worker read itself (a benchmark fetch handler in
# adapters/frontend). The stale `api-catalog` in the id below is kept so
# results stay comparable across benchmark runs.

- id: global-test30-api-catalog-workers
  prompt: |-
    Are any workers online right now? Give me a quick summary of the workers and when they last pinged.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - list_workers
    forbiddenToolsUsed:
      - write_script
      - deploy_workspace_item
  # Read-only workspace inspection produces no draft; validate via tool use.
  skipJudge: true
  judgeChecklist:
    - reads worker state through list_workers instead of guessing or fabricating
    - reports worker status from the returned data

- id: global-test37-ducklake-declared-measure
  prompt: |-
    We track orders in the main ducklake. Write me a duckdb script that reports total
    revenue by month. Keep it as a draft, don't deploy it.
  initial: ai_evals/fixtures/frontend/global/initial/ducklake_orders_metrics.json
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 1
  toolExpect:
    requiredToolsUsed:
      - list_data_metrics
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  # The judge runs: the point is not that the tool was called but that the number it
  # describes is the declared one. `revenue` excludes test rows, so an aggregate that
  # reproduces it without the filter is plausible, runnable and wrong.
  judgeChecklist:
    - totals revenue with the declared sum over the amount column rather than an invented aggregate over a guessed column
    - excludes test orders from the total, as the declared revenue measure does
    - groups by month using the declared order_month expression over order_date
    - does not introduce column names absent from the declarations

- id: global-test31-draft-test-run-not-deployed
  prompt: |-
    Update `f/evals/global/format_greeting` so the provided name is uppercased in the greeting, then run it with name "ada" to check it works.
    Keep it as an AI draft only; do not deploy or save it.
  initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        path: f/evals/global/format_greeting
        language: bun
        valueIncludes:
          - toUpperCase
  toolExpect:
    requiredToolsUsed:
      - test_run_script
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  # The judge only sees the drafts artifact and cannot observe runs, so it always
  # docks the prompt's "run it" requirement — validate deterministically instead:
  # draft content via valueIncludes, the test run via toolExpect.
  skipJudge: true
  judgeChecklist:
    - creates an AI draft of f/evals/global/format_greeting with the name uppercased in the greeting
    - does not deploy or save the draft

- id: global-test32-delete-via-dedicated-tool
  prompt: |-
    Delete the deployed script `f/evals/global/send_report_digest` from the workspace — we don't use it anymore.
  initial: ai_evals/fixtures/frontend/global/initial/report_digest_script.json
  runtime:
    maxTurns: 6
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - delete_workspace_item
    forbiddenToolsUsed:
      - write_script
  # Deletion produces no draft; validate via tool use.
  skipJudge: true
  judgeChecklist:
    - deletes the deployed script via delete_workspace_item

- id: global-test33-run-deployed-script-with-form
  prompt: |-
    Run the deployed script `f/evals/global/format_greeting` for me with the name "ada".
  initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json
  runtime:
    maxTurns: 8
    # A session chat is where the run card has a preview pane beside it; run_script
    # itself is offered in every chat.
    sessionChat: true
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - run_script
      # A draft may declare different arguments than the deployed version being run, so
      # the names to prefill have to come from the deployed schema.
      - read_workspace_item
    forbiddenToolsUsed:
      - test_run_script
      - write_script
      - deploy_workspace_item
    # An empty form pushes the work back onto the user, so the prefill is part of
    # what the tool is for.
    toolCallArgs:
      - tool: run_script
        field: args.name
        stringIncludesAnyOf:
          - ada
  # Running produces no draft, and the judge cannot observe runs; validate via tool use.
  skipJudge: true
  judgeChecklist:
    - runs the deployed script through run_script rather than a preview test run
    - passes the name "ada" so the confirmation form comes up prefilled

- id: global-test34-run-with-secret-from-variable
  prompt: |-
    Run the deployed `f/evals/global/billing_sync` for the account `acme` — use the billing
    API token we already keep in the workspace.
  initial: ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json
  runtime:
    maxTurns: 10
    # A session chat is where the run card has a preview pane beside it; run_script
    # itself is offered in every chat.
    sessionChat: true
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - run_script
    forbiddenToolsUsed:
      - write_script
      - deploy_workspace_item
    # A secret argument is filled by naming the variable that holds it: the value stays in
    # the variable and only its path travels. A literal reaches the job as a reference too,
    # minted on the way in, but it stays in the tool call the model emitted.
    toolCallArgs:
      - tool: run_script
        field: args.api_token
        stringIncludesAnyOf:
          - "$var:f/evals/global/stripe_api_token"
      - tool: run_script
        field: args.account
        stringIncludesAnyOf:
          - acme
  # Running produces no draft, and the judge cannot observe runs; validate via tool use.
  skipJudge: true
  judgeChecklist:
    - fills the secret argument with a reference to the existing workspace variable rather than a literal token
    - passes the account "acme"
    - does not invent or guess the token's value

- id: global-test35-run-deployed-flow-with-form
  prompt: |-
    Run the deployed flow `f/evals/global/notify_customer` for me — the customer is `acme`.
  initial: ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json
  runtime:
    maxTurns: 8
    # A session chat is where the run card has a preview pane beside it; run_flow
    # itself is offered in every chat.
    sessionChat: true
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - run_flow
      # A draft may declare different arguments than the deployed version being run, so
      # the names to prefill have to come from the deployed schema.
      - read_workspace_item
    forbiddenToolsUsed:
      - test_run_flow
      - write_flow
      - deploy_workspace_item
    # An empty form pushes the work back onto the user, so the prefill is part of
    # what the tool is for.
    toolCallArgs:
      - tool: run_flow
        field: args.customer
        stringIncludesAnyOf:
          - acme
  # Running produces no draft, and the judge cannot observe runs; validate via tool use.
  skipJudge: true
  judgeChecklist:
    - runs the deployed flow through run_flow rather than a preview test run
    - passes the customer "acme" so the confirmation form comes up prefilled

- id: global-test36-draft-flow-test-run-not-deployed
  prompt: |-
    Update the `calculate_total` step of `f/evals/global/process_invoice` so it applies 8% tax and
    returns `subtotal`, `tax` and `total`, then run it to check it works.
    Keep it as an AI draft only; do not deploy or save it.
  initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: flow
        path: f/evals/global/process_invoice
  toolExpect:
    # A one-step flow is as well checked by running the step as the whole flow, so both
    # count: what matters is that the run is against the draft.
    requiredToolsAnyOf:
      - [test_run_flow, test_run_step]
    # The draft is what the user asked to check, and run_flow would run the deployed
    # version instead — the edit would not be in what ran.
    forbiddenToolsUsed:
      - run_flow
      - deploy_workspace_item
  # The judge cannot observe runs, and the edit's content is already pinned by
  # global-test5 on this fixture; what this case guards is where the run went.
  skipJudge: true
  judgeChecklist:
    - creates an AI draft of f/evals/global/process_invoice applying 8% tax
    - does not deploy or save the draft

- id: global-test38-chat-flow-follow-up-same-conversation
  prompt: |-
    I want to check that my support chat flow `f/evals/global/support_chat` remembers what was said.
    Test it: first send "My name is Ada", then send "What is my name?" as a follow-up in the same chat.
  initial: ai_evals/fixtures/frontend/global/initial/support_chat_flow.json
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - test_run_flow
    # A chat flow's memory lives in its conversation, so a follow-up only reaches the first
    # turn's history when both test runs name the same conversation.
    toolCallArgs:
      - tool: test_run_flow
        field: memory_id
        sharedByAtLeast: 2
    forbiddenToolsUsed:
      - run_flow
      - deploy_workspace_item
  # The judge cannot observe runs; what this case guards is the conversation the runs share.
  skipJudge: true
  judgeChecklist:
    - test-runs the chat flow twice, the second message as a follow-up in the first run's conversation

- id: global-undo-created-draft
  prompt: |-
    Create a draft Postgres resource at `u/admin/scratch_db` for host db.example.com port 5432, database `orders`, user `app`, and tell me what fields it ended up with.
    Once you've shown me that, delete it from the workspace again — I only wanted to see the shape of it.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - write_resource
      - discard_local_draft
    forbiddenToolsUsed:
      - delete_workspace_item
      - deploy_workspace_item
  # Undoing a draft leaves no draft behind; validate via tool use.
  skipJudge: true
  judgeChecklist:
    - undoes its own never-deployed resource with discard_local_draft rather than delete_workspace_item

- id: global-draft-diff-report
  prompt: |-
    Update the existing workspace script at `f/evals/global/format_greeting` so the returned message ends with an exclamation mark, keeping everything else the same.
    Then show me exactly what your draft changes compared to the deployed version.
    Leave the result as an AI draft only; do not deploy or save it.
  initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        path: f/evals/global/format_greeting
        language: bun
        valueIncludes:
          - "!"
  toolExpect:
    requiredToolsUsed:
      - diff
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: diff
        field: path
        stringIncludesAnyOf:
          - f/evals/global/format_greeting
  judgeChecklist:
    - creates an AI draft for the existing f/evals/global/format_greeting script with the exclamation-mark change
    - the draft changes only the returned message's punctuation — summary, language, path, and the rest of the code are untouched
    - does not deploy or save the draft to the workspace

- id: global-resource-manual-credentials
  prompt: |-
    Set up a resource for our production Postgres database at `f/evals/global/prod_db` (host db.internal.example.com, port 5432, database `orders`, user `app`).
    I don't want to paste the password into this chat — prepare everything so I can enter it myself.
    Leave it as an AI draft only; do not deploy or save it.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 12
  validate:
    draftCountAtLeast: 1
    requiredDrafts:
      - type: resource
        path: f/evals/global/prod_db
        valueIncludes:
          - db.internal.example.com
          - orders
  toolExpect:
    requiredToolsUsed:
      - write_resource
      - open_page
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      # The model may land the user in the resource's drawer or in the drawer of
      # the secret variable it created for the password — both are correct.
      - tool: open_page
        field: page
        stringIncludesAnyOf:
          - resources
          - variables
      - tool: open_page
        field: open
        stringIncludesAnyOf:
          - prod_db
          - password
  judgeChecklist:
    - creates a postgres resource draft at f/evals/global/prod_db with the provided host, port, database, and user
    - the password is left for the user to provide (empty, a placeholder, or a secret variable reference) — no invented password value presented as real
    - does not deploy or save anything to the workspace

- id: global-test29-email-trigger-draft
  prompt: |-
    Set up a draft auto-reply job.
    Create a Bun script at `f/evals/global/email_pong` that returns the string "pong".
    Then set it up so it runs whenever an email is received at the inbox `pong`.
    Leave everything as AI drafts only; do not deploy or save anything to the workspace.
  runtime:
    maxTurns: 10
  validate:
    requiredDrafts:
      - type: script
        path: f/evals/global/email_pong
        language: bun
        valueIncludes:
          - pong
  toolExpect:
    requiredToolsUsed:
      - write_script
      - write_trigger
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      # "Runs when an email is received" must resolve to the native email trigger kind,
      # never a faked HTTP webhook. Assert the recorded tool-call kind (not the draft):
      # it holds even on a CE backend where email trigger routes (smtp+private) 404.
      - tool: write_trigger
        field: kind
        stringIncludesAnyOf:
          - email
  skipJudge: true

# `List Pages` takes a cursor without ever following one, so where the next cursor
# comes from (`_links.next`) is stated in the authored metadata and in no script body.
# Paginating is therefore the one task where reading scripts is not an equal route.
- id: global-hub4-confluence-paginate-all-pages
  prompt: |-
    I need a script that returns every page in one of my Confluence spaces, not just the
    first batch — it should keep fetching until there are no more.
    Save it as a draft at `f/evals/global/confluence_all_pages` and leave it as an AI
    draft; do not deploy it.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 14
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        path: f/evals/global/confluence_all_pages
        valueIncludes:
          - /wiki/
          - Basic
  toolExpect:
    requiredToolsUsed:
      - write_script
    # Either route to the integration's conventions is fine; the checklist is where the
    # difference between them has to show up.
    requiredToolsAnyOf:
      - - search_hub_scripts
        - get_hub_integration
    forbiddenToolsUsed:
      - deploy_workspace_item
  judgeChecklist:
    - the script loops until the API stops returning a next link, rather than making a single request
    - the next cursor is read from the response's `_links.next` or the Link header, not from an invented field such as `next_cursor`, `nextPageToken` or `offset`
    - it authenticates with HTTP Basic built from the Confluence resource's email and API token rather than a bearer token
    - the script takes a Confluence resource as an input parameter rather than loose credential strings
    - the result stays an AI draft and is not deployed

# Unlike the Confluence case, both facts this needs — the {"result"} envelope and
# PATCH-not-PUT — are visible in ServiceNow's own shipped scripts. That makes it the
# control for whether the metadata earns its place when the scripts already answer.
- id: global-hub5-servicenow-assign-incident
  prompt: |-
    Write me a script that assigns an existing ServiceNow incident to a person, looking
    that person up by their email address.
    Save it as a draft at `f/evals/global/servicenow_assign_incident` and leave it as an
    AI draft; do not deploy it.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 14
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        path: f/evals/global/servicenow_assign_incident
  toolExpect:
    requiredToolsUsed:
      - write_script
    requiredToolsAnyOf:
      - - search_hub_scripts
        - get_hub_integration
    forbiddenToolsUsed:
      - deploy_workspace_item
  judgeChecklist:
    - it looks the person up in the sys_user table by email rather than assuming a sys_id was supplied
    - it reads records out of the `result` envelope the ServiceNow API wraps responses in
    - it sets assigned_to to the user's sys_id rather than to the raw email address
    - it updates the incident with a partial update (PATCH) rather than a full replace that would clear the fields it did not send
    - the result stays an AI draft and is not deployed

# Outreach is JSON:API. Its content type and payload envelope are both stated in the
# metadata and visible in the shipped update scripts, so this is a second control.
- id: global-hub6-outreach-reschedule-task
  prompt: |-
    Write me a script that moves an existing Outreach task to a new due date.
    Save it as a draft at `f/evals/global/outreach_reschedule_task` and leave it as an AI
    draft; do not deploy it.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 14
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        path: f/evals/global/outreach_reschedule_task
  toolExpect:
    requiredToolsUsed:
      - write_script
    requiredToolsAnyOf:
      - - search_hub_scripts
        - get_hub_integration
    forbiddenToolsUsed:
      - deploy_workspace_item
  judgeChecklist:
    - the request sends the JSON:API content type application/vnd.api+json for its body
    - the payload wraps the change as data.type and data.attributes rather than sending bare top-level fields
    - it identifies the task by id inside the payload rather than only in the URL, and treats the id as a number
    - it updates the existing task with PATCH rather than creating a new one
    - the result stays an AI draft and is not deployed

# --- Windmill Hub reuse (search_hub_scripts + read_workspace_item on a hub/ path) ---
# Holded's API is obscure enough that a model writing from memory cannot reproduce
# its endpoint and `key` auth header — so the draft's fidelity to the published
# script is what proves the hub content was actually fetched, not guessed.
- id: global-hub1-reuse-hub-script
  prompt: |-
    I want to email one of my Holded invoices to a customer from Windmill.
    There is already a script for that on the Windmill hub — reuse it instead of writing
    your own, and save it as a draft script at `f/evals/global/holded_send_document`.
    Leave it as an AI draft; do not deploy it.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 12
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: script
        path: f/evals/global/holded_send_document
        language: bun
        valueIncludes:
          - api.holded.com/api/invoicing/v1/documents
          # `mailTemplateId` is an optional field of the published script's body
          # that a model writing from memory does not invent, so it is what
          # separates reusing the hub script from re-deriving one that merely
          # hits the same endpoint.
          - mailTemplateId
  toolExpect:
    requiredToolsUsed:
      - search_hub_scripts
      - read_workspace_item
      - write_script
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      - tool: read_workspace_item
        field: path
        stringIncludesAnyOf:
          - hub/
  judgeChecklist:
    - the draft sends an existing Holded document by email rather than creating one
    - the request targets Holded's document send endpoint, not an invented URL
    - authentication uses Holded's own key header rather than a bearer token
    - the document type, document id, and recipient emails are inputs to the script
    - the result stays an AI draft and is not deployed

# Same task as global-hub1 with the hub never mentioned: this is what tells us the
# model reaches for the hub on its own rather than only when pointed at it.
- id: global-hub2-checks-hub-unprompted
  prompt: |-
    I want to email one of my Holded invoices to a customer from Windmill.
    Save it as a draft script at `f/evals/global/holded_send_document` and leave it as
    an AI draft; do not deploy it.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 12
  validate:
    draftCountExactly: 1
    requiredDrafts:
      # No language assertion: the Holded script's body is `//native`, so a faithful
      # fork lands on bun or bunnative depending on how literally it is copied.
      - type: script
        path: f/evals/global/holded_send_document
        valueIncludes:
          - api.holded.com/api/invoicing/v1/documents
          - mailTemplateId
  toolExpect:
    requiredToolsUsed:
      - search_hub_scripts
      - write_script
    forbiddenToolsUsed:
      - deploy_workspace_item
  judgeChecklist:
    - the draft sends an existing Holded document by email rather than creating one
    - the request targets Holded's document send endpoint, not an invented URL
    - authentication uses Holded's own key header rather than a bearer token
    - the result stays an AI draft and is not deployed

# The fixture hub carries two Baremetrics scripts, neither for annotations, so nothing
# does what is asked. Its conventions — an `apiKey` resource, bearer auth, the
# api.baremetrics.com/v1 base — are only knowable by reading one of those near-misses.
- id: global-hub3-integration-as-example
  prompt: |-
    Write me a script that adds an annotation to my Baremetrics account.
    Save it as a draft at `f/evals/global/baremetrics_create_annotation` and leave it as
    an AI draft; do not deploy it.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 14
  validate:
    draftCountExactly: 1
    requiredDrafts:
      # No language assertion: matching the hub script's own dialect is the point,
      # and Baremetrics' scripts are `//native`, so both bun and bunnative are right.
      - type: script
        path: f/evals/global/baremetrics_create_annotation
        valueIncludes:
          - api.baremetrics.com/v1
          - apiKey
  toolExpect:
    requiredToolsUsed:
      - search_hub_scripts
      - write_script
    # Reading a near-miss script and reading the integration's own metadata are two
    # routes to the same conventions, so requiring either keeps the case about the
    # outcome rather than about which route the model took.
    requiredToolsAnyOf:
      - - read_workspace_item
        - get_hub_integration
    forbiddenToolsUsed:
      - deploy_workspace_item
  judgeChecklist:
    - the script takes a Baremetrics resource as an input parameter rather than a bare API key string
    - the resource type it declares carries an apiKey field, matching how other Baremetrics hub scripts take credentials
    - the request authenticates with a bearer token built from that resource
    - the script posts an annotation rather than reading data back
    - the result stays an AI draft and is not deployed

# The value of a secret variable is unreadable, so a metadata-only edit must leave
# it untouched: passing any `value` here means inventing one, which silently
# replaces the real secret at deploy.
- id: global-secret-variable-description-only-edit
  prompt: |-
    The variable `f/evals/global/stripe_api_token` has a confusing description.
    Change it to say it is the Stripe key used by the nightly billing sync, and leave
    everything else about the variable alone. Keep it as an AI draft; do not deploy it.
  initial: ai_evals/fixtures/frontend/global/initial/secret_api_token_variable.json
  runtime:
    maxTurns: 8
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: variable
        path: f/evals/global/stripe_api_token
        valueIncludes:
          # "billing" is already in the seeded description and labels, so it would pass
          # without any edit; "nightly" can only come from the new description.
          - nightly
          - '"is_secret": true'
        valueExcludes:
          # The draft must not carry an invented value, a self-reference, or the
          # real secret it was never shown.
          - '$var:'
          - sk_live_do_not_leak_me
  toolExpect:
    requiredToolsUsed:
      - write_variable
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
    toolCallArgs:
      # Restating `is_secret: true` is fine; supplying a value is not.
      - tool: write_variable
        field: value
        fieldMustBeAbsent: true
  judgeChecklist:
    - updates the description of f/evals/global/stripe_api_token to mention Stripe and the nightly billing sync
    - keeps the variable secret
    - does not invent, guess, or restate a value for the variable
    # A secret draft stores "" when it stages no new value, and this case must not stage
    # one — the judge has to be told, or it reads the "" as the value having been cleared.
    - 'the draft''s empty value string is expected and correct: a secret variable draft stores "" when no new value is staged, which is exactly right for a description-only edit, so it does NOT mean the value was cleared or changed'
    - leaves the result as an AI draft and does not deploy it

# Reproduces GIT-967: a hello-world flow with a React app in front of it. The agent
# built a backend runnable that fetched the Windmill REST API directly, guessing at
# WM_TOKEN / BASE_INTERNAL_URL / WM_BASE_URL until it fell back to localhost:8000, and
# wired the app to a flow that was never deployed. Both are pinned here.
- id: global-app-triggers-flow
  prompt: |-
    Create a hello world workflow at `f/evals/global/hello_world_flow` that just returns a
    greeting, and a React app at `f/evals/global/hello_world_app` with a button that runs
    that workflow and shows the result on the page.
  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
  runtime:
    maxTurns: 24
  validate:
    draftCountAtLeast: 2
    requiredDrafts:
      - type: flow
        path: f/evals/global/hello_world_flow
      - type: app
        path: f/evals/global/hello_world_app
        valueIncludes:
          # The frontend must go through the generated bindings, which is the only
          # credentialed way it can reach anything server-side.
          - './wmill'
        valueExcludes:
          # Every artifact of hand-rolling HTTP against the Windmill API from a
          # runnable instead of using the client or a path runnable.
          - BASE_INTERNAL_URL
          - WM_BASE_URL
          - WM_TOKEN
          - localhost:8000
          - getResultMaybe
          - jobs/list
          - getWorkspaceToken
          - getBaseUrl
  toolExpect:
    requiredToolsUsed:
      - write_flow
      - init_app
      - write_app_runnable
  judgeChecklist:
    - creates a flow at f/evals/global/hello_world_flow that returns a greeting
    - creates a React raw app at f/evals/global/hello_world_app with a button that runs the flow
    - the app actually invokes the flow rather than reimplementing its logic in an inline runnable
    # Both items staying drafts is the CORRECT outcome — the chat must not deploy without
    # being asked. What is judged is that the flow is named as the one item needing a deploy.
    - 'leaving both the flow and the app as drafts is expected and correct: the chat deploys nothing unless asked. Judge only whether the flow is identified as the single item that will need deploying, and that the app is NOT presented as needing deployment to be tried'
    # The judge has repeatedly flagged a correct app as broken over this.
    - '`/wmill.d.ts` is generated by Windmill from the app''s runnables and is deliberately absent from the app''s files — its absence is correct and is NOT a missing-module bug'
    - the app reaches the flow through a backend runnable, not through hand-written HTTP calls to the Windmill API
    - the app's frontend calls the runnable via the generated ./wmill bindings — backend, or backendAsync together with waitJob/getJob/streamJob, all of which are real exports of that module
    - does not read WM_TOKEN, BASE_INTERNAL_URL or WM_BASE_URL, and does not construct a Windmill API URL anywhere
    - does not call windmill-client functions that do not exist, such as getBaseUrl or getWorkspaceToken
  # The point of the case: a path runnable (and wmill.runFlow*) resolves the deployed
  # item, so a flow left as a draft makes the app dead on arrival and the user has to be
  # told. This can't live in judgeChecklist — the global judge only ever sees the drafts,
  # never what the assistant said.
  assistantExpect:
    # Plain substring test: an alternative must name the FLOW and read as an outstanding
    # obligation. Flow-agnostic wording is satisfied by "the app must be deployed"; tense-neutral
    # wording by a deploy the agent only claims to have made. The mirror expectation "don't ask
    # to deploy the app" can't be a forbiddenMentions entry, since correct answers negate it.
    requiredMentionsAnyOf:
      - - deploy the flow
        - deploy that flow
        - deploy this flow
        - deploy just the flow
        - deploy the workflow
        - deploy that workflow
        - deploy hello_world_flow
        - flow must be deployed
        - flow needs to be deployed
        - flow has to be deployed
        - flow needs deploying
        - flow will need to be deployed
        - flow will have to be deployed
        - once the flow is deployed
        - until the flow is deployed
        - workflow must be deployed
        - workflow needs to be deployed
        - workflow has to be deployed

# --- AI agent steps: the provider config must name a model the referenced resource serves ---
# The workspace AI settings are not a runtime gate, so the model id can only come from the
# resources themselves. Both cases seed AI provider resources; without them the assistant has
# nothing to write but a guessed id, which is the defect these pin.

- id: global-ai-agent-step-uses-workspace-model
  prompt: |-
    Create a draft flow at `f/evals/global/support_answer` with a single AI agent step that answers
    the user's question. The question comes in as a flow input called `query`. No tools.
    Leave it as an AI draft only; do not deploy or save it.
  initial: ai_evals/fixtures/frontend/global/initial/ai_provider_anthropic.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: flow
        path: f/evals/global/support_answer
        valueIncludes:
          - aiagent
          - $res:f/evals/global/anthropic_main
          - claude-sonnet-5
  toolExpect:
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    - creates a draft flow at f/evals/global/support_answer with one AI agent step
    - the step's provider is an object carrying kind, resource and model, not a bare resource string
    - the provider references the workspace's own AI provider resource f/evals/global/anthropic_main
    - the model is one the workspace configured for that resource, not an id invented from memory
    - the user's question reaches the agent from the query flow input
    - the result stays as an AI draft and is not deployed

- id: global-ai-agent-step-asks-which-provider
  prompt: |-
    Create a draft flow at `f/evals/global/triage_ticket` with a single AI agent step that summarises
    an incoming ticket. The ticket text comes in as a flow input called `ticket`. No tools.
    Leave it as an AI draft only; do not deploy or save it.
  initial: ai_evals/fixtures/frontend/global/initial/ai_providers_two.json
  runtime:
    maxTurns: 10
  toolExpect:
    requiredToolsUsed:
      - askUserQuestion
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  # Two configured providers make the choice the user's: the step may only be written once the
  # question is answered, and it must use one of the workspace's own resources. Which one depends
  # on the answer, so the assertion covers the shared path prefix rather than a fixed resource.
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: flow
        path: f/evals/global/triage_ticket
        valueIncludes:
          - aiagent
          - $res:f/evals/global/
  # Whether the assistant asked before writing lives in the tool record, not in the draft the
  # global judge sees.
  skipJudge: true

# Only satisfiable from the resource's own model listing: the workspace AI settings configure just
# claude-sonnet-5, so an assistant working off the settings has no Opus id to write.
- id: global-ai-agent-step-uses-listed-model
  prompt: |-
    Create a draft flow at `f/evals/global/deep_review` with a single AI agent step that reviews a
    pull request diff. The diff comes in as a flow input called `diff`. Use the Opus model — this
    one needs the strongest model available. No tools.
    Leave it as an AI draft only; do not deploy or save it.
  initial: ai_evals/fixtures/frontend/global/initial/ai_provider_anthropic.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 1
    requiredDrafts:
      - type: flow
        path: f/evals/global/deep_review
        valueIncludes:
          - aiagent
          - $res:f/evals/global/anthropic_main
          - claude-opus-5
  toolExpect:
    forbiddenToolsUsed:
      - deploy_workspace_item
      - delete_workspace_item
  judgeChecklist:
    - creates a draft flow at f/evals/global/deep_review with one AI agent step
    - the step uses the workspace's anthropic resource f/evals/global/anthropic_main
    - the model is the Opus one the user asked for, taken from the models that resource serves
    - the diff flow input reaches the agent

# The failure this pins: passing a resource as `{"$res": "<path>"}` (or as a bare path), which
# reaches the script unresolved because the backend only substitutes a string value that itself
# starts with `$res:`. The mock preview echoes args back and reports success, so nothing in the
# loop corrects a wrong shape — the arg form is the whole test.
- id: global-run-arg-resource-reference
  prompt: |-
    Run `f/evals/global/github_repo_stats` against the `windmill-labs/windmill` repo, passing our
    GitHub credentials at `f/evals/global/github_main` as its `gh_auth` input, and tell me whether
    it went through.
  initial: ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json
  runtime:
    maxTurns: 10
  validate:
    draftCountExactly: 0
  toolExpect:
    requiredToolsUsed:
      - test_run_script
    forbiddenToolsUsed:
      - write_script
      - deploy_workspace_item
    toolCallArgs:
      # Exact: the mock never resolves the reference, so a near-miss path like
      # `$res:f/evals/global/github_main_backup` would otherwise pass.
      - tool: test_run_script
        field: args.gh_auth
        stringEqualsAnyOf:
          - "$res:f/evals/global/github_main"
  # The judge only sees drafts, and this case makes none — the deliverable is the shape of the
  # run argument, checked deterministically above.
  skipJudge: true
  judgeChecklist:
    - runs the existing script rather than rewriting it
    - passes the GitHub resource as the bare string $res:f/evals/global/github_main

- id: global-drag-and-drop-app-not-editable
  prompt: |-
    Add a refresh button to the ops console app, and the same to the sales board app.
  initial: ai_evals/fixtures/frontend/global/initial/apps_code_and_drag_and_drop.json
  runtime:
    maxTurns: 12
  validate:
    # One request, two apps, only one of them editable: the code app must come back with a
    # draft and the drag-and-drop one must not. Refusing both, or editing both, fails here —
    # which is what makes this a test of the distinction rather than of caution.
    draftCountExactly: 1
    requiredDrafts:
      - type: app
        path: f/evals/global/ops_console
    forbiddenDrafts:
      - type: app
        path: f/evals/global/sales_board
  toolExpect:
    # Deliberately not constraining write_app_file/patch_app_file by argument: an entry there
    # fails when its tool was never called, so naming both would fail on whichever the model
    # did not pick. The draft assertions above cover the same ground, tool-agnostically.
    forbiddenToolsUsed:
      - init_app
      - deploy_workspace_item
      - delete_app_file
      - delete_app_runnable
  assistantExpect:
    # A refusal leaves no draft for the judge to read, so the explanation is checked here.
    # Only the app kind: substring tests cannot see paraphrase, and every wording of "I can't
    # edit it" defeats a fixed list.
    requiredMentionsAnyOf:
      - - drag-and-drop
        - drag and drop
        - low-code
        - no-code
  skipJudge: true