fix: run each ai eval model in its own job and let the docs tools reach the backend (#11533)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

.github/workflows/ai-evals-scheduled.yml

@@ -4,8 +4,9 @@ name: AI Evals (scheduled)
 # the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is tracked
 # over time. Weekly, since one pass of every suite at --runs 3 costs about 50M
 # tokens per model: every suite runs on MODELS, and global (the mode users get)
-# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Run it by
-# hand to measure a branch against main.
+# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Each mode and
+# model is its own job, so a slow model neither delays nor shares a timeout with
+# the others. Run it by hand to measure a branch against main.
 on:
   schedule:
     - cron: "0 3 * * 1"
@@ -42,14 +43,17 @@ jobs:
   setup:
     runs-on: ubuntu-latest
     outputs:
-      modes: ${{ steps.modes.outputs.modes }}
+      matrix: ${{ steps.matrix.outputs.matrix }}
     steps:
-      - id: modes
+      - id: matrix
         env:
           MODES: ${{ inputs.modes || 'global flow app script cli' }}
         run: |
-          echo "modes=$(jq -cn --arg m "$MODES" '$m | split(" ")
-            | map(select(IN("global", "flow", "app", "script", "cli")))')" >> "$GITHUB_OUTPUT"
+          echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" --arg extra "$GLOBAL_EXTRA_MODELS" '
+            def words: split(" ") | map(select(. != ""));
+            [ $modes | words[] | select(IN("global", "flow", "app", "script", "cli")) as $mode
+              | ($models | words) + (if $mode == "global" then $extra | words else [] end)
+              | .[] | {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT"
 
   evals:
     needs: setup
@@ -58,7 +62,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        mode: ${{ fromJSON(needs.setup.outputs.modes) }}
+        include: ${{ fromJSON(needs.setup.outputs.matrix) }}
     services:
       postgres:
         image: postgres:16
@@ -118,10 +122,11 @@ jobs:
         working-directory: ./cli
         run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh
 
-      - name: Run ${{ matrix.mode }} evals
+      - name: Run ${{ matrix.mode }} evals on ${{ matrix.model }}
         working-directory: ./ai_evals
         env:
           MODE: ${{ matrix.mode }}
+          MODEL: ${{ matrix.model }}
           WMILL_AI_EVAL_BACKEND_URL: http://localhost:8000
           WMILL_AI_EVAL_BACKEND_WORKSPACE: integration-tests
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
@@ -131,14 +136,10 @@ jobs:
         run: |
           bun install
           mkdir -p results
-          models="$MODELS"
-          [ "$MODE" = global ] && models="$models $GLOBAL_EXTRA_MODELS"
           reasoning=()
           [ -n "$REASONING" ] && [ "$MODE" != cli ] && reasoning=(--reasoning "$REASONING")
-          for m in $models; do
-            bun run cli -- run "$MODE" --model "$m" --runs "$RUNS" "${reasoning[@]}" \
-              --output "$PWD/results/$MODE-$m.json" || echo "::warning::$MODE on $m errored"
-          done
+          bun run cli -- run "$MODE" --model "$MODEL" --runs "$RUNS" "${reasoning[@]}" \
+            --output "$PWD/results/$MODE-$MODEL.json" || echo "::warning::$MODE on $MODEL errored"
 
       - name: Post results to the dashboard
         if: always()
@@ -166,7 +167,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: ai-evals-${{ matrix.mode }}
+          name: ai-evals-${{ matrix.mode }}-${{ matrix.model }}
           path: |
             ai_evals/logs
             ai_evals/results