fix: run each ai eval model in its own job and let the docs tools reach the backend (#11533)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

.github/workflows/ai-evals-scheduled.yml

@@ -4,8 +4,9 @@ name: AI Evals (scheduled)
 # the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is tracked
 # over time. Weekly, since one pass of every suite at --runs 3 costs about 50M
 # tokens per model: every suite runs on MODELS, and global (the mode users get)
-# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Run it by
-# hand to measure a branch against main.
+# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Each mode and
+# model is its own job, so a slow model neither delays nor shares a timeout with
+# the others. Run it by hand to measure a branch against main.
 on:
   schedule:
     - cron: "0 3 * * 1"
@@ -42,14 +43,17 @@ jobs:
   setup:
     runs-on: ubuntu-latest
     outputs:
-      modes: ${{ steps.modes.outputs.modes }}
+      matrix: ${{ steps.matrix.outputs.matrix }}
     steps:
-      - id: modes
+      - id: matrix
         env:
           MODES: ${{ inputs.modes || 'global flow app script cli' }}
         run: |
-          echo "modes=$(jq -cn --arg m "$MODES" '$m | split(" ")
-            | map(select(IN("global", "flow", "app", "script", "cli")))')" >> "$GITHUB_OUTPUT"
+          echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" --arg extra "$GLOBAL_EXTRA_MODELS" '
+            def words: split(" ") | map(select(. != ""));
+            [ $modes | words[] | select(IN("global", "flow", "app", "script", "cli")) as $mode
+              | ($models | words) + (if $mode == "global" then $extra | words else [] end)
+              | .[] | {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT"
 
   evals:
     needs: setup
@@ -58,7 +62,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        mode: ${{ fromJSON(needs.setup.outputs.modes) }}
+        include: ${{ fromJSON(needs.setup.outputs.matrix) }}
     services:
       postgres:
         image: postgres:16
@@ -118,10 +122,11 @@ jobs:
         working-directory: ./cli
         run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh
 
-      - name: Run ${{ matrix.mode }} evals
+      - name: Run ${{ matrix.mode }} evals on ${{ matrix.model }}
         working-directory: ./ai_evals
         env:
           MODE: ${{ matrix.mode }}
+          MODEL: ${{ matrix.model }}
           WMILL_AI_EVAL_BACKEND_URL: http://localhost:8000
           WMILL_AI_EVAL_BACKEND_WORKSPACE: integration-tests
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
@@ -131,14 +136,10 @@ jobs:
         run: |
           bun install
           mkdir -p results
-          models="$MODELS"
-          [ "$MODE" = global ] && models="$models $GLOBAL_EXTRA_MODELS"
           reasoning=()
           [ -n "$REASONING" ] && [ "$MODE" != cli ] && reasoning=(--reasoning "$REASONING")
-          for m in $models; do
-            bun run cli -- run "$MODE" --model "$m" --runs "$RUNS" "${reasoning[@]}" \
-              --output "$PWD/results/$MODE-$m.json" || echo "::warning::$MODE on $m errored"
-          done
+          bun run cli -- run "$MODE" --model "$MODEL" --runs "$RUNS" "${reasoning[@]}" \
+            --output "$PWD/results/$MODE-$MODEL.json" || echo "::warning::$MODE on $MODEL errored"
 
       - name: Post results to the dashboard
         if: always()
@@ -166,7 +167,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: ai-evals-${{ matrix.mode }}
+          name: ai-evals-${{ matrix.mode }}-${{ matrix.model }}
           path: |
             ai_evals/logs
             ai_evals/results

ai_evals/adapters/frontend/vitestAdapter.test.ts

@@ -6,6 +6,8 @@ import { dirname, resolve } from 'path'
 // @ts-ignore - Node.js url
 import { fileURLToPath } from 'url'
 import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend'
+import { resolveWindmillBackendSettings } from '../../core/windmillBackendSettings'
+import { WindmillBackendClient } from './windmillBackend'
 
 const FRONTEND_DIR = fileURLToPath(new URL('../../../frontend/', import.meta.url))
 
@@ -28,6 +30,11 @@ globalThis.fetch = (async (input: unknown, init?: RequestInit) => {
 	if (typeof url === 'string' && hasBenchmarkApiHandler(url)) {
 		return handleBenchmarkApiFetch(url, init)
 	}
+	// The docs tools read the documentation bundled into the backend: hand them to the
+	// benchmark's real backend, logged in like the AI proxy calls.
+	if (typeof url === 'string' && url.startsWith('/api/docs/')) {
+		return new WindmillBackendClient(resolveWindmillBackendSettings()).request(url.slice('/api'.length), init)
+	}
 	// The parsers behind inferArgs load their wasm from a vite `?url` path, which only a dev
 	// server serves. Unserved, every script schema infers as empty, and the tools then tell the
 	// model its arguments are undeclared, so it rewrites a correct script until it runs out of turns.
@@ -684,6 +691,7 @@ benchmarkIt(
 			resetBenchmarkMockBackend()
 		}
 	},
-	// Full-suite runs (30+ cases at concurrency 2-3) routinely exceed 10 minutes.
-	7_200_000
+	// A full suite on a slow reasoning model runs for hours (gpt-6-astra on global went past
+	// 2h); the CI job's own timeout is the real bound.
+	18_000_000
 )