From 222dad1d8cc9cbb6925fdff709e00a49bb63bc4a Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 5 Oct 2026 16:43:41 +0200 Subject: [PATCH] fix: run each ai eval model in its own job and let the docs tools reach the backend (#11533) Co-authored-by: Claude Opus 5.5 --- .github/workflows/ai-evals-scheduled.yml | 31 ++++++++++--------- .../adapters/frontend/vitestAdapter.test.ts | 12 +++++-- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ai-evals-scheduled.yml b/.github/workflows/ai-evals-scheduled.yml index cc1c0d6763..36ef8e00a7 100644 --- a/.github/workflows/ai-evals-scheduled.yml +++ b/.github/workflows/ai-evals-scheduled.yml @@ -4,8 +4,9 @@ name: AI Evals (scheduled) # the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is tracked # over time. Weekly, since one pass of every suite at --runs 3 costs about 50M # tokens per model: every suite runs on MODELS, and global (the mode users get) -# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Run it by -# hand to measure a branch against main. +# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Each mode and +# model is its own job, so a slow model neither delays nor shares a timeout with +# the others. Run it by hand to measure a branch against main. on: schedule: - cron: "0 3 * * 1" @@ -42,14 +43,17 @@ jobs: setup: runs-on: ubuntu-latest outputs: - modes: ${{ steps.modes.outputs.modes }} + matrix: ${{ steps.matrix.outputs.matrix }} steps: - - id: modes + - id: matrix env: MODES: ${{ inputs.modes || 'global flow app script cli' }} run: | - echo "modes=$(jq -cn --arg m "$MODES" '$m | split(" ") - | map(select(IN("global", "flow", "app", "script", "cli")))')" >> "$GITHUB_OUTPUT" + echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" --arg extra "$GLOBAL_EXTRA_MODELS" ' + def words: split(" ") | map(select(. != "")); + [ $modes | words[] | select(IN("global", "flow", "app", "script", "cli")) as $mode + | ($models | words) + (if $mode == "global" then $extra | words else [] end) + | .[] | {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT" evals: needs: setup @@ -58,7 +62,7 @@ jobs: strategy: fail-fast: false matrix: - mode: ${{ fromJSON(needs.setup.outputs.modes) }} + include: ${{ fromJSON(needs.setup.outputs.matrix) }} services: postgres: image: postgres:16 @@ -118,10 +122,11 @@ jobs: working-directory: ./cli run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh - - name: Run ${{ matrix.mode }} evals + - name: Run ${{ matrix.mode }} evals on ${{ matrix.model }} working-directory: ./ai_evals env: MODE: ${{ matrix.mode }} + MODEL: ${{ matrix.model }} WMILL_AI_EVAL_BACKEND_URL: http://localhost:8000 WMILL_AI_EVAL_BACKEND_WORKSPACE: integration-tests ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} @@ -131,14 +136,10 @@ jobs: run: | bun install mkdir -p results - models="$MODELS" - [ "$MODE" = global ] && models="$models $GLOBAL_EXTRA_MODELS" reasoning=() [ -n "$REASONING" ] && [ "$MODE" != cli ] && reasoning=(--reasoning "$REASONING") - for m in $models; do - bun run cli -- run "$MODE" --model "$m" --runs "$RUNS" "${reasoning[@]}" \ - --output "$PWD/results/$MODE-$m.json" || echo "::warning::$MODE on $m errored" - done + bun run cli -- run "$MODE" --model "$MODEL" --runs "$RUNS" "${reasoning[@]}" \ + --output "$PWD/results/$MODE-$MODEL.json" || echo "::warning::$MODE on $MODEL errored" - name: Post results to the dashboard if: always() @@ -166,7 +167,7 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: ai-evals-${{ matrix.mode }} + name: ai-evals-${{ matrix.mode }}-${{ matrix.model }} path: | ai_evals/logs ai_evals/results diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 9adb4d05ac..288c67e3eb 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -6,6 +6,8 @@ import { dirname, resolve } from 'path' // @ts-ignore - Node.js url import { fileURLToPath } from 'url' import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend' +import { resolveWindmillBackendSettings } from '../../core/windmillBackendSettings' +import { WindmillBackendClient } from './windmillBackend' const FRONTEND_DIR = fileURLToPath(new URL('../../../frontend/', import.meta.url)) @@ -28,6 +30,11 @@ globalThis.fetch = (async (input: unknown, init?: RequestInit) => { if (typeof url === 'string' && hasBenchmarkApiHandler(url)) { return handleBenchmarkApiFetch(url, init) } + // The docs tools read the documentation bundled into the backend: hand them to the + // benchmark's real backend, logged in like the AI proxy calls. + if (typeof url === 'string' && url.startsWith('/api/docs/')) { + return new WindmillBackendClient(resolveWindmillBackendSettings()).request(url.slice('/api'.length), init) + } // The parsers behind inferArgs load their wasm from a vite `?url` path, which only a dev // server serves. Unserved, every script schema infers as empty, and the tools then tell the // model its arguments are undeclared, so it rewrites a correct script until it runs out of turns. @@ -684,6 +691,7 @@ benchmarkIt( resetBenchmarkMockBackend() } }, - // Full-suite runs (30+ cases at concurrency 2-3) routinely exceed 10 minutes. - 7_200_000 + // A full suite on a slow reasoning model runs for hours (gpt-6-astra on global went past + // 2h); the CI job's own timeout is the real bound. + 18_000_000 )