From ed9f8d9fc4c47b2d51a94922aeb34870130261da Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 6 Oct 2026 14:46:08 +0200 Subject: [PATCH] run the weekly ai evals on global and cli across sonnet, opus, gpt-6 and deepseek (#11540) * feat: run the weekly ai evals on global and cli across sonnet, opus, gpt-6 and deepseek Co-Authored-By: Claude Opus 5.5 * fix: keep claude- prefixed and capitalised aliases in the cli model filter Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/ai-evals-scheduled.yml | 38 +++++++++++------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ai-evals-scheduled.yml b/.github/workflows/ai-evals-scheduled.yml index 36ef8e00a7..1284154a91 100644 --- a/.github/workflows/ai-evals-scheduled.yml +++ b/.github/workflows/ai-evals-scheduled.yml @@ -1,26 +1,22 @@ name: AI Evals (scheduled) -# Full ai_evals suites on current models, posted to the AI evals dashboard in -# the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is tracked -# over time. Weekly, since one pass of every suite at --runs 3 costs about 50M -# tokens per model: every suite runs on MODELS, and global (the mode users get) -# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Each mode and -# model is its own job, so a slow model neither delays nor shares a timeout with -# the others. Run it by hand to measure a branch against main. +# The global (AI chat) and cli suites on current models, posted to the AI evals +# dashboard in the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is +# tracked over time. Weekly: one global pass at --runs 3 is about 45M tokens per +# model. Each mode and model is its own job, so a slow model neither delays nor +# shares a timeout with the others. Run it by hand to measure a branch against +# main, or to include the flow, app and script suites. on: schedule: - cron: "0 3 * * 1" workflow_dispatch: inputs: modes: - description: "Space-separated modes" - default: "global flow app script cli" + description: "Space-separated modes (global, cli, flow, app, script)" + default: "global cli" models: - description: "Space-separated model aliases (bun run cli -- models)" - default: "sonnet-5.5" - global_extra_models: - description: "Extra model aliases for global mode only" - default: "gpt-6-astra gemini-3.8-flash" + description: "Space-separated model aliases (bun run cli -- models); cli runs only the Claude ones" + default: "sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro" runs: description: "Runs per case" default: "3" @@ -32,8 +28,7 @@ concurrency: group: ai-evals-scheduled-${{ github.ref }} env: - MODELS: ${{ inputs.models || 'sonnet-5.5' }} - GLOBAL_EXTRA_MODELS: ${{ inputs.global_extra_models || 'gpt-6-astra gemini-3.8-flash' }} + MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro' }} RUNS: ${{ inputs.runs || '3' }} REASONING: ${{ inputs.reasoning }} # Routes on the cloud instance are prefixed with their workspace. @@ -47,13 +42,16 @@ jobs: steps: - id: matrix env: - MODES: ${{ inputs.modes || 'global flow app script cli' }} + MODES: ${{ inputs.modes || 'global cli' }} + # cli drives the Claude Agent SDK, so it only takes the Claude aliases + # (`cli` entries in ai_evals/core/models.ts). run: | - echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" --arg extra "$GLOBAL_EXTRA_MODELS" ' + echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" ' def words: split(" ") | map(select(. != "")); [ $modes | words[] | select(IN("global", "flow", "app", "script", "cli")) as $mode - | ($models | words) + (if $mode == "global" then $extra | words else [] end) - | .[] | {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT" + | $models | words[] + | select($mode != "cli" or test("^(claude-)?(haiku|sonnet|opus)"; "i")) + | {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT" evals: needs: setup