starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-12 08:02:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

run the weekly ai evals on global and cli across sonnet, opus, gpt-6 and deepseek (#11540)

Browse Source 
* feat: run the weekly ai evals on global and cli across sonnet, opus, gpt-6 and deepseek

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: keep claude- prefixed and capitalised aliases in the cli model filter

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
hugocasa
2026-10-06 14:46:08 +02:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 3d4d80d4ca
commit ed9f8d9fc4
1 changed files with 18 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ai-evals-scheduled.yml
+18 -20
View File
@@ -1,26 +1,22 @@
name: AI Evals (scheduled)
# Full ai_evals suites on current models, posted to the AI evals dashboard in
# the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is tracked
# over time. Weekly, since one pass of every suite at --runs 3 costs about 50M
# tokens per model: every suite runs on MODELS, and global (the mode users get)
# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Each mode and
# model is its own job, so a slow model neither delays nor shares a timeout with
# the others. Run it by hand to measure a branch against main.
# The global (AI chat) and cli suites on current models, posted to the AI evals
# dashboard in the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is
# tracked over time. Weekly: one global pass at --runs 3 is about 45M tokens per
# model. Each mode and model is its own job, so a slow model neither delays nor
# shares a timeout with the others. Run it by hand to measure a branch against
# main, or to include the flow, app and script suites.
on:
schedule:
- cron: "0 3 * * 1"
workflow_dispatch:
inputs:
modes:
description: "Space-separated modes"
default: "global flow app script cli"
description: "Space-separated modes (global, cli, flow, app, script)"
default: "global cli"
models:
description: "Space-separated model aliases (bun run cli -- models)"
default: "sonnet-5.5"
global_extra_models:
description: "Extra model aliases for global mode only"
default: "gpt-6-astra gemini-3.8-flash"
description: "Space-separated model aliases (bun run cli -- models); cli runs only the Claude ones"
default: "sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro"
runs:
description: "Runs per case"
default: "3"
@@ -32,8 +28,7 @@ concurrency:
group: ai-evals-scheduled-${{ github.ref }}
env:
MODELS: ${{ inputs.models || 'sonnet-5.5' }}
GLOBAL_EXTRA_MODELS: ${{ inputs.global_extra_models || 'gpt-6-astra gemini-3.8-flash' }}
MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro' }}
RUNS: ${{ inputs.runs || '3' }}
REASONING: ${{ inputs.reasoning }}
# Routes on the cloud instance are prefixed with their workspace.
@@ -47,13 +42,16 @@ jobs:
steps:
- id: matrix
env:
MODES: ${{ inputs.modes || 'global flow app script cli' }}
MODES: ${{ inputs.modes || 'global cli' }}
# cli drives the Claude Agent SDK, so it only takes the Claude aliases
# (`cli` entries in ai_evals/core/models.ts).
run: |
echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" --arg extra "$GLOBAL_EXTRA_MODELS" '
echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" '
def words: split(" ") | map(select(. != ""));
[ $modes | words[] | select(IN("global", "flow", "app", "script", "cli")) as $mode
| ($models | words) + (if $mode == "global" then $extra | words else [] end)
| .[] | {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT"
| $models | words[]
| select($mode != "cli" or test("^(claude-)?(haiku|sonnet|opus)"; "i"))
| {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT"
evals:
needs: setup