starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-18 16:03:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

run the cli ai evals on codex and deepseek (#11591)

Browse Source 
* feat: run the cli ai evals on codex and deepseek

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* ci: pick cli eval models from the harness and install codex

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: stop counting wmill in a file name as a cli eval mutation

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: harden the codex and deepseek cli eval runtimes

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: keep codex tool order and keep other keys from deepseek

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
hugocasa
2026-10-07 14:27:40 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 0792332859
commit e50e1f5b31
11 changed files with 581 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ai-evals-scheduled.yml
+20 -5
View File
@@ -15,7 +15,7 @@ on:
description: "Space-separated modes (global, cli, flow, app, script)"
default: "global cli"
models:
description: "Space-separated model aliases (bun run cli -- models); cli runs only the Claude ones"
description: "Space-separated model aliases (bun run cli -- models); cli skips those it has no agent for"
default: "sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro"
runs:
description: "Runs per case"
@@ -40,17 +40,28 @@ jobs:
outputs:
matrix: ${{ steps.matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v4
with:
sparse-checkout: ai_evals/core
- uses: oven-sh/setup-bun@v2
with:
bun-version: 1.4.0
- id: matrix
env:
MODES: ${{ inputs.modes || 'global cli' }}
# cli drives the Claude Agent SDK, so it only takes the Claude aliases
# (`cli` entries in ai_evals/core/models.ts).
# cli takes only the aliases with a `cli` entry in ai_evals/core/models.ts.
run: |
echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" '
cli_models=$(bun -e '
import { EVAL_MODELS } from "./ai_evals/core/models.ts";
console.log(JSON.stringify(EVAL_MODELS.filter((m) => m.cli)
.flatMap((m) => [m.id, ...m.aliases]).map((a) => a.toLowerCase())));')
echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" --argjson cli "$cli_models" '
def words: split(" ") | map(select(. != ""));
[ $modes | words[] | select(IN("global", "flow", "app", "script", "cli")) as $mode
| $models | words[]
| select($mode != "cli" or test("^(claude-)?(haiku|sonnet|opus)"; "i"))
| select($mode != "cli" or (ascii_downcase | IN($cli[])))
| {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT"
evals:
@@ -120,6 +131,10 @@ jobs:
working-directory: ./cli
run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh
- name: Install Codex CLI
if: matrix.mode == 'cli'
run: npm install --global @openai/codex@0.159.3
- name: Run ${{ matrix.mode }} evals on ${{ matrix.model }}
working-directory: ./ai_evals
env: