starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-19 16:03:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into oidc-external-signer

Browse Source 
# Conflicts:
#	backend/ee-repo-ref.txt
This commit is contained in:
Ruben Fiszel
2026-10-08 12:29:02 +02:00
parent aff2e23c7f d505916609
commit 3b3dfc59f8
388 changed files with 13888 additions and 6152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ai-evals-scheduled.yml
+22 -7
View File
@@ -15,8 +15,8 @@ on:
description: "Space-separated modes (global, cli, flow, app, script)"
default: "global cli"
models:
description: "Space-separated model aliases (bun run cli -- models); cli runs only the Claude ones"
default: "sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro"
description: "Space-separated model aliases (bun run cli -- models); cli skips those it has no agent for"
default: "sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro"
runs:
description: "Runs per case"
default: "3"
@@ -28,7 +28,7 @@ concurrency:
group: ai-evals-scheduled-${{ github.ref }}
env:
MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro' }}
MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro' }}
RUNS: ${{ inputs.runs || '3' }}
REASONING: ${{ inputs.reasoning }}
# Routes on the cloud instance are prefixed with their workspace.
@@ -40,17 +40,28 @@ jobs:
outputs:
matrix: ${{ steps.matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v4
with:
sparse-checkout: ai_evals/core
- uses: oven-sh/setup-bun@v2
with:
bun-version: 1.4.0
- id: matrix
env:
MODES: ${{ inputs.modes || 'global cli' }}
# cli drives the Claude Agent SDK, so it only takes the Claude aliases
# (`cli` entries in ai_evals/core/models.ts).
# cli takes only the aliases with a `cli` entry in ai_evals/core/models.ts.
run: |
echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" '
cli_models=$(bun -e '
import { EVAL_MODELS } from "./ai_evals/core/models.ts";
console.log(JSON.stringify(EVAL_MODELS.filter((m) => m.cli)
.flatMap((m) => [m.id, ...m.aliases]).map((a) => a.toLowerCase())));')
echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" --argjson cli "$cli_models" '
def words: split(" ") | map(select(. != ""));
[ $modes | words[] | select(IN("global", "flow", "app", "script", "cli")) as $mode
| $models | words[]
| select($mode != "cli" or test("^(claude-)?(haiku|sonnet|opus)"; "i"))
| select($mode != "cli" or (ascii_downcase | IN($cli[])))
| {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT"
evals:
@@ -120,6 +131,10 @@ jobs:
working-directory: ./cli
run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh
- name: Install Codex CLI
if: matrix.mode == 'cli'
run: npm install --global @openai/codex@0.159.3
- name: Run ${{ matrix.mode }} evals on ${{ matrix.model }}
working-directory: ./ai_evals
env: