From e50e1f5b311b2bdbbe90526a82486263c0749bd7 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 7 Oct 2026 16:27:40 +0200 Subject: [PATCH] run the cli ai evals on codex and deepseek (#11591) * feat: run the cli ai evals on codex and deepseek Co-Authored-By: Claude Opus 5.5 * ci: pick cli eval models from the harness and install codex Co-Authored-By: Claude Opus 5.5 * fix: stop counting wmill in a file name as a cli eval mutation Co-Authored-By: Claude Opus 5.5 * fix: harden the codex and deepseek cli eval runtimes Co-Authored-By: Claude Opus 5.5 * fix: keep codex tool order and keep other keys from deepseek Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/ai-evals-scheduled.yml | 25 ++- ai_evals/README.md | 16 +- ai_evals/adapters/cli/codex.test.ts | 71 +++++++ ai_evals/adapters/cli/codex.ts | 241 +++++++++++++++++++++++ ai_evals/adapters/cli/mutation.ts | 10 + ai_evals/adapters/cli/runtime.test.ts | 31 +++ ai_evals/adapters/cli/runtime.ts | 173 +++++++++------- ai_evals/core/cases.test.ts | 2 - ai_evals/core/models.test.ts | 14 +- ai_evals/core/models.ts | 52 ++++- ai_evals/modes/cli.ts | 41 +++- 11 files changed, 581 insertions(+), 95 deletions(-) create mode 100644 ai_evals/adapters/cli/codex.test.ts create mode 100644 ai_evals/adapters/cli/codex.ts create mode 100644 ai_evals/adapters/cli/mutation.ts diff --git a/.github/workflows/ai-evals-scheduled.yml b/.github/workflows/ai-evals-scheduled.yml index 1284154a91..4780eb02eb 100644 --- a/.github/workflows/ai-evals-scheduled.yml +++ b/.github/workflows/ai-evals-scheduled.yml @@ -15,7 +15,7 @@ on: description: "Space-separated modes (global, cli, flow, app, script)" default: "global cli" models: - description: "Space-separated model aliases (bun run cli -- models); cli runs only the Claude ones" + description: "Space-separated model aliases (bun run cli -- models); cli skips those it has no agent for" default: "sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro" runs: description: "Runs per case" @@ -40,17 +40,28 @@ jobs: outputs: matrix: ${{ steps.matrix.outputs.matrix }} steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: ai_evals/core + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.4.0 + - id: matrix env: MODES: ${{ inputs.modes || 'global cli' }} - # cli drives the Claude Agent SDK, so it only takes the Claude aliases - # (`cli` entries in ai_evals/core/models.ts). + # cli takes only the aliases with a `cli` entry in ai_evals/core/models.ts. run: | - echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" ' + cli_models=$(bun -e ' + import { EVAL_MODELS } from "./ai_evals/core/models.ts"; + console.log(JSON.stringify(EVAL_MODELS.filter((m) => m.cli) + .flatMap((m) => [m.id, ...m.aliases]).map((a) => a.toLowerCase())));') + echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" --argjson cli "$cli_models" ' def words: split(" ") | map(select(. != "")); [ $modes | words[] | select(IN("global", "flow", "app", "script", "cli")) as $mode | $models | words[] - | select($mode != "cli" or test("^(claude-)?(haiku|sonnet|opus)"; "i")) + | select($mode != "cli" or (ascii_downcase | IN($cli[]))) | {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT" evals: @@ -120,6 +131,10 @@ jobs: working-directory: ./cli run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh + - name: Install Codex CLI + if: matrix.mode == 'cli' + run: npm install --global @openai/codex@0.159.3 + - name: Run ${{ matrix.mode }} evals on ${{ matrix.model }} working-directory: ./ai_evals env: diff --git a/ai_evals/README.md b/ai_evals/README.md index 88623c7f33..44870ba2a2 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -89,7 +89,21 @@ Notes: - the command also prints accepted alias spellings such as `gpt-4o`, `gpt-55`, `claude-opus-4.6`, and `claude-haiku-4.5` - frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, Gemini, and DeepSeek-backed aliases -- `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there +- `cli` mode runs a coding agent over the workspace, chosen by the alias's `cli` entry; aliases without one are rejected there: + + | Agent | Models | Label | Needs | + | --- | --- | --- | --- | + | Claude Code (`@anthropic-ai/claude-agent-sdk`) | Claude aliases | `anthropic:` | `ANTHROPIC_API_KEY` | + | Claude Code on DeepSeek's Anthropic-compatible endpoint | `deepseek-v4-pro` | `claude-code:` | `DEEPSEEK_API_KEY` | + | Codex (`codex exec --json`) | `gpt-6.1-sol`, `gpt-6-astra` | `codex:` | `codex` on `PATH` (CI pins 0.159.3), `OPENAI_API_KEY` | + + Codex runs with a throwaway `HOME`, so your own `~/.codex` config and skills stay out of the run. It has no + Skill tool: it finds `.agents/skills` on its own and loads a skill by reading its `SKILL.md`, so the trace + records any command that touches `.agents/skills//SKILL.md` as a `Skill` call to ``, just + before that command, or just after it when the command mutates something before reading the skill. Its shell commands are recorded as `Bash` and its patches as `Edit`, so the + `invokes skill … before first mutation` checks compare the same things for both agents. Codex has no + turn cap; a run stops at `maxTurns` rounds of tool calls (parallel calls count once) and fails, as a + Claude Code run does at its cap. - the judge model is separate and currently defaults to `claude-sonnet-5-5`; use `--skip-judge` for deterministic-only runs ## Case Format diff --git a/ai_evals/adapters/cli/codex.test.ts b/ai_evals/adapters/cli/codex.test.ts new file mode 100644 index 0000000000..fdeda1d26b --- /dev/null +++ b/ai_evals/adapters/cli/codex.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "bun:test"; +import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runCodex, toToolInvocations, unwrapShellCommand } from "./codex"; + +describe("unwrapShellCommand", () => { + it("strips the login-shell wrapper Codex reports", () => { + expect(unwrapShellCommand(`/usr/bin/zsh -lc 'cat .agents/skills/x/SKILL.md'`)).toBe( + "cat .agents/skills/x/SKILL.md" + ); + expect(unwrapShellCommand(`/bin/bash -lc "printf 'hi' > \\"a b\\""`)).toBe( + `printf 'hi' > "a b"` + ); + expect(unwrapShellCommand(`/bin/bash -lc 'echo '\\''x'\\'''`)).toBe(`echo 'x'`); + expect(unwrapShellCommand(`bash -lc 'wmill sync push'`)).toBe("wmill sync push"); + expect(unwrapShellCommand(`/bin/bash -lc 'rg -g '"'"'!x'"'"' f'`)).toBe(`rg -g '!x' f`); + }); +}); + +describe("toToolInvocations", () => { + it("records reading a SKILL.md as a Skill call before the command", () => { + const tools = toToolInvocations({ + type: "command_execution", + command: `/bin/bash -lc 'sed -n 1,200p .agents/skills/write-script-bun/SKILL.md && mkdir f'`, + }); + expect(tools.map((tool) => [tool.tool, tool.input])).toEqual([ + ["Skill", { skill: "write-script-bun" }], + ["Bash", { command: "sed -n 1,200p .agents/skills/write-script-bun/SKILL.md && mkdir f" }], + ]); + }); + + it("records a skill read after a mutation earlier in the same command", () => { + const tools = toToolInvocations({ + type: "command_execution", + command: `/bin/bash -lc 'printf main > f/a.ts && cat .agents/skills/write-script-bun/SKILL.md'`, + }); + expect(tools.map((tool) => tool.tool)).toEqual(["Bash", "Skill"]); + }); + + it("maps a patch to Edit so it counts as a mutation", () => { + expect( + toToolInvocations({ type: "file_change", changes: [{ path: "a.ts", kind: "add" }] })[0] + ).toMatchObject({ tool: "Edit", input: { changes: [{ path: "a.ts", kind: "add" }] } }); + }); +}); + +describe("runCodex", () => { + it("keeps parallel tool calls in the order they started", async () => { + const dir = await mkdtemp(join(tmpdir(), "fake-codex-")); + const events = [ + { type: "item.started", item: { id: "a", type: "command_execution", command: "mkdir f" } }, + { type: "item.started", item: { id: "b", type: "command_execution", command: "cat .agents/skills/s/SKILL.md" } }, + { type: "item.completed", item: { id: "b", type: "command_execution", command: "cat .agents/skills/s/SKILL.md" } }, + { type: "item.completed", item: { id: "a", type: "command_execution", command: "mkdir f" } }, + { type: "item.completed", item: { id: "c", type: "agent_message", text: "done" } }, + { type: "turn.completed", usage: { input_tokens: 10, output_tokens: 2 } }, + ]; + const lines = events.map((event) => `printf '%s\\n' '${JSON.stringify(event)}'`).join("\n"); + await writeFile(join(dir, "codex"), `#!/bin/sh\ncat > /dev/null\n${lines}\n`); + await chmod(join(dir, "codex"), 0o755); + try { + const result = await runCodex("p", dir, 12, "m", { PATH: `${dir}:${process.env.PATH}` }); + expect(result.toolsUsed.map((tool) => tool.tool)).toEqual(["Bash", "Skill", "Bash"]); + expect(result.output).toBe("done"); + expect(result.tokenUsage).toEqual({ prompt: 10, completion: 2, total: 12 }); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/ai_evals/adapters/cli/codex.ts b/ai_evals/adapters/cli/codex.ts new file mode 100644 index 0000000000..b49c441556 --- /dev/null +++ b/ai_evals/adapters/cli/codex.ts @@ -0,0 +1,241 @@ +import { spawn } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createInterface } from "node:readline"; +import type { BenchmarkTokenUsage, CliToolInvocation } from "../../core/types"; +import { isLikelyMutatingBashCommand } from "./mutation"; + +export interface AgentRunResult { + output: string; + toolsUsed: CliToolInvocation[]; + assistantMessageCount: number; + tokenUsage: BenchmarkTokenUsage | null; + finalContextTokens: number | null; +} + +interface CodexItem { + type?: string; + text?: string; + command?: string; + changes?: { path?: string; kind?: string }[]; + [key: string]: unknown; +} + +interface CodexEvent { + type?: string; + item?: CodexItem; + usage?: { input_tokens?: number; output_tokens?: number }; + message?: string; + error?: { message?: string }; +} + +const SKILL_FILE_PATTERN = /\.agents\/skills\/([^/\s'"`*?[]+)\/SKILL\.md/g; + +// Drives `codex exec` and maps its events onto the Claude Code tool vocabulary +// the validators read: shell commands become `Bash`, patches `Edit`, and +// reading `.agents/skills//SKILL.md` (how Codex loads a skill) becomes a +// `Skill` call recorded just before that command. +export async function runCodex( + prompt: string, + cwd: string, + maxTurns: number, + model: string, + env: Record +): Promise { + // A fresh HOME keeps the user's ~/.codex config and ~/.agents/skills out of the run. + const home = await mkdtemp(join(tmpdir(), "wmill-cli-benchmark-codex-home-")); + const result: AgentRunResult = { + output: "", + toolsUsed: [], + assistantMessageCount: 0, + tokenUsage: null, + finalContextTokens: null, + }; + const messages: string[] = []; + let failure: string | null = null; + let lastError: string | null = null; + let stderr = ""; + + try { + const child = spawn( + "codex", + [ + "exec", + "--json", + "--ephemeral", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--model", + model, + "--cd", + cwd, + "-", + ], + { + cwd, + env: { + ...env, + HOME: home, + CODEX_HOME: home, + CODEX_API_KEY: env.OPENAI_API_KEY ?? "", + }, + stdio: ["pipe", "pipe", "pipe"], + } + ); + // Without this, a missing `codex` binary leaves the stdout loop waiting forever. + await new Promise((resolve, reject) => { + child.once("spawn", resolve); + child.once("error", reject); + }); + child.stdin.on("error", () => {}); + child.stdin.end(prompt); + child.stderr.on("data", (chunk) => { + stderr = (stderr + chunk).slice(-4000); + }); + const exited = new Promise((resolve, reject) => { + child.on("error", reject); + child.on("close", resolve); + }); + + // Codex has no turn cap. A turn starts when the agent issues tool calls with + // none in flight, so parallel calls count once, as they do for Claude Code. + // Claude Code's cap includes the final reply, so the maxTurns-th tool round is + // already over it. + // ponytail: parallel calls that finish before the next one starts count as + // separate turns; count model calls if Codex ever reports them. + let turns = 0; + let inFlight = 0; + let capped = false; + // Parallel calls complete out of order; the trace keeps the order they started in. + const slots = new Map(); + const ordered: CliToolInvocation[][] = []; + const slotFor = (item: CodexItem) => { + const id = typeof item.id === "string" ? item.id : undefined; + if (id !== undefined && slots.has(id)) { + return slots.get(id)!; + } + ordered.push([]); + if (id !== undefined) { + slots.set(id, ordered.length - 1); + } + return ordered.length - 1; + }; + + for await (const line of createInterface({ input: child.stdout })) { + const event = parseEvent(line); + if (!event) { + continue; + } + const item = event.item; + + if (event.type === "item.started" && item && isToolItem(item)) { + if (inFlight === 0) { + turns += 1; + } + inFlight += 1; + slotFor(item); + if (turns >= maxTurns && !capped) { + capped = true; + child.kill("SIGTERM"); + } + } else if (event.type === "item.completed" && item) { + if (item.type === "agent_message") { + result.assistantMessageCount += 1; + if (typeof item.text === "string") { + messages.push(item.text); + } + } else if (isToolItem(item)) { + inFlight = Math.max(0, inFlight - 1); + ordered[slotFor(item)] = toToolInvocations(item); + } + } else if (event.type === "turn.completed" && event.usage) { + const prompt = event.usage.input_tokens ?? 0; + const completion = event.usage.output_tokens ?? 0; + result.tokenUsage = { prompt, completion, total: prompt + completion }; + } else if (event.type === "turn.failed") { + failure = event.error?.message ?? JSON.stringify(event); + } else if (event.type === "error") { + // Also sent for retries Codex recovers from ("Reconnecting... 1/5"). + lastError = event.message ?? JSON.stringify(event); + } + } + + result.toolsUsed = ordered.flat(); + const code = await exited; + if (failure) { + throw new Error(`codex: ${failure}`); + } + // Fail the attempt, as Claude Code does at its cap. + if (capped) { + throw new Error(`codex: reached maximum number of turns (${maxTurns})`); + } + if (code !== 0) { + throw new Error(`codex exited with ${code}: ${lastError ?? stderr.trim()}`); + } + } finally { + await rm(home, { recursive: true, force: true }); + } + + result.output = messages.join("\n"); + return result; +} + +export function toToolInvocations(item: CodexItem): CliToolInvocation[] { + const timestamp = Date.now(); + + if (item.type === "command_execution" && typeof item.command === "string") { + const command = unwrapShellCommand(item.command); + // A skill read counts as preceding the command only when nothing before it in + // the command mutates; otherwise `printf x > f && cat SKILL.md` would pass a + // skill-before-first-mutation check. + const before: string[] = []; + const after: string[] = []; + for (const match of command.matchAll(SKILL_FILE_PATTERN)) { + const skill = match[1]!; + if (before.includes(skill) || after.includes(skill)) { + continue; + } + (isLikelyMutatingBashCommand(command.slice(0, match.index)) ? after : before).push(skill); + } + const skillCall = (skill: string) => ({ tool: "Skill", input: { skill }, timestamp }); + return [ + ...before.map(skillCall), + { tool: "Bash", input: { command }, timestamp }, + ...after.map(skillCall), + ]; + } + + if (item.type === "file_change") { + return [{ tool: "Edit", input: { changes: item.changes ?? [] }, timestamp }]; + } + + const { id: _id, type, status: _status, ...input } = item; + return [{ tool: String(type), input, timestamp }]; +} + +// Codex reports commands as run through the user's login shell, e.g. +// `/usr/bin/zsh -lc 'cat SKILL.md'`. +export function unwrapShellCommand(command: string): string { + const match = command.match(/^(?:\S*\/)?(?:ba|z)?sh -l?c (['"])([\s\S]*)\1$/); + if (!match) { + return command; + } + const body = match[2]!; + return match[1] === "'" + ? body.replaceAll(`'\\''`, "'").replaceAll(`'"'"'`, "'") + : body.replace(/\\(["\\$`])/g, "$1"); +} + +function isToolItem(item: CodexItem): boolean { + return !["agent_message", "reasoning", "todo_list", "error"].includes(item.type ?? ""); +} + +function parseEvent(line: string): CodexEvent | null { + try { + const value = JSON.parse(line); + return value && typeof value === "object" ? (value as CodexEvent) : null; + } catch { + return null; + } +} diff --git a/ai_evals/adapters/cli/mutation.ts b/ai_evals/adapters/cli/mutation.ts new file mode 100644 index 0000000000..05b29d68d7 --- /dev/null +++ b/ai_evals/adapters/cli/mutation.ts @@ -0,0 +1,10 @@ +export function isLikelyMutatingBashCommand(command: string): boolean { + return ( + /\b(?:mkdir|touch|rm|mv|cp|install|tee)\b/.test(command) || + /\b(?:cat|echo|printf)\b.*(?:>|>>|\|\s*tee\b)/.test(command) || + /\bsed\s+-i\b/.test(command) || + /\bperl\s+-pi\b/.test(command) || + // In command position only: `cat AGENTS.wmill.md` is a read. + /(?:^|[;&|(]|\b(?:then|do|xargs)\s)\s*wmill(?:\s|$)/m.test(command) + ); +} diff --git a/ai_evals/adapters/cli/runtime.test.ts b/ai_evals/adapters/cli/runtime.test.ts index 3ea53ca587..d3e537b659 100644 --- a/ai_evals/adapters/cli/runtime.test.ts +++ b/ai_evals/adapters/cli/runtime.test.ts @@ -2,7 +2,9 @@ import { describe, expect, it } from "bun:test"; import { anthropicUsageToBenchmarkTokenUsage, extractCliResultTokenUsage, + endpointEnv, extractProposedWmillCommands, + getFirstMutationToolIndex, parseWmillInvocationLog, } from "./runtime"; @@ -147,3 +149,32 @@ push ]); }); }); + +describe("getFirstMutationToolIndex", () => { + it("counts wmill only as a command, not in a file name", () => { + const bash = (command: string) => ({ tool: "Bash", input: { command }, timestamp: 0 }); + expect( + getFirstMutationToolIndex([ + bash("cat AGENTS.wmill.md .agents/skills/x/SKILL.md"), + bash("ls && wmill generate-metadata"), + ]) + ).toBe(1); + expect(getFirstMutationToolIndex([bash("cd f\nwmill sync push")])).toBe(0); + }); +}); + +describe("endpointEnv", () => { + it("passes the endpoint only its own key", () => { + const env = endpointEnv( + { baseUrl: "https://example.test", apiKeyEnv: "DEEPSEEK_API_KEY" }, + "m", + { PATH: "/bin", DEEPSEEK_API_KEY: "ds", OPENAI_API_KEY: "oa", ANTHROPIC_AUTH_TOKEN: "t" } + ); + expect(env).toEqual({ + PATH: "/bin", + ANTHROPIC_BASE_URL: "https://example.test", + ANTHROPIC_API_KEY: "ds", + ANTHROPIC_DEFAULT_HAIKU_MODEL: "m", + }); + }); +}); diff --git a/ai_evals/adapters/cli/runtime.ts b/ai_evals/adapters/cli/runtime.ts index b45df70011..e7368db366 100644 --- a/ai_evals/adapters/cli/runtime.ts +++ b/ai_evals/adapters/cli/runtime.ts @@ -2,7 +2,14 @@ import { query, type Options } from "@anthropic-ai/claude-agent-sdk"; import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; import { delimiter, join } from "path"; import { fileURLToPath } from "url"; -import { getCliEvalModel, resolveEvalModel, type CliEvalModelConfig } from "../../core/models"; +import { + getCliEvalModel, + resolveEvalModel, + type CliEvalModelConfig, +} from "../../core/models"; +import { runCodex, type AgentRunResult } from "./codex"; +import { isLikelyMutatingBashCommand } from "./mutation"; +export { formatCliRunModelLabel } from "../../core/models"; import type { BenchmarkTokenUsage, CliToolInvocation, @@ -141,16 +148,94 @@ export async function runPromptAndCapture( maxTurns: number = 3, modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL ): Promise { - const toolsUsed: ToolInvocation[] = []; - const skillsInvoked: string[] = []; - const bashCommands: string[] = []; - let output = ""; - let assistantMessageCount = 0; - let tokenUsage: BenchmarkTokenUsage | null = null; - let finalContextTokens: number | null = null; const startedAt = Date.now(); const stubBinDir = join(cwd, WMILL_STUB_DIR_NAME); const wmillLogPath = join(cwd, WMILL_LOG_FILE_NAME); + const env = { + ...getQueryEnv(), + PATH: process.env.PATH ? `${stubBinDir}${delimiter}${process.env.PATH}` : stubBinDir, + WMILL_BENCHMARK_LOG_PATH: wmillLogPath, + }; + + await installWmillStub(stubBinDir); + + const run = + modelConfig.runtime === "codex" + ? await runCodex(prompt, cwd, maxTurns, modelConfig.model, env) + : await runClaudeCode(prompt, cwd, maxTurns, modelConfig, env); + + const skillsInvoked: string[] = []; + const bashCommands: string[] = []; + for (const tool of run.toolsUsed) { + if (tool.tool === "Skill" && typeof tool.input.skill === "string") { + pushUnique(skillsInvoked, tool.input.skill); + } + if (tool.tool === "Bash") { + for (const command of extractBashCommands(tool.input)) { + pushUnique(bashCommands, command); + } + } + } + + const wmillInvocations = await readWmillInvocationLog(wmillLogPath); + + return { + output: run.output, + durationMs: Date.now() - startedAt, + tokenUsage: run.tokenUsage, + finalContextTokens: run.finalContextTokens, + trace: { + toolsUsed: run.toolsUsed, + skillsInvoked, + assistantMessageCount: run.assistantMessageCount, + bashCommands, + proposedCommands: extractProposedWmillCommands(run.output), + executedWmillCommands: wmillInvocations.map(formatExecutedWmillCommand), + wmillInvocations, + firstMutationToolIndex: getFirstMutationToolIndex(run.toolsUsed), + }, + }; +} + +// Only the endpoint's own key may reach it: a missing key would make Claude Code +// fall back to the user's Claude login, and the agent's Bash inherits this env, +// so any other credential could end up in a tool result sent to the endpoint. +export function endpointEnv( + endpoint: NonNullable, + model: string, + env: Record +): Record { + const key = env[endpoint.apiKeyEnv]; + if (!key) { + throw new Error(`${endpoint.apiKeyEnv} is not set`); + } + const rest = Object.fromEntries( + Object.entries(env).filter(([name]) => !/KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL/i.test(name)) + ); + return { + ...rest, + ANTHROPIC_BASE_URL: endpoint.baseUrl, + ANTHROPIC_API_KEY: key, + // Claude Code's background calls default to Haiku, which the endpoint does not serve. + ANTHROPIC_DEFAULT_HAIKU_MODEL: model, + }; +} + +async function runClaudeCode( + prompt: string, + cwd: string, + maxTurns: number, + modelConfig: CliEvalModelConfig, + env: Record +): Promise { + const result: AgentRunResult = { + output: "", + toolsUsed: [], + assistantMessageCount: 0, + tokenUsage: null, + finalContextTokens: null, + }; + const endpoint = modelConfig.anthropicEndpoint; const options: Options = { cwd, @@ -158,80 +243,42 @@ export async function runPromptAndCapture( maxTurns, settingSources: ["project"], allowedTools: ["Skill", "Read", "Glob", "Grep", "Bash", "Write", "Edit"], - env: { - ...getQueryEnv(), - PATH: process.env.PATH ? `${stubBinDir}${delimiter}${process.env.PATH}` : stubBinDir, - WMILL_BENCHMARK_LOG_PATH: wmillLogPath, - }, + env: endpoint ? endpointEnv(endpoint, modelConfig.model, env) : env, }; - await installWmillStub(stubBinDir); - for await (const message of query({ prompt, options })) { if (message.type === "assistant") { - assistantMessageCount += 1; + result.assistantMessageCount += 1; const turnContext = anthropicUsageToBenchmarkTokenUsage( message.message?.usage )?.prompt; if (turnContext && turnContext > 0) { - finalContextTokens = turnContext; + result.finalContextTokens = turnContext; } const content = message.message?.content; if (Array.isArray(content)) { for (const block of content) { if (block.type === "tool_use") { - const input = normalizeToolInput(block.input); - toolsUsed.push({ + result.toolsUsed.push({ tool: block.name, - input, + input: normalizeToolInput(block.input), timestamp: Date.now() }); - - if (block.name === "Skill") { - const skillInput = input as { skill?: string }; - if (skillInput.skill) { - pushUnique(skillsInvoked, skillInput.skill); - } - } - - if (block.name === "Bash") { - for (const command of extractBashCommands(input)) { - pushUnique(bashCommands, command); - } - } } else if (block.type === "text") { - output += block.text; + result.output += block.text; } } } } else if (message.type === "result") { const resultMessage = message as { result?: string }; - tokenUsage = extractCliResultTokenUsage(message) ?? tokenUsage; + result.tokenUsage = extractCliResultTokenUsage(message) ?? result.tokenUsage; if (typeof resultMessage.result === "string") { - output += resultMessage.result; + result.output += resultMessage.result; } } } - const proposedCommands = extractProposedWmillCommands(output); - const wmillInvocations = await readWmillInvocationLog(wmillLogPath); - - return { - output, - durationMs: Date.now() - startedAt, - tokenUsage, - finalContextTokens, - trace: { - toolsUsed, - skillsInvoked, - assistantMessageCount, - bashCommands, - proposedCommands, - executedWmillCommands: wmillInvocations.map(formatExecutedWmillCommand), - wmillInvocations, - firstMutationToolIndex: getFirstMutationToolIndex(toolsUsed), - }, - }; + return result; } export function wasSkillInvoked(result: PromptRunResult, skillName: string): boolean { @@ -242,10 +289,6 @@ export function wasToolUsed(result: PromptRunResult, toolName: string): boolean return result.trace.toolsUsed.some((tool) => tool.tool === toolName); } -export function formatCliRunModelLabel(modelConfig: CliEvalModelConfig): string { - return `${modelConfig.provider}:${modelConfig.model}`; -} - export function getToolInputs( result: PromptRunResult, toolName: string @@ -512,7 +555,7 @@ function formatExecutedWmillCommand(entry: CliWmillInvocation): string { return ["wmill", ...entry.argv].join(" ").trim(); } -function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null { +export function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null { for (const [index, tool] of toolsUsed.entries()) { if (tool.tool === "Write" || tool.tool === "Edit") { return index; @@ -526,16 +569,6 @@ function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null { return null; } -function isLikelyMutatingBashCommand(command: string): boolean { - return ( - /\b(?:mkdir|touch|rm|mv|cp|install|tee)\b/.test(command) || - /\b(?:cat|echo|printf)\b.*(?:>|>>|\|\s*tee\b)/.test(command) || - /\bsed\s+-i\b/.test(command) || - /\bperl\s+-pi\b/.test(command) || - /\bwmill\b/.test(command) - ); -} - function pushUnique(values: string[], value: string): void { if (!values.includes(value)) { values.push(value); diff --git a/ai_evals/core/cases.test.ts b/ai_evals/core/cases.test.ts index 5d6e3245db..b01d4143ba 100644 --- a/ai_evals/core/cases.test.ts +++ b/ai_evals/core/cases.test.ts @@ -192,8 +192,6 @@ describe("loadCases", () => { requiredSkills: ["write-script-bun"], requiredSkillsBeforeFirstMutation: ["write-script-bun"], forbiddenSkills: ["write-script-python3", "write-flow"], - orderedAssistantMentions: ["wmill generate-metadata", "wmill sync push"], - orderedProposedCommands: ["wmill generate-metadata", "wmill sync push"], forbiddenExecutedCommands: ["^wmill generate-metadata", "^wmill sync push"], }); }); diff --git a/ai_evals/core/models.test.ts b/ai_evals/core/models.test.ts index d51d61d224..96f16778d0 100644 --- a/ai_evals/core/models.test.ts +++ b/ai_evals/core/models.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "bun:test"; -import { resolveEvalModel } from "./models"; +import { formatRunModelLabel, resolveEvalModel } from "./models"; describe("resolveEvalModel", () => { it("supports GPT-5.5 aliases for frontend evals", () => { @@ -47,6 +47,18 @@ describe("resolveEvalModel", () => { }); }); + it("labels cli runs by agent, keeping Claude's historical label", () => { + expect(formatRunModelLabel("cli", resolveEvalModel("cli", "sonnet-5.5"))).toBe( + "anthropic:claude-sonnet-5-5", + ); + expect(formatRunModelLabel("cli", resolveEvalModel("cli", "gpt-6.1-sol"))).toBe( + "codex:gpt-6.1-sol", + ); + expect(formatRunModelLabel("cli", resolveEvalModel("cli", "deepseek-v4-pro"))).toBe( + "claude-code:deepseek-v4-pro", + ); + }); + it("rejects Gemini aliases for cli evals", () => { expect(() => resolveEvalModel("cli", "gemini-3-flash-preview")).toThrow( "Model gemini-3-flash-preview is not supported for cli mode", diff --git a/ai_evals/core/models.ts b/ai_evals/core/models.ts index 06689c36e1..22af20d5c3 100644 --- a/ai_evals/core/models.ts +++ b/ai_evals/core/models.ts @@ -5,9 +5,13 @@ export interface FrontendEvalModelConfig { model: string; } +// The coding agent that drives the model in cli mode (adapters/cli/runtime.ts). export interface CliEvalModelConfig { - provider: "anthropic"; + runtime: "claude-code" | "codex"; model: string; + // An Anthropic-compatible endpoint other than Anthropic's, for claude-code, + // and the env var holding its key. + anthropicEndpoint?: { baseUrl: string; apiKeyEnv: string }; } export interface EvalModelSpec { @@ -35,7 +39,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-haiku-4-5-20251001", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "haiku", }, }, @@ -55,7 +59,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-sonnet-4-5-20250929", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "sonnet", }, }, @@ -74,7 +78,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-opus-4-6", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "opus", }, }, @@ -87,7 +91,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-sonnet-5-5", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "claude-sonnet-5-5", }, }, @@ -100,7 +104,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-opus-5-5", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "claude-opus-5-5", }, }, @@ -139,6 +143,10 @@ export const EVAL_MODELS: EvalModelSpec[] = [ provider: "openai", model: "gpt-6-astra", }, + cli: { + runtime: "codex", + model: "gpt-6-astra", + }, }, { id: "gpt-6-sol", @@ -149,6 +157,19 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "gpt-6-sol", }, }, + { + id: "gpt-6.1-sol", + label: "GPT-6.1 Sol", + aliases: ["gpt-6.1-sol"], + frontend: { + provider: "openai", + model: "gpt-6.1-sol", + }, + cli: { + runtime: "codex", + model: "gpt-6.1-sol", + }, + }, { id: "gpt-6-luna", label: "GPT-6 Luna", @@ -215,6 +236,14 @@ export const EVAL_MODELS: EvalModelSpec[] = [ provider: "deepseek", model: "deepseek-v4-pro", }, + cli: { + runtime: "claude-code", + model: "deepseek-v4-pro", + anthropicEndpoint: { + baseUrl: "https://api.deepseek.com/anthropic", + apiKeyEnv: "DEEPSEEK_API_KEY", + }, + }, }, ]; @@ -253,11 +282,20 @@ export function formatRunModelLabel( model: EvalModelSpec, ): string { if (mode === "cli") { - return `${model.cli!.provider}:${model.cli!.model}`; + return formatCliRunModelLabel(model.cli!); } return `${model.frontend!.provider}:${model.frontend!.model}`; } +// Claude on Claude Code keeps the `anthropic:` prefix its dashboard history was +// recorded under; any other pairing names the agent. +export function formatCliRunModelLabel(config: CliEvalModelConfig): string { + if (config.runtime === "claude-code" && !config.anthropicEndpoint) { + return `anthropic:${config.model}`; + } + return `${config.runtime}:${config.model}`; +} + export function getFrontendEvalModel( model: EvalModelSpec, ): FrontendEvalModelConfig { diff --git a/ai_evals/modes/cli.ts b/ai_evals/modes/cli.ts index 205cdfa9c4..f337f99f40 100644 --- a/ai_evals/modes/cli.ts +++ b/ai_evals/modes/cli.ts @@ -39,17 +39,36 @@ interface CliRunActual { trace: CliTrace; } -const CLAUDE_PROJECT_PREAMBLE = [ - "Follow the project instructions from AGENTS.md exactly.", - "Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.", - "Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.", - "Do not skip the Skill step.", +// Only the skill wording differs per agent: Claude Code has a Skill tool, Codex +// loads a skill by reading its SKILL.md. +const SKILL_STEP: Record = { + "claude-code": [ + "Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.", + "Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.", + "Do not skip the Skill step.", + ], + codex: [ + "Before creating or modifying any Windmill entity, you MUST read the relevant skill's SKILL.md under .agents/skills and follow it.", + "Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.", + "Do not skip the skill step.", + ], +}; + +const HARNESS_PREAMBLE = [ "You are running inside an automated benchmark harness, not an interactive user session.", "Act autonomously and complete the requested file changes directly in the workspace.", "Do not ask for confirmation, do not ask the user to save or create files manually, and do not wait for approval.", "Do not respond with a plan when you can make the change directly.", "Only describe what was done after you have written the files.", -].join(" "); +]; + +function projectPreamble(runtime: CliEvalModelConfig["runtime"]): string { + return [ + "Follow the project instructions from AGENTS.md exactly.", + ...SKILL_STEP[runtime], + ...HARNESS_PREAMBLE, + ].join(" "); +} export function createCliModeRunner( modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL @@ -89,7 +108,7 @@ export function createCliModeRunner( }); await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8"); - const renderedPrompt = await renderPrompt(prompt, workspaceDir); + const renderedPrompt = await renderPrompt(prompt, workspaceDir, modelConfig.runtime); const run = await runPromptAndCapture( renderedPrompt, workspaceDir, @@ -193,7 +212,11 @@ export function getCliRunModelLabel( return formatCliRunModelLabel(modelConfig); } -async function renderPrompt(prompt: string, workspaceDir: string): Promise { +async function renderPrompt( + prompt: string, + workspaceDir: string, + runtime: CliEvalModelConfig["runtime"] +): Promise { const renderedUserPrompt = prompt.replaceAll("{{workspace_root}}", workspaceDir); const agentsInstructions = await readFile(path.join(workspaceDir, "AGENTS.md"), "utf8"); @@ -202,7 +225,7 @@ async function renderPrompt(prompt: string, workspaceDir: string): Promise