mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-10-12 16:02:56 +00:00
run the cli ai evals on codex and deepseek (#11591)
* feat: run the cli ai evals on codex and deepseek Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * ci: pick cli eval models from the harness and install codex Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: stop counting wmill in a file name as a cli eval mutation Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: harden the codex and deepseek cli eval runtimes Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: keep codex tool order and keep other keys from deepseek Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
hugocasa
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
0792332859
commit
e50e1f5b31
@@ -15,7 +15,7 @@ on:
|
||||
description: "Space-separated modes (global, cli, flow, app, script)"
|
||||
default: "global cli"
|
||||
models:
|
||||
description: "Space-separated model aliases (bun run cli -- models); cli runs only the Claude ones"
|
||||
description: "Space-separated model aliases (bun run cli -- models); cli skips those it has no agent for"
|
||||
default: "sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro"
|
||||
runs:
|
||||
description: "Runs per case"
|
||||
@@ -40,17 +40,28 @@ jobs:
|
||||
outputs:
|
||||
matrix: ${{ steps.matrix.outputs.matrix }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
sparse-checkout: ai_evals/core
|
||||
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: 1.4.0
|
||||
|
||||
- id: matrix
|
||||
env:
|
||||
MODES: ${{ inputs.modes || 'global cli' }}
|
||||
# cli drives the Claude Agent SDK, so it only takes the Claude aliases
|
||||
# (`cli` entries in ai_evals/core/models.ts).
|
||||
# cli takes only the aliases with a `cli` entry in ai_evals/core/models.ts.
|
||||
run: |
|
||||
echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" '
|
||||
cli_models=$(bun -e '
|
||||
import { EVAL_MODELS } from "./ai_evals/core/models.ts";
|
||||
console.log(JSON.stringify(EVAL_MODELS.filter((m) => m.cli)
|
||||
.flatMap((m) => [m.id, ...m.aliases]).map((a) => a.toLowerCase())));')
|
||||
echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" --argjson cli "$cli_models" '
|
||||
def words: split(" ") | map(select(. != ""));
|
||||
[ $modes | words[] | select(IN("global", "flow", "app", "script", "cli")) as $mode
|
||||
| $models | words[]
|
||||
| select($mode != "cli" or test("^(claude-)?(haiku|sonnet|opus)"; "i"))
|
||||
| select($mode != "cli" or (ascii_downcase | IN($cli[])))
|
||||
| {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
evals:
|
||||
@@ -120,6 +131,10 @@ jobs:
|
||||
working-directory: ./cli
|
||||
run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh
|
||||
|
||||
- name: Install Codex CLI
|
||||
if: matrix.mode == 'cli'
|
||||
run: npm install --global @openai/codex@0.159.3
|
||||
|
||||
- name: Run ${{ matrix.mode }} evals on ${{ matrix.model }}
|
||||
working-directory: ./ai_evals
|
||||
env:
|
||||
|
||||
+15
-1
@@ -89,7 +89,21 @@ Notes:
|
||||
|
||||
- the command also prints accepted alias spellings such as `gpt-4o`, `gpt-55`, `claude-opus-4.6`, and `claude-haiku-4.5`
|
||||
- frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, Gemini, and DeepSeek-backed aliases
|
||||
- `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there
|
||||
- `cli` mode runs a coding agent over the workspace, chosen by the alias's `cli` entry; aliases without one are rejected there:
|
||||
|
||||
| Agent | Models | Label | Needs |
|
||||
| --- | --- | --- | --- |
|
||||
| Claude Code (`@anthropic-ai/claude-agent-sdk`) | Claude aliases | `anthropic:<model>` | `ANTHROPIC_API_KEY` |
|
||||
| Claude Code on DeepSeek's Anthropic-compatible endpoint | `deepseek-v4-pro` | `claude-code:<model>` | `DEEPSEEK_API_KEY` |
|
||||
| Codex (`codex exec --json`) | `gpt-6.1-sol`, `gpt-6-astra` | `codex:<model>` | `codex` on `PATH` (CI pins 0.159.3), `OPENAI_API_KEY` |
|
||||
|
||||
Codex runs with a throwaway `HOME`, so your own `~/.codex` config and skills stay out of the run. It has no
|
||||
Skill tool: it finds `.agents/skills` on its own and loads a skill by reading its `SKILL.md`, so the trace
|
||||
records any command that touches `.agents/skills/<name>/SKILL.md` as a `Skill` call to `<name>`, just
|
||||
before that command, or just after it when the command mutates something before reading the skill. Its shell commands are recorded as `Bash` and its patches as `Edit`, so the
|
||||
`invokes skill … before first mutation` checks compare the same things for both agents. Codex has no
|
||||
turn cap; a run stops at `maxTurns` rounds of tool calls (parallel calls count once) and fails, as a
|
||||
Claude Code run does at its cap.
|
||||
- the judge model is separate and currently defaults to `claude-sonnet-5-5`; use `--skip-judge` for deterministic-only runs
|
||||
|
||||
## Case Format
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { runCodex, toToolInvocations, unwrapShellCommand } from "./codex";
|
||||
|
||||
describe("unwrapShellCommand", () => {
|
||||
it("strips the login-shell wrapper Codex reports", () => {
|
||||
expect(unwrapShellCommand(`/usr/bin/zsh -lc 'cat .agents/skills/x/SKILL.md'`)).toBe(
|
||||
"cat .agents/skills/x/SKILL.md"
|
||||
);
|
||||
expect(unwrapShellCommand(`/bin/bash -lc "printf 'hi' > \\"a b\\""`)).toBe(
|
||||
`printf 'hi' > "a b"`
|
||||
);
|
||||
expect(unwrapShellCommand(`/bin/bash -lc 'echo '\\''x'\\'''`)).toBe(`echo 'x'`);
|
||||
expect(unwrapShellCommand(`bash -lc 'wmill sync push'`)).toBe("wmill sync push");
|
||||
expect(unwrapShellCommand(`/bin/bash -lc 'rg -g '"'"'!x'"'"' f'`)).toBe(`rg -g '!x' f`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("toToolInvocations", () => {
|
||||
it("records reading a SKILL.md as a Skill call before the command", () => {
|
||||
const tools = toToolInvocations({
|
||||
type: "command_execution",
|
||||
command: `/bin/bash -lc 'sed -n 1,200p .agents/skills/write-script-bun/SKILL.md && mkdir f'`,
|
||||
});
|
||||
expect(tools.map((tool) => [tool.tool, tool.input])).toEqual([
|
||||
["Skill", { skill: "write-script-bun" }],
|
||||
["Bash", { command: "sed -n 1,200p .agents/skills/write-script-bun/SKILL.md && mkdir f" }],
|
||||
]);
|
||||
});
|
||||
|
||||
it("records a skill read after a mutation earlier in the same command", () => {
|
||||
const tools = toToolInvocations({
|
||||
type: "command_execution",
|
||||
command: `/bin/bash -lc 'printf main > f/a.ts && cat .agents/skills/write-script-bun/SKILL.md'`,
|
||||
});
|
||||
expect(tools.map((tool) => tool.tool)).toEqual(["Bash", "Skill"]);
|
||||
});
|
||||
|
||||
it("maps a patch to Edit so it counts as a mutation", () => {
|
||||
expect(
|
||||
toToolInvocations({ type: "file_change", changes: [{ path: "a.ts", kind: "add" }] })[0]
|
||||
).toMatchObject({ tool: "Edit", input: { changes: [{ path: "a.ts", kind: "add" }] } });
|
||||
});
|
||||
});
|
||||
|
||||
describe("runCodex", () => {
|
||||
it("keeps parallel tool calls in the order they started", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "fake-codex-"));
|
||||
const events = [
|
||||
{ type: "item.started", item: { id: "a", type: "command_execution", command: "mkdir f" } },
|
||||
{ type: "item.started", item: { id: "b", type: "command_execution", command: "cat .agents/skills/s/SKILL.md" } },
|
||||
{ type: "item.completed", item: { id: "b", type: "command_execution", command: "cat .agents/skills/s/SKILL.md" } },
|
||||
{ type: "item.completed", item: { id: "a", type: "command_execution", command: "mkdir f" } },
|
||||
{ type: "item.completed", item: { id: "c", type: "agent_message", text: "done" } },
|
||||
{ type: "turn.completed", usage: { input_tokens: 10, output_tokens: 2 } },
|
||||
];
|
||||
const lines = events.map((event) => `printf '%s\\n' '${JSON.stringify(event)}'`).join("\n");
|
||||
await writeFile(join(dir, "codex"), `#!/bin/sh\ncat > /dev/null\n${lines}\n`);
|
||||
await chmod(join(dir, "codex"), 0o755);
|
||||
try {
|
||||
const result = await runCodex("p", dir, 12, "m", { PATH: `${dir}:${process.env.PATH}` });
|
||||
expect(result.toolsUsed.map((tool) => tool.tool)).toEqual(["Bash", "Skill", "Bash"]);
|
||||
expect(result.output).toBe("done");
|
||||
expect(result.tokenUsage).toEqual({ prompt: 10, completion: 2, total: 12 });
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,241 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import type { BenchmarkTokenUsage, CliToolInvocation } from "../../core/types";
|
||||
import { isLikelyMutatingBashCommand } from "./mutation";
|
||||
|
||||
export interface AgentRunResult {
|
||||
output: string;
|
||||
toolsUsed: CliToolInvocation[];
|
||||
assistantMessageCount: number;
|
||||
tokenUsage: BenchmarkTokenUsage | null;
|
||||
finalContextTokens: number | null;
|
||||
}
|
||||
|
||||
interface CodexItem {
|
||||
type?: string;
|
||||
text?: string;
|
||||
command?: string;
|
||||
changes?: { path?: string; kind?: string }[];
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
interface CodexEvent {
|
||||
type?: string;
|
||||
item?: CodexItem;
|
||||
usage?: { input_tokens?: number; output_tokens?: number };
|
||||
message?: string;
|
||||
error?: { message?: string };
|
||||
}
|
||||
|
||||
const SKILL_FILE_PATTERN = /\.agents\/skills\/([^/\s'"`*?[]+)\/SKILL\.md/g;
|
||||
|
||||
// Drives `codex exec` and maps its events onto the Claude Code tool vocabulary
|
||||
// the validators read: shell commands become `Bash`, patches `Edit`, and
|
||||
// reading `.agents/skills/<name>/SKILL.md` (how Codex loads a skill) becomes a
|
||||
// `Skill` call recorded just before that command.
|
||||
export async function runCodex(
|
||||
prompt: string,
|
||||
cwd: string,
|
||||
maxTurns: number,
|
||||
model: string,
|
||||
env: Record<string, string>
|
||||
): Promise<AgentRunResult> {
|
||||
// A fresh HOME keeps the user's ~/.codex config and ~/.agents/skills out of the run.
|
||||
const home = await mkdtemp(join(tmpdir(), "wmill-cli-benchmark-codex-home-"));
|
||||
const result: AgentRunResult = {
|
||||
output: "",
|
||||
toolsUsed: [],
|
||||
assistantMessageCount: 0,
|
||||
tokenUsage: null,
|
||||
finalContextTokens: null,
|
||||
};
|
||||
const messages: string[] = [];
|
||||
let failure: string | null = null;
|
||||
let lastError: string | null = null;
|
||||
let stderr = "";
|
||||
|
||||
try {
|
||||
const child = spawn(
|
||||
"codex",
|
||||
[
|
||||
"exec",
|
||||
"--json",
|
||||
"--ephemeral",
|
||||
"--skip-git-repo-check",
|
||||
"--dangerously-bypass-approvals-and-sandbox",
|
||||
"--model",
|
||||
model,
|
||||
"--cd",
|
||||
cwd,
|
||||
"-",
|
||||
],
|
||||
{
|
||||
cwd,
|
||||
env: {
|
||||
...env,
|
||||
HOME: home,
|
||||
CODEX_HOME: home,
|
||||
CODEX_API_KEY: env.OPENAI_API_KEY ?? "",
|
||||
},
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
}
|
||||
);
|
||||
// Without this, a missing `codex` binary leaves the stdout loop waiting forever.
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
child.once("spawn", resolve);
|
||||
child.once("error", reject);
|
||||
});
|
||||
child.stdin.on("error", () => {});
|
||||
child.stdin.end(prompt);
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr = (stderr + chunk).slice(-4000);
|
||||
});
|
||||
const exited = new Promise<number | null>((resolve, reject) => {
|
||||
child.on("error", reject);
|
||||
child.on("close", resolve);
|
||||
});
|
||||
|
||||
// Codex has no turn cap. A turn starts when the agent issues tool calls with
|
||||
// none in flight, so parallel calls count once, as they do for Claude Code.
|
||||
// Claude Code's cap includes the final reply, so the maxTurns-th tool round is
|
||||
// already over it.
|
||||
// ponytail: parallel calls that finish before the next one starts count as
|
||||
// separate turns; count model calls if Codex ever reports them.
|
||||
let turns = 0;
|
||||
let inFlight = 0;
|
||||
let capped = false;
|
||||
// Parallel calls complete out of order; the trace keeps the order they started in.
|
||||
const slots = new Map<string, number>();
|
||||
const ordered: CliToolInvocation[][] = [];
|
||||
const slotFor = (item: CodexItem) => {
|
||||
const id = typeof item.id === "string" ? item.id : undefined;
|
||||
if (id !== undefined && slots.has(id)) {
|
||||
return slots.get(id)!;
|
||||
}
|
||||
ordered.push([]);
|
||||
if (id !== undefined) {
|
||||
slots.set(id, ordered.length - 1);
|
||||
}
|
||||
return ordered.length - 1;
|
||||
};
|
||||
|
||||
for await (const line of createInterface({ input: child.stdout })) {
|
||||
const event = parseEvent(line);
|
||||
if (!event) {
|
||||
continue;
|
||||
}
|
||||
const item = event.item;
|
||||
|
||||
if (event.type === "item.started" && item && isToolItem(item)) {
|
||||
if (inFlight === 0) {
|
||||
turns += 1;
|
||||
}
|
||||
inFlight += 1;
|
||||
slotFor(item);
|
||||
if (turns >= maxTurns && !capped) {
|
||||
capped = true;
|
||||
child.kill("SIGTERM");
|
||||
}
|
||||
} else if (event.type === "item.completed" && item) {
|
||||
if (item.type === "agent_message") {
|
||||
result.assistantMessageCount += 1;
|
||||
if (typeof item.text === "string") {
|
||||
messages.push(item.text);
|
||||
}
|
||||
} else if (isToolItem(item)) {
|
||||
inFlight = Math.max(0, inFlight - 1);
|
||||
ordered[slotFor(item)] = toToolInvocations(item);
|
||||
}
|
||||
} else if (event.type === "turn.completed" && event.usage) {
|
||||
const prompt = event.usage.input_tokens ?? 0;
|
||||
const completion = event.usage.output_tokens ?? 0;
|
||||
result.tokenUsage = { prompt, completion, total: prompt + completion };
|
||||
} else if (event.type === "turn.failed") {
|
||||
failure = event.error?.message ?? JSON.stringify(event);
|
||||
} else if (event.type === "error") {
|
||||
// Also sent for retries Codex recovers from ("Reconnecting... 1/5").
|
||||
lastError = event.message ?? JSON.stringify(event);
|
||||
}
|
||||
}
|
||||
|
||||
result.toolsUsed = ordered.flat();
|
||||
const code = await exited;
|
||||
if (failure) {
|
||||
throw new Error(`codex: ${failure}`);
|
||||
}
|
||||
// Fail the attempt, as Claude Code does at its cap.
|
||||
if (capped) {
|
||||
throw new Error(`codex: reached maximum number of turns (${maxTurns})`);
|
||||
}
|
||||
if (code !== 0) {
|
||||
throw new Error(`codex exited with ${code}: ${lastError ?? stderr.trim()}`);
|
||||
}
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
result.output = messages.join("\n");
|
||||
return result;
|
||||
}
|
||||
|
||||
export function toToolInvocations(item: CodexItem): CliToolInvocation[] {
|
||||
const timestamp = Date.now();
|
||||
|
||||
if (item.type === "command_execution" && typeof item.command === "string") {
|
||||
const command = unwrapShellCommand(item.command);
|
||||
// A skill read counts as preceding the command only when nothing before it in
|
||||
// the command mutates; otherwise `printf x > f && cat SKILL.md` would pass a
|
||||
// skill-before-first-mutation check.
|
||||
const before: string[] = [];
|
||||
const after: string[] = [];
|
||||
for (const match of command.matchAll(SKILL_FILE_PATTERN)) {
|
||||
const skill = match[1]!;
|
||||
if (before.includes(skill) || after.includes(skill)) {
|
||||
continue;
|
||||
}
|
||||
(isLikelyMutatingBashCommand(command.slice(0, match.index)) ? after : before).push(skill);
|
||||
}
|
||||
const skillCall = (skill: string) => ({ tool: "Skill", input: { skill }, timestamp });
|
||||
return [
|
||||
...before.map(skillCall),
|
||||
{ tool: "Bash", input: { command }, timestamp },
|
||||
...after.map(skillCall),
|
||||
];
|
||||
}
|
||||
|
||||
if (item.type === "file_change") {
|
||||
return [{ tool: "Edit", input: { changes: item.changes ?? [] }, timestamp }];
|
||||
}
|
||||
|
||||
const { id: _id, type, status: _status, ...input } = item;
|
||||
return [{ tool: String(type), input, timestamp }];
|
||||
}
|
||||
|
||||
// Codex reports commands as run through the user's login shell, e.g.
|
||||
// `/usr/bin/zsh -lc 'cat SKILL.md'`.
|
||||
export function unwrapShellCommand(command: string): string {
|
||||
const match = command.match(/^(?:\S*\/)?(?:ba|z)?sh -l?c (['"])([\s\S]*)\1$/);
|
||||
if (!match) {
|
||||
return command;
|
||||
}
|
||||
const body = match[2]!;
|
||||
return match[1] === "'"
|
||||
? body.replaceAll(`'\\''`, "'").replaceAll(`'"'"'`, "'")
|
||||
: body.replace(/\\(["\\$`])/g, "$1");
|
||||
}
|
||||
|
||||
function isToolItem(item: CodexItem): boolean {
|
||||
return !["agent_message", "reasoning", "todo_list", "error"].includes(item.type ?? "");
|
||||
}
|
||||
|
||||
function parseEvent(line: string): CodexEvent | null {
|
||||
try {
|
||||
const value = JSON.parse(line);
|
||||
return value && typeof value === "object" ? (value as CodexEvent) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
export function isLikelyMutatingBashCommand(command: string): boolean {
|
||||
return (
|
||||
/\b(?:mkdir|touch|rm|mv|cp|install|tee)\b/.test(command) ||
|
||||
/\b(?:cat|echo|printf)\b.*(?:>|>>|\|\s*tee\b)/.test(command) ||
|
||||
/\bsed\s+-i\b/.test(command) ||
|
||||
/\bperl\s+-pi\b/.test(command) ||
|
||||
// In command position only: `cat AGENTS.wmill.md` is a read.
|
||||
/(?:^|[;&|(]|\b(?:then|do|xargs)\s)\s*wmill(?:\s|$)/m.test(command)
|
||||
);
|
||||
}
|
||||
@@ -2,7 +2,9 @@ import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
anthropicUsageToBenchmarkTokenUsage,
|
||||
extractCliResultTokenUsage,
|
||||
endpointEnv,
|
||||
extractProposedWmillCommands,
|
||||
getFirstMutationToolIndex,
|
||||
parseWmillInvocationLog,
|
||||
} from "./runtime";
|
||||
|
||||
@@ -147,3 +149,32 @@ push
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getFirstMutationToolIndex", () => {
|
||||
it("counts wmill only as a command, not in a file name", () => {
|
||||
const bash = (command: string) => ({ tool: "Bash", input: { command }, timestamp: 0 });
|
||||
expect(
|
||||
getFirstMutationToolIndex([
|
||||
bash("cat AGENTS.wmill.md .agents/skills/x/SKILL.md"),
|
||||
bash("ls && wmill generate-metadata"),
|
||||
])
|
||||
).toBe(1);
|
||||
expect(getFirstMutationToolIndex([bash("cd f\nwmill sync push")])).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("endpointEnv", () => {
|
||||
it("passes the endpoint only its own key", () => {
|
||||
const env = endpointEnv(
|
||||
{ baseUrl: "https://example.test", apiKeyEnv: "DEEPSEEK_API_KEY" },
|
||||
"m",
|
||||
{ PATH: "/bin", DEEPSEEK_API_KEY: "ds", OPENAI_API_KEY: "oa", ANTHROPIC_AUTH_TOKEN: "t" }
|
||||
);
|
||||
expect(env).toEqual({
|
||||
PATH: "/bin",
|
||||
ANTHROPIC_BASE_URL: "https://example.test",
|
||||
ANTHROPIC_API_KEY: "ds",
|
||||
ANTHROPIC_DEFAULT_HAIKU_MODEL: "m",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,7 +2,14 @@ import { query, type Options } from "@anthropic-ai/claude-agent-sdk";
|
||||
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { delimiter, join } from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import { getCliEvalModel, resolveEvalModel, type CliEvalModelConfig } from "../../core/models";
|
||||
import {
|
||||
getCliEvalModel,
|
||||
resolveEvalModel,
|
||||
type CliEvalModelConfig,
|
||||
} from "../../core/models";
|
||||
import { runCodex, type AgentRunResult } from "./codex";
|
||||
import { isLikelyMutatingBashCommand } from "./mutation";
|
||||
export { formatCliRunModelLabel } from "../../core/models";
|
||||
import type {
|
||||
BenchmarkTokenUsage,
|
||||
CliToolInvocation,
|
||||
@@ -141,16 +148,94 @@ export async function runPromptAndCapture(
|
||||
maxTurns: number = 3,
|
||||
modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL
|
||||
): Promise<PromptRunResult> {
|
||||
const toolsUsed: ToolInvocation[] = [];
|
||||
const skillsInvoked: string[] = [];
|
||||
const bashCommands: string[] = [];
|
||||
let output = "";
|
||||
let assistantMessageCount = 0;
|
||||
let tokenUsage: BenchmarkTokenUsage | null = null;
|
||||
let finalContextTokens: number | null = null;
|
||||
const startedAt = Date.now();
|
||||
const stubBinDir = join(cwd, WMILL_STUB_DIR_NAME);
|
||||
const wmillLogPath = join(cwd, WMILL_LOG_FILE_NAME);
|
||||
const env = {
|
||||
...getQueryEnv(),
|
||||
PATH: process.env.PATH ? `${stubBinDir}${delimiter}${process.env.PATH}` : stubBinDir,
|
||||
WMILL_BENCHMARK_LOG_PATH: wmillLogPath,
|
||||
};
|
||||
|
||||
await installWmillStub(stubBinDir);
|
||||
|
||||
const run =
|
||||
modelConfig.runtime === "codex"
|
||||
? await runCodex(prompt, cwd, maxTurns, modelConfig.model, env)
|
||||
: await runClaudeCode(prompt, cwd, maxTurns, modelConfig, env);
|
||||
|
||||
const skillsInvoked: string[] = [];
|
||||
const bashCommands: string[] = [];
|
||||
for (const tool of run.toolsUsed) {
|
||||
if (tool.tool === "Skill" && typeof tool.input.skill === "string") {
|
||||
pushUnique(skillsInvoked, tool.input.skill);
|
||||
}
|
||||
if (tool.tool === "Bash") {
|
||||
for (const command of extractBashCommands(tool.input)) {
|
||||
pushUnique(bashCommands, command);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const wmillInvocations = await readWmillInvocationLog(wmillLogPath);
|
||||
|
||||
return {
|
||||
output: run.output,
|
||||
durationMs: Date.now() - startedAt,
|
||||
tokenUsage: run.tokenUsage,
|
||||
finalContextTokens: run.finalContextTokens,
|
||||
trace: {
|
||||
toolsUsed: run.toolsUsed,
|
||||
skillsInvoked,
|
||||
assistantMessageCount: run.assistantMessageCount,
|
||||
bashCommands,
|
||||
proposedCommands: extractProposedWmillCommands(run.output),
|
||||
executedWmillCommands: wmillInvocations.map(formatExecutedWmillCommand),
|
||||
wmillInvocations,
|
||||
firstMutationToolIndex: getFirstMutationToolIndex(run.toolsUsed),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Only the endpoint's own key may reach it: a missing key would make Claude Code
|
||||
// fall back to the user's Claude login, and the agent's Bash inherits this env,
|
||||
// so any other credential could end up in a tool result sent to the endpoint.
|
||||
export function endpointEnv(
|
||||
endpoint: NonNullable<CliEvalModelConfig["anthropicEndpoint"]>,
|
||||
model: string,
|
||||
env: Record<string, string>
|
||||
): Record<string, string> {
|
||||
const key = env[endpoint.apiKeyEnv];
|
||||
if (!key) {
|
||||
throw new Error(`${endpoint.apiKeyEnv} is not set`);
|
||||
}
|
||||
const rest = Object.fromEntries(
|
||||
Object.entries(env).filter(([name]) => !/KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL/i.test(name))
|
||||
);
|
||||
return {
|
||||
...rest,
|
||||
ANTHROPIC_BASE_URL: endpoint.baseUrl,
|
||||
ANTHROPIC_API_KEY: key,
|
||||
// Claude Code's background calls default to Haiku, which the endpoint does not serve.
|
||||
ANTHROPIC_DEFAULT_HAIKU_MODEL: model,
|
||||
};
|
||||
}
|
||||
|
||||
async function runClaudeCode(
|
||||
prompt: string,
|
||||
cwd: string,
|
||||
maxTurns: number,
|
||||
modelConfig: CliEvalModelConfig,
|
||||
env: Record<string, string>
|
||||
): Promise<AgentRunResult> {
|
||||
const result: AgentRunResult = {
|
||||
output: "",
|
||||
toolsUsed: [],
|
||||
assistantMessageCount: 0,
|
||||
tokenUsage: null,
|
||||
finalContextTokens: null,
|
||||
};
|
||||
const endpoint = modelConfig.anthropicEndpoint;
|
||||
|
||||
const options: Options = {
|
||||
cwd,
|
||||
@@ -158,80 +243,42 @@ export async function runPromptAndCapture(
|
||||
maxTurns,
|
||||
settingSources: ["project"],
|
||||
allowedTools: ["Skill", "Read", "Glob", "Grep", "Bash", "Write", "Edit"],
|
||||
env: {
|
||||
...getQueryEnv(),
|
||||
PATH: process.env.PATH ? `${stubBinDir}${delimiter}${process.env.PATH}` : stubBinDir,
|
||||
WMILL_BENCHMARK_LOG_PATH: wmillLogPath,
|
||||
},
|
||||
env: endpoint ? endpointEnv(endpoint, modelConfig.model, env) : env,
|
||||
};
|
||||
|
||||
await installWmillStub(stubBinDir);
|
||||
|
||||
for await (const message of query({ prompt, options })) {
|
||||
if (message.type === "assistant") {
|
||||
assistantMessageCount += 1;
|
||||
result.assistantMessageCount += 1;
|
||||
const turnContext = anthropicUsageToBenchmarkTokenUsage(
|
||||
message.message?.usage
|
||||
)?.prompt;
|
||||
if (turnContext && turnContext > 0) {
|
||||
finalContextTokens = turnContext;
|
||||
result.finalContextTokens = turnContext;
|
||||
}
|
||||
const content = message.message?.content;
|
||||
if (Array.isArray(content)) {
|
||||
for (const block of content) {
|
||||
if (block.type === "tool_use") {
|
||||
const input = normalizeToolInput(block.input);
|
||||
toolsUsed.push({
|
||||
result.toolsUsed.push({
|
||||
tool: block.name,
|
||||
input,
|
||||
input: normalizeToolInput(block.input),
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
if (block.name === "Skill") {
|
||||
const skillInput = input as { skill?: string };
|
||||
if (skillInput.skill) {
|
||||
pushUnique(skillsInvoked, skillInput.skill);
|
||||
}
|
||||
}
|
||||
|
||||
if (block.name === "Bash") {
|
||||
for (const command of extractBashCommands(input)) {
|
||||
pushUnique(bashCommands, command);
|
||||
}
|
||||
}
|
||||
} else if (block.type === "text") {
|
||||
output += block.text;
|
||||
result.output += block.text;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (message.type === "result") {
|
||||
const resultMessage = message as { result?: string };
|
||||
tokenUsage = extractCliResultTokenUsage(message) ?? tokenUsage;
|
||||
result.tokenUsage = extractCliResultTokenUsage(message) ?? result.tokenUsage;
|
||||
if (typeof resultMessage.result === "string") {
|
||||
output += resultMessage.result;
|
||||
result.output += resultMessage.result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const proposedCommands = extractProposedWmillCommands(output);
|
||||
const wmillInvocations = await readWmillInvocationLog(wmillLogPath);
|
||||
|
||||
return {
|
||||
output,
|
||||
durationMs: Date.now() - startedAt,
|
||||
tokenUsage,
|
||||
finalContextTokens,
|
||||
trace: {
|
||||
toolsUsed,
|
||||
skillsInvoked,
|
||||
assistantMessageCount,
|
||||
bashCommands,
|
||||
proposedCommands,
|
||||
executedWmillCommands: wmillInvocations.map(formatExecutedWmillCommand),
|
||||
wmillInvocations,
|
||||
firstMutationToolIndex: getFirstMutationToolIndex(toolsUsed),
|
||||
},
|
||||
};
|
||||
return result;
|
||||
}
|
||||
|
||||
export function wasSkillInvoked(result: PromptRunResult, skillName: string): boolean {
|
||||
@@ -242,10 +289,6 @@ export function wasToolUsed(result: PromptRunResult, toolName: string): boolean
|
||||
return result.trace.toolsUsed.some((tool) => tool.tool === toolName);
|
||||
}
|
||||
|
||||
export function formatCliRunModelLabel(modelConfig: CliEvalModelConfig): string {
|
||||
return `${modelConfig.provider}:${modelConfig.model}`;
|
||||
}
|
||||
|
||||
export function getToolInputs(
|
||||
result: PromptRunResult,
|
||||
toolName: string
|
||||
@@ -512,7 +555,7 @@ function formatExecutedWmillCommand(entry: CliWmillInvocation): string {
|
||||
return ["wmill", ...entry.argv].join(" ").trim();
|
||||
}
|
||||
|
||||
function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null {
|
||||
export function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null {
|
||||
for (const [index, tool] of toolsUsed.entries()) {
|
||||
if (tool.tool === "Write" || tool.tool === "Edit") {
|
||||
return index;
|
||||
@@ -526,16 +569,6 @@ function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function isLikelyMutatingBashCommand(command: string): boolean {
|
||||
return (
|
||||
/\b(?:mkdir|touch|rm|mv|cp|install|tee)\b/.test(command) ||
|
||||
/\b(?:cat|echo|printf)\b.*(?:>|>>|\|\s*tee\b)/.test(command) ||
|
||||
/\bsed\s+-i\b/.test(command) ||
|
||||
/\bperl\s+-pi\b/.test(command) ||
|
||||
/\bwmill\b/.test(command)
|
||||
);
|
||||
}
|
||||
|
||||
function pushUnique(values: string[], value: string): void {
|
||||
if (!values.includes(value)) {
|
||||
values.push(value);
|
||||
|
||||
@@ -192,8 +192,6 @@ describe("loadCases", () => {
|
||||
requiredSkills: ["write-script-bun"],
|
||||
requiredSkillsBeforeFirstMutation: ["write-script-bun"],
|
||||
forbiddenSkills: ["write-script-python3", "write-flow"],
|
||||
orderedAssistantMentions: ["wmill generate-metadata", "wmill sync push"],
|
||||
orderedProposedCommands: ["wmill generate-metadata", "wmill sync push"],
|
||||
forbiddenExecutedCommands: ["^wmill generate-metadata", "^wmill sync push"],
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { resolveEvalModel } from "./models";
|
||||
import { formatRunModelLabel, resolveEvalModel } from "./models";
|
||||
|
||||
describe("resolveEvalModel", () => {
|
||||
it("supports GPT-5.5 aliases for frontend evals", () => {
|
||||
@@ -47,6 +47,18 @@ describe("resolveEvalModel", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("labels cli runs by agent, keeping Claude's historical label", () => {
|
||||
expect(formatRunModelLabel("cli", resolveEvalModel("cli", "sonnet-5.5"))).toBe(
|
||||
"anthropic:claude-sonnet-5-5",
|
||||
);
|
||||
expect(formatRunModelLabel("cli", resolveEvalModel("cli", "gpt-6.1-sol"))).toBe(
|
||||
"codex:gpt-6.1-sol",
|
||||
);
|
||||
expect(formatRunModelLabel("cli", resolveEvalModel("cli", "deepseek-v4-pro"))).toBe(
|
||||
"claude-code:deepseek-v4-pro",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects Gemini aliases for cli evals", () => {
|
||||
expect(() => resolveEvalModel("cli", "gemini-3-flash-preview")).toThrow(
|
||||
"Model gemini-3-flash-preview is not supported for cli mode",
|
||||
|
||||
+45
-7
@@ -5,9 +5,13 @@ export interface FrontendEvalModelConfig {
|
||||
model: string;
|
||||
}
|
||||
|
||||
// The coding agent that drives the model in cli mode (adapters/cli/runtime.ts).
|
||||
export interface CliEvalModelConfig {
|
||||
provider: "anthropic";
|
||||
runtime: "claude-code" | "codex";
|
||||
model: string;
|
||||
// An Anthropic-compatible endpoint other than Anthropic's, for claude-code,
|
||||
// and the env var holding its key.
|
||||
anthropicEndpoint?: { baseUrl: string; apiKeyEnv: string };
|
||||
}
|
||||
|
||||
export interface EvalModelSpec {
|
||||
@@ -35,7 +39,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
|
||||
model: "claude-haiku-4-5-20251001",
|
||||
},
|
||||
cli: {
|
||||
provider: "anthropic",
|
||||
runtime: "claude-code",
|
||||
model: "haiku",
|
||||
},
|
||||
},
|
||||
@@ -55,7 +59,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
|
||||
model: "claude-sonnet-4-5-20250929",
|
||||
},
|
||||
cli: {
|
||||
provider: "anthropic",
|
||||
runtime: "claude-code",
|
||||
model: "sonnet",
|
||||
},
|
||||
},
|
||||
@@ -74,7 +78,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
|
||||
model: "claude-opus-4-6",
|
||||
},
|
||||
cli: {
|
||||
provider: "anthropic",
|
||||
runtime: "claude-code",
|
||||
model: "opus",
|
||||
},
|
||||
},
|
||||
@@ -87,7 +91,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
|
||||
model: "claude-sonnet-5-5",
|
||||
},
|
||||
cli: {
|
||||
provider: "anthropic",
|
||||
runtime: "claude-code",
|
||||
model: "claude-sonnet-5-5",
|
||||
},
|
||||
},
|
||||
@@ -100,7 +104,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
|
||||
model: "claude-opus-5-5",
|
||||
},
|
||||
cli: {
|
||||
provider: "anthropic",
|
||||
runtime: "claude-code",
|
||||
model: "claude-opus-5-5",
|
||||
},
|
||||
},
|
||||
@@ -139,6 +143,10 @@ export const EVAL_MODELS: EvalModelSpec[] = [
|
||||
provider: "openai",
|
||||
model: "gpt-6-astra",
|
||||
},
|
||||
cli: {
|
||||
runtime: "codex",
|
||||
model: "gpt-6-astra",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "gpt-6-sol",
|
||||
@@ -149,6 +157,19 @@ export const EVAL_MODELS: EvalModelSpec[] = [
|
||||
model: "gpt-6-sol",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "gpt-6.1-sol",
|
||||
label: "GPT-6.1 Sol",
|
||||
aliases: ["gpt-6.1-sol"],
|
||||
frontend: {
|
||||
provider: "openai",
|
||||
model: "gpt-6.1-sol",
|
||||
},
|
||||
cli: {
|
||||
runtime: "codex",
|
||||
model: "gpt-6.1-sol",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "gpt-6-luna",
|
||||
label: "GPT-6 Luna",
|
||||
@@ -215,6 +236,14 @@ export const EVAL_MODELS: EvalModelSpec[] = [
|
||||
provider: "deepseek",
|
||||
model: "deepseek-v4-pro",
|
||||
},
|
||||
cli: {
|
||||
runtime: "claude-code",
|
||||
model: "deepseek-v4-pro",
|
||||
anthropicEndpoint: {
|
||||
baseUrl: "https://api.deepseek.com/anthropic",
|
||||
apiKeyEnv: "DEEPSEEK_API_KEY",
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
@@ -253,11 +282,20 @@ export function formatRunModelLabel(
|
||||
model: EvalModelSpec,
|
||||
): string {
|
||||
if (mode === "cli") {
|
||||
return `${model.cli!.provider}:${model.cli!.model}`;
|
||||
return formatCliRunModelLabel(model.cli!);
|
||||
}
|
||||
return `${model.frontend!.provider}:${model.frontend!.model}`;
|
||||
}
|
||||
|
||||
// Claude on Claude Code keeps the `anthropic:` prefix its dashboard history was
|
||||
// recorded under; any other pairing names the agent.
|
||||
export function formatCliRunModelLabel(config: CliEvalModelConfig): string {
|
||||
if (config.runtime === "claude-code" && !config.anthropicEndpoint) {
|
||||
return `anthropic:${config.model}`;
|
||||
}
|
||||
return `${config.runtime}:${config.model}`;
|
||||
}
|
||||
|
||||
export function getFrontendEvalModel(
|
||||
model: EvalModelSpec,
|
||||
): FrontendEvalModelConfig {
|
||||
|
||||
+32
-9
@@ -39,17 +39,36 @@ interface CliRunActual {
|
||||
trace: CliTrace;
|
||||
}
|
||||
|
||||
const CLAUDE_PROJECT_PREAMBLE = [
|
||||
"Follow the project instructions from AGENTS.md exactly.",
|
||||
"Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.",
|
||||
"Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.",
|
||||
"Do not skip the Skill step.",
|
||||
// Only the skill wording differs per agent: Claude Code has a Skill tool, Codex
|
||||
// loads a skill by reading its SKILL.md.
|
||||
const SKILL_STEP: Record<CliEvalModelConfig["runtime"], string[]> = {
|
||||
"claude-code": [
|
||||
"Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.",
|
||||
"Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.",
|
||||
"Do not skip the Skill step.",
|
||||
],
|
||||
codex: [
|
||||
"Before creating or modifying any Windmill entity, you MUST read the relevant skill's SKILL.md under .agents/skills and follow it.",
|
||||
"Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.",
|
||||
"Do not skip the skill step.",
|
||||
],
|
||||
};
|
||||
|
||||
const HARNESS_PREAMBLE = [
|
||||
"You are running inside an automated benchmark harness, not an interactive user session.",
|
||||
"Act autonomously and complete the requested file changes directly in the workspace.",
|
||||
"Do not ask for confirmation, do not ask the user to save or create files manually, and do not wait for approval.",
|
||||
"Do not respond with a plan when you can make the change directly.",
|
||||
"Only describe what was done after you have written the files.",
|
||||
].join(" ");
|
||||
];
|
||||
|
||||
function projectPreamble(runtime: CliEvalModelConfig["runtime"]): string {
|
||||
return [
|
||||
"Follow the project instructions from AGENTS.md exactly.",
|
||||
...SKILL_STEP[runtime],
|
||||
...HARNESS_PREAMBLE,
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
export function createCliModeRunner(
|
||||
modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL
|
||||
@@ -89,7 +108,7 @@ export function createCliModeRunner(
|
||||
});
|
||||
await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8");
|
||||
|
||||
const renderedPrompt = await renderPrompt(prompt, workspaceDir);
|
||||
const renderedPrompt = await renderPrompt(prompt, workspaceDir, modelConfig.runtime);
|
||||
const run = await runPromptAndCapture(
|
||||
renderedPrompt,
|
||||
workspaceDir,
|
||||
@@ -193,7 +212,11 @@ export function getCliRunModelLabel(
|
||||
return formatCliRunModelLabel(modelConfig);
|
||||
}
|
||||
|
||||
async function renderPrompt(prompt: string, workspaceDir: string): Promise<string> {
|
||||
async function renderPrompt(
|
||||
prompt: string,
|
||||
workspaceDir: string,
|
||||
runtime: CliEvalModelConfig["runtime"]
|
||||
): Promise<string> {
|
||||
const renderedUserPrompt = prompt.replaceAll("{{workspace_root}}", workspaceDir);
|
||||
const agentsInstructions = await readFile(path.join(workspaceDir, "AGENTS.md"), "utf8");
|
||||
|
||||
@@ -202,7 +225,7 @@ async function renderPrompt(prompt: string, workspaceDir: string): Promise<strin
|
||||
agentsInstructions.trim(),
|
||||
"",
|
||||
"# Benchmark Harness",
|
||||
CLAUDE_PROJECT_PREAMBLE,
|
||||
projectPreamble(runtime),
|
||||
"",
|
||||
"# User Request",
|
||||
renderedUserPrompt,
|
||||
|
||||
Reference in New Issue
Block a user