starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-18 16:03:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: simplify ai eval history

Browse Source 
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
centdix
2026-04-08 12:19:10 +02:00
co-authored by Claude Opus 4.5
parent f734a0464b
commit d176983c21
12 changed files with 15 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ai_evals/.gitignore
+1
View File
@@ -0,0 +1 @@
.env
ai_evals/cli/index.ts
+1 -1
View File
@@ -335,7 +335,7 @@ async function handleHistory(limit: number, json: boolean) {
process.stdout.write(`Official runs: ${entries.length}\n`);
for (const entry of entries) {
process.stdout.write(
`- ${entry.timestamp} | ${formatSurfaceLabel(entry.surface)} | ${entry.variant_name} | ${formatPercent(entry.metrics.quality.pass_rate)} | flake ${formatPercent(entry.metrics.reliability.flake_rate)} | avg ${formatNumber(entry.metrics.efficiency.latency_ms_mean)} ms | ${entry.case_count} cases x ${entry.runs_per_case} runs\n`
`- ${entry.timestamp} | ${formatSurfaceLabel(entry.surface)} | ${entry.label} | ${formatPercent(entry.metrics.quality.pass_rate)} | flake ${formatPercent(entry.metrics.reliability.flake_rate)} | avg ${formatNumber(entry.metrics.efficiency.latency_ms_mean)} ms | ${entry.case_count} cases x ${entry.runs_per_case} runs\n`
);
}
}
ai_evals/cli/results.ts
+1 -1
View File
@@ -224,7 +224,7 @@ export function buildOfficialRunFromResult(
suite_version: "cli-benchmark-v1",
scoring_version: "cli-deterministic-v1",
surface: result.canonicalSurface,
variant_name: labelOverride ?? result.label,
label: labelOverride ?? result.label,
provider: result.provider,
model: result.model,
judge_model: result.judgeModel,
ai_evals/history/README.md
+2 -4
View File
@@ -27,7 +27,6 @@ untracked location.
- `benchmark-run.schema.json`: contract for one official run snapshot
- `runs/`: detailed per-run JSON snapshots
- `summary.jsonl`: one compact summary row per official run
- `rollups/`: chart-ready aggregates rebuilt from `summary.jsonl`
## Summary Metrics
@@ -61,7 +60,6 @@ This command will:
1. validate the input shape
2. write the detailed snapshot under `runs/`
3. upsert one summary row in `summary.jsonl`
4. rebuild the rollup JSON files
The writer is intentionally simple so frontend and CLI runners can adopt the
same format later.
The writer is intentionally simple. Official history is just the detailed run
files plus the compact summary index.
ai_evals/history/benchmark-run.schema.json
+1 -2
View File
@@ -9,7 +9,6 @@
"suite_version",
"scoring_version",
"surface",
"variant_name",
"provider",
"model",
"runs_per_case",
@@ -42,7 +41,7 @@
"type": "string",
"minLength": 1
},
"variant_name": {
"label": {
"type": "string",
"minLength": 1
},
ai_evals/history/rollups/by_model.json
-50
View File
@@ -1,50 +0,0 @@
{
"generatedAt": "2026-04-03T16:13:24.774Z",
"groupKey": "provider:model",
"groups": {
"anthropic:claude-haiku-4-5-20251001": [
{
"run_id": "2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4",
"timestamp": "2026-04-03T16:13:24.771Z",
"git_sha": "8c9603a01fa48a3e45b3380daac7814b9ed46acf",
"suite_version": "cli-benchmark-v1",
"scoring_version": "cli-deterministic-v1",
"surface": "frontend-flow",
"variant_name": "baseline",
"provider": "anthropic",
"model": "claude-haiku-4-5-20251001",
"judge_model": "claude-sonnet-4-6",
"runs_per_case": 1,
"case_count": 7,
"run_path": "runs/2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4.json",
"metrics": {
"quality": {
"pass_rate": 0.8571428571428571,
"deterministic_pass_rate": 0.8571428571428571,
"judge_score_mean": 85.42857142857143,
"judge_score_median": 88,
"judge_score_p10": 72,
"quality_score": 85.71428571428571
},
"reliability": {
"runs_per_case": 1,
"flake_rate": 0,
"path_consistency": 1
},
"efficiency": {
"latency_ms_mean": 143566.42857142858,
"latency_ms_median": 147598,
"tokens_total_mean": 0,
"tool_calls_mean": 11.714285714285714,
"iterations_mean": 12.714285714285714,
"estimated_cost_mean": 0,
"cost_per_success": 0,
"latency_per_success": 141550.33333333334,
"efficiency_score": 34.1017456040819,
"value_score": 29.230067660641627
}
}
}
]
}
}
ai_evals/history/rollups/by_surface.json
-50
View File
@@ -1,50 +0,0 @@
{
"generatedAt": "2026-04-03T16:13:24.774Z",
"groupKey": "surface",
"groups": {
"frontend-flow": [
{
"run_id": "2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4",
"timestamp": "2026-04-03T16:13:24.771Z",
"git_sha": "8c9603a01fa48a3e45b3380daac7814b9ed46acf",
"suite_version": "cli-benchmark-v1",
"scoring_version": "cli-deterministic-v1",
"surface": "frontend-flow",
"variant_name": "baseline",
"provider": "anthropic",
"model": "claude-haiku-4-5-20251001",
"judge_model": "claude-sonnet-4-6",
"runs_per_case": 1,
"case_count": 7,
"run_path": "runs/2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4.json",
"metrics": {
"quality": {
"pass_rate": 0.8571428571428571,
"deterministic_pass_rate": 0.8571428571428571,
"judge_score_mean": 85.42857142857143,
"judge_score_median": 88,
"judge_score_p10": 72,
"quality_score": 85.71428571428571
},
"reliability": {
"runs_per_case": 1,
"flake_rate": 0,
"path_consistency": 1
},
"efficiency": {
"latency_ms_mean": 143566.42857142858,
"latency_ms_median": 147598,
"tokens_total_mean": 0,
"tool_calls_mean": 11.714285714285714,
"iterations_mean": 12.714285714285714,
"estimated_cost_mean": 0,
"cost_per_success": 0,
"latency_per_success": 141550.33333333334,
"efficiency_score": 34.1017456040819,
"value_score": 29.230067660641627
}
}
}
]
}
}
ai_evals/history/rollups/by_variant.json
-50
View File
@@ -1,50 +0,0 @@
{
"generatedAt": "2026-04-03T16:13:24.774Z",
"groupKey": "surface:variant_name",
"groups": {
"frontend-flow:baseline": [
{
"run_id": "2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4",
"timestamp": "2026-04-03T16:13:24.771Z",
"git_sha": "8c9603a01fa48a3e45b3380daac7814b9ed46acf",
"suite_version": "cli-benchmark-v1",
"scoring_version": "cli-deterministic-v1",
"surface": "frontend-flow",
"variant_name": "baseline",
"provider": "anthropic",
"model": "claude-haiku-4-5-20251001",
"judge_model": "claude-sonnet-4-6",
"runs_per_case": 1,
"case_count": 7,
"run_path": "runs/2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4.json",
"metrics": {
"quality": {
"pass_rate": 0.8571428571428571,
"deterministic_pass_rate": 0.8571428571428571,
"judge_score_mean": 85.42857142857143,
"judge_score_median": 88,
"judge_score_p10": 72,
"quality_score": 85.71428571428571
},
"reliability": {
"runs_per_case": 1,
"flake_rate": 0,
"path_consistency": 1
},
"efficiency": {
"latency_ms_mean": 143566.42857142858,
"latency_ms_median": 147598,
"tokens_total_mean": 0,
"tool_calls_mean": 11.714285714285714,
"iterations_mean": 12.714285714285714,
"estimated_cost_mean": 0,
"cost_per_success": 0,
"latency_per_success": 141550.33333333334,
"efficiency_score": 34.1017456040819,
"value_score": 29.230067660641627
}
}
}
]
}
}
ai_evals/history/rollups/latest.json
-89
View File
@@ -1,89 +0,0 @@
{
"generatedAt": "2026-04-03T16:13:24.774Z",
"latestRun": {
"run_id": "2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4",
"timestamp": "2026-04-03T16:13:24.771Z",
"git_sha": "8c9603a01fa48a3e45b3380daac7814b9ed46acf",
"suite_version": "cli-benchmark-v1",
"scoring_version": "cli-deterministic-v1",
"surface": "frontend-flow",
"variant_name": "baseline",
"provider": "anthropic",
"model": "claude-haiku-4-5-20251001",
"judge_model": "claude-sonnet-4-6",
"runs_per_case": 1,
"case_count": 7,
"run_path": "runs/2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4.json",
"metrics": {
"quality": {
"pass_rate": 0.8571428571428571,
"deterministic_pass_rate": 0.8571428571428571,
"judge_score_mean": 85.42857142857143,
"judge_score_median": 88,
"judge_score_p10": 72,
"quality_score": 85.71428571428571
},
"reliability": {
"runs_per_case": 1,
"flake_rate": 0,
"path_consistency": 1
},
"efficiency": {
"latency_ms_mean": 143566.42857142858,
"latency_ms_median": 147598,
"tokens_total_mean": 0,
"tool_calls_mean": 11.714285714285714,
"iterations_mean": 12.714285714285714,
"estimated_cost_mean": 0,
"cost_per_success": 0,
"latency_per_success": 141550.33333333334,
"efficiency_score": 34.1017456040819,
"value_score": 29.230067660641627
}
}
},
"latestBySurface": {
"frontend-flow": {
"run_id": "2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4",
"timestamp": "2026-04-03T16:13:24.771Z",
"git_sha": "8c9603a01fa48a3e45b3380daac7814b9ed46acf",
"suite_version": "cli-benchmark-v1",
"scoring_version": "cli-deterministic-v1",
"surface": "frontend-flow",
"variant_name": "baseline",
"provider": "anthropic",
"model": "claude-haiku-4-5-20251001",
"judge_model": "claude-sonnet-4-6",
"runs_per_case": 1,
"case_count": 7,
"run_path": "runs/2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4.json",
"metrics": {
"quality": {
"pass_rate": 0.8571428571428571,
"deterministic_pass_rate": 0.8571428571428571,
"judge_score_mean": 85.42857142857143,
"judge_score_median": 88,
"judge_score_p10": 72,
"quality_score": 85.71428571428571
},
"reliability": {
"runs_per_case": 1,
"flake_rate": 0,
"path_consistency": 1
},
"efficiency": {
"latency_ms_mean": 143566.42857142858,
"latency_ms_median": 147598,
"tokens_total_mean": 0,
"tool_calls_mean": 11.714285714285714,
"iterations_mean": 12.714285714285714,
"estimated_cost_mean": 0,
"cost_per_success": 0,
"latency_per_success": 141550.33333333334,
"efficiency_score": 34.1017456040819,
"value_score": 29.230067660641627
}
}
}
}
}
ai_evals/history/runs/2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4.json
+1 -1
View File
@@ -4,7 +4,7 @@
"suite_version": "cli-benchmark-v1",
"scoring_version": "cli-deterministic-v1",
"surface": "frontend-flow",
"variant_name": "baseline",
"label": "baseline",
"provider": "anthropic",
"model": "claude-haiku-4-5-20251001",
"judge_model": "claude-sonnet-4-6",
ai_evals/history/summary.jsonl
+1 -1
View File
@@ -1 +1 @@
{"run_id":"2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4","timestamp":"2026-04-03T16:13:24.771Z","git_sha":"8c9603a01fa48a3e45b3380daac7814b9ed46acf","suite_version":"cli-benchmark-v1","scoring_version":"cli-deterministic-v1","surface":"frontend-flow","variant_name":"baseline","provider":"anthropic","model":"claude-haiku-4-5-20251001","judge_model":"claude-sonnet-4-6","runs_per_case":1,"case_count":7,"run_path":"runs/2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4.json","metrics":{"quality":{"pass_rate":0.8571428571428571,"deterministic_pass_rate":0.8571428571428571,"judge_score_mean":85.42857142857143,"judge_score_median":88,"judge_score_p10":72,"quality_score":85.71428571428571},"reliability":{"runs_per_case":1,"flake_rate":0,"path_consistency":1},"efficiency":{"latency_ms_mean":143566.42857142858,"latency_ms_median":147598,"tokens_total_mean":0,"tool_calls_mean":11.714285714285714,"iterations_mean":12.714285714285714,"estimated_cost_mean":0,"cost_per_success":0,"latency_per_success":141550.33333333334,"efficiency_score":34.1017456040819,"value_score":29.230067660641627}}}
{"run_id":"2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4","timestamp":"2026-04-03T16:13:24.771Z","git_sha":"8c9603a01fa48a3e45b3380daac7814b9ed46acf","suite_version":"cli-benchmark-v1","scoring_version":"cli-deterministic-v1","surface":"frontend-flow","label":"baseline","provider":"anthropic","model":"claude-haiku-4-5-20251001","judge_model":"claude-sonnet-4-6","runs_per_case":1,"case_count":7,"run_path":"runs/2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4.json","metrics":{"quality":{"pass_rate":0.8571428571428571,"deterministic_pass_rate":0.8571428571428571,"judge_score_mean":85.42857142857143,"judge_score_median":88,"judge_score_p10":72,"quality_score":85.71428571428571},"reliability":{"runs_per_case":1,"flake_rate":0,"path_consistency":1},"efficiency":{"latency_ms_mean":143566.42857142858,"latency_ms_median":147598,"tokens_total_mean":0,"tool_calls_mean":11.714285714285714,"iterations_mean":12.714285714285714,"estimated_cost_mean":0,"cost_per_success":0,"latency_per_success":141550.33333333334,"efficiency_score":34.1017456040819,"value_score":29.230067660641627}}}
ai_evals/history/writer.mjs
+7 -99
View File
@@ -7,7 +7,6 @@ const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url));
export const DEFAULT_HISTORY_DIR = MODULE_DIR;
const SUMMARY_FILENAME = "summary.jsonl";
const RUNS_DIRNAME = "runs";
const ROLLUPS_DIRNAME = "rollups";
export async function appendOfficialRun(input, options = {}) {
const absoluteHistoryDir = path.resolve(options.historyDir ?? DEFAULT_HISTORY_DIR);
@@ -26,7 +25,6 @@ export async function appendOfficialRun(input, options = {}) {
const nextSummaries = upsertSummaryEntry(summaries, summaryEntry);
await writeSummaryEntries(summaryPath, nextSummaries);
await writeRollups(absoluteHistoryDir, nextSummaries);
return {
status: "ok",
@@ -36,33 +34,11 @@ export async function appendOfficialRun(input, options = {}) {
};
}
export async function loadLatestHistoryRollup(historyDir = DEFAULT_HISTORY_DIR) {
const rollupPath = path.join(path.resolve(historyDir), ROLLUPS_DIRNAME, "latest.json");
return await loadJsonFile(rollupPath);
}
export async function loadHistoryRollup(
name,
historyDir = DEFAULT_HISTORY_DIR
) {
const rollupPath = path.join(path.resolve(historyDir), ROLLUPS_DIRNAME, name);
return await loadJsonFile(rollupPath);
}
export async function loadSummaryHistory(historyDir = DEFAULT_HISTORY_DIR) {
const summaryPath = path.join(path.resolve(historyDir), SUMMARY_FILENAME);
return await loadSummaryEntries(summaryPath);
}
async function loadJsonFile(filePath) {
const raw = await readFile(filePath, "utf8");
try {
return JSON.parse(raw);
} catch (error) {
throw new Error(`Failed to parse JSON from ${filePath}: ${error.message}`);
}
}
function normalizeRun(input) {
assertPlainObject(input, "benchmark run");
@@ -74,7 +50,7 @@ function normalizeRun(input) {
"scoring_version"
);
const surface = assertNonEmptyString(input.surface, "surface");
const variantName = assertNonEmptyString(input.variant_name, "variant_name");
const label = assertNonEmptyString(input.label, "label");
const provider = assertNonEmptyString(input.provider, "provider");
const model = assertNonEmptyString(input.model, "model");
const judgeModel =
@@ -98,7 +74,7 @@ function normalizeRun(input) {
: buildRunId({
timestamp,
surface,
variantName,
label,
provider,
model,
gitSha
@@ -112,7 +88,7 @@ function normalizeRun(input) {
suite_version: suiteVersion,
scoring_version: scoringVersion,
surface,
variant_name: variantName,
label,
provider,
model,
judge_model: judgeModel,
@@ -230,14 +206,14 @@ function normalizeCases(input) {
});
}
function buildRunId({ timestamp, surface, variantName, provider, model, gitSha }) {
function buildRunId({ timestamp, surface, label, provider, model, gitSha }) {
const timestampSlug = timestamp.replaceAll(":", "-").replaceAll(".", "-");
const shortSha = gitSha.slice(0, 12);
return [
slugify(timestampSlug),
slugify(surface),
slugify(variantName),
slugify(label),
slugify(provider),
slugify(model),
shortSha
@@ -254,7 +230,7 @@ function buildSummaryEntry(run, runRelativePath) {
suite_version: run.suite_version,
scoring_version: run.scoring_version,
surface: run.surface,
variant_name: run.variant_name,
label: run.label,
provider: run.provider,
model: run.model,
judge_model: run.judge_model,
@@ -303,76 +279,8 @@ async function writeSummaryEntries(summaryPath, entries) {
await writeFile(summaryPath, content, "utf8");
}
async function writeRollups(historyDir, summaries) {
const rollupsDir = path.join(historyDir, ROLLUPS_DIRNAME);
const generatedAt = new Date().toISOString();
const latestFirst = [...summaries].sort((left, right) =>
right.timestamp.localeCompare(left.timestamp)
);
const latestBySurface = {};
for (const summary of latestFirst) {
if (!(summary.surface in latestBySurface)) {
latestBySurface[summary.surface] = summary;
}
}
const latestRollup = {
generatedAt,
latestRun: latestFirst[0] ?? null,
latestBySurface
};
const bySurfaceRollup = {
generatedAt,
groupKey: "surface",
groups: groupSummaries(summaries, (summary) => summary.surface)
};
const byVariantRollup = {
generatedAt,
groupKey: "surface:variant_name",
groups: groupSummaries(
summaries,
(summary) => `${summary.surface}:${summary.variant_name}`
)
};
const byModelRollup = {
generatedAt,
groupKey: "provider:model",
groups: groupSummaries(
summaries,
(summary) => `${summary.provider}:${summary.model}`
)
};
await Promise.all([
writeJsonFile(path.join(rollupsDir, "latest.json"), latestRollup),
writeJsonFile(path.join(rollupsDir, "by_surface.json"), bySurfaceRollup),
writeJsonFile(path.join(rollupsDir, "by_variant.json"), byVariantRollup),
writeJsonFile(path.join(rollupsDir, "by_model.json"), byModelRollup)
]);
}
function groupSummaries(summaries, getGroupKey) {
const groups = {};
for (const summary of summaries) {
const key = getGroupKey(summary);
groups[key] ??= [];
groups[key].push(summary);
}
for (const groupEntries of Object.values(groups)) {
groupEntries.sort((left, right) => left.timestamp.localeCompare(right.timestamp));
}
return groups;
}
async function ensureHistoryLayout(historyDir) {
await Promise.all([
mkdir(path.join(historyDir, RUNS_DIRNAME), { recursive: true }),
mkdir(path.join(historyDir, ROLLUPS_DIRNAME), { recursive: true })
]);
await mkdir(path.join(historyDir, RUNS_DIRNAME), { recursive: true });
}
async function writeJsonFile(filePath, value) {