starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-25 08:03:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add frontend benchmark surfaces

Browse Source 
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
centdix
2026-04-01 13:16:22 +02:00
co-authored by Claude Opus 4.5
parent 9e569c4000
commit 2392448728
12 changed files with 1068 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ai_evals/README.md
+8
View File
@@ -56,9 +56,17 @@ Current usage:
```bash
cd ai_evals
bun run cli -- list-variants --surface cli
bun run cli -- list-variants --surface frontend-flow
bun run cli -- list-variants --surface frontend-app
bun run cli -- snapshot-variant --surface cli --variant candidate
bun run cli -- list-cases --surface cli
bun run cli -- list-cases --surface frontend-flow
bun run cli -- list-cases --surface frontend-app
bun run cli -- run --surface cli --case bun-hello-script --variant baseline --runs 5
bun run cli -- run --surface frontend-flow --case flow-test1-user-role-actions --variant baseline --runs 1
bun run cli -- run --surface frontend-app --case app-test1-counter-create --variant baseline --runs 1
bun run cli -- compare --surface frontend-flow --case flow-test1-user-role-actions --variant baseline --variant baseline --runs 1
bun run cli -- compare --surface frontend-app --case app-test1-counter-create --variant baseline --variant baseline --runs 1
bun run cli -- compare --surface cli --case bun-hello-script --variant baseline --variant baseline --runs 5
bun run cli -- compare --surface cli --case bun-hello-script --variant baseline-frozen --variant candidate --runs 5 --write-history
bun run cli -- history --view latest
ai_evals/adapters/frontend/manifests.ts
+51
View File
@@ -0,0 +1,51 @@
import { readdir, readFile } from 'node:fs/promises'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
export type FrontendSurfaceName = 'frontend-flow' | 'frontend-app'
interface FrontendCaseManifest {
id: string
title: string
}
interface FrontendVariantManifest {
id: string
description?: string
}
const REPO_ROOT = fileURLToPath(new URL('../../../', import.meta.url))
export async function loadFrontendCases(surface: FrontendSurfaceName): Promise<FrontendCaseManifest[]> {
const raw = await readFile(
path.join(REPO_ROOT, 'ai_evals', 'cases', 'frontend', `${surfaceToManifestName(surface)}.json`),
'utf8'
)
return JSON.parse(raw) as FrontendCaseManifest[]
}
export async function loadFrontendVariants(
surface: FrontendSurfaceName
): Promise<FrontendVariantManifest[]> {
const variantsDir = path.join(
REPO_ROOT,
'ai_evals',
'variants',
'frontend',
surfaceToManifestName(surface)
)
const filenames = (await readdir(variantsDir))
.filter((entry) => entry.endsWith('.json'))
.sort((left, right) => left.localeCompare(right))
return await Promise.all(
filenames.map(async (filename) => {
const raw = await readFile(path.join(variantsDir, filename), 'utf8')
return JSON.parse(raw) as FrontendVariantManifest
})
)
}
function surfaceToManifestName(surface: FrontendSurfaceName): 'flow' | 'app' {
return surface === 'frontend-flow' ? 'flow' : 'app'
}
ai_evals/adapters/frontend/runtime.ts
+97
View File
@@ -0,0 +1,97 @@
import { execFile as execFileCallback } from 'node:child_process'
import { mkdtemp, readFile, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import path from 'node:path'
import { promisify } from 'node:util'
import { fileURLToPath } from 'node:url'
const execFile = promisify(execFileCallback)
const REPO_ROOT = fileURLToPath(new URL('../../../', import.meta.url))
const FRONTEND_DIR = path.join(REPO_ROOT, 'frontend')
const FRONTEND_BENCHMARK_TEST = 'src/lib/components/copilot/chat/__tests__/benchmarkCli/aiEvalsAdapter.test.ts'
export type FrontendSurfaceName = 'frontend-flow' | 'frontend-app'
export interface FrontendAdapterAttempt {
attempt: number
passed: boolean
durationMs: number
assistantMessageCount: number
toolCallCount: number
toolsUsed: string[]
checks: Array<{ name: string; passed: boolean; required?: boolean }>
requiredFailedChecks: string[]
judgeScore: number | null
judgeStatement: string | null
error: string | null
}
export interface FrontendAdapterCaseResult {
caseId: string
attempts: FrontendAdapterAttempt[]
}
export interface FrontendAdapterVariantResult {
variant: string
provider: string
model: string
judgeModel: string | null
caseResults: FrontendAdapterCaseResult[]
}
export interface FrontendAdapterPayload {
surface: 'flow' | 'app'
runs: number
variants: FrontendAdapterVariantResult[]
}
export async function runFrontendBenchmarkAdapter(input: {
surface: FrontendSurfaceName
caseIds: string[]
variantIds: string[]
runs: number
}): Promise<FrontendAdapterPayload> {
const tempDir = await mkdtemp(path.join(tmpdir(), 'wmill-frontend-benchmark-'))
const outputPath = path.join(tempDir, 'result.json')
try {
await execFile(
path.join(FRONTEND_DIR, 'node_modules', '.bin', 'vitest'),
[
'run',
FRONTEND_BENCHMARK_TEST,
'--project',
'server',
'--config',
'vite.config.js'
],
{
cwd: FRONTEND_DIR,
env: {
...process.env,
WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH: outputPath,
WMILL_FRONTEND_AI_EVAL_SURFACE: frontendSurfaceToAdapterSurface(input.surface),
WMILL_FRONTEND_AI_EVAL_CASE_IDS: JSON.stringify(input.caseIds),
WMILL_FRONTEND_AI_EVAL_VARIANT_IDS: JSON.stringify(input.variantIds),
WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs)
},
maxBuffer: 10 * 1024 * 1024
}
)
const raw = await readFile(outputPath, 'utf8')
return JSON.parse(raw) as FrontendAdapterPayload
} catch (error) {
const executionError = error as Error & { stdout?: string; stderr?: string }
const details = [executionError.message, executionError.stdout, executionError.stderr]
.filter(Boolean)
.join('\n')
throw new Error(`Frontend benchmark adapter failed:\n${details}`)
} finally {
await rm(tempDir, { recursive: true, force: true })
}
}
function frontendSurfaceToAdapterSurface(surface: FrontendSurfaceName): 'flow' | 'app' {
return surface === 'frontend-flow' ? 'flow' : 'app'
}
ai_evals/cli/README.md
+55 -2
View File
@@ -11,7 +11,7 @@ The current implementation is intentionally small:
- `compare` command
- `history` command
- `list-cases` and `list-variants` discovery commands
- `cli` surface only
- `cli`, `frontend-flow`, and `frontend-app` surfaces
This is the benchmark entrypoint for prompt and artifact evaluation.
@@ -35,6 +35,14 @@ cd ai_evals
bun run cli -- list-cases --surface cli
```
List available frontend cases:
```bash
cd ai_evals
bun run cli -- list-cases --surface frontend-flow
bun run cli -- list-cases --surface frontend-app
```
List available CLI variants:
```bash
@@ -42,6 +50,14 @@ cd ai_evals
bun run cli -- list-variants --surface cli
```
List available frontend variants:
```bash
cd ai_evals
bun run cli -- list-variants --surface frontend-flow
bun run cli -- list-variants --surface frontend-app
```
Snapshot the current guidance bundle into a named CLI variant:
```bash
@@ -56,6 +72,20 @@ cd ai_evals
bun run cli -- run --surface cli --case bun-hello-script --variant baseline
```
Run one frontend flow case:
```bash
cd ai_evals
bun run cli -- run --surface frontend-flow --case flow-test1-user-role-actions --variant baseline --runs 1
```
Run one frontend app case:
```bash
cd ai_evals
bun run cli -- run --surface frontend-app --case app-test1-counter-create --variant baseline --runs 1
```
Keep the temp workspace for inspection:
```bash
@@ -91,6 +121,20 @@ cd ai_evals
bun run cli -- compare --surface cli --case bun-hello-script --variant baseline-frozen --variant candidate --runs 5 --write-history
```
Compare frontend prompt variants:
```bash
cd ai_evals
bun run cli -- compare --surface frontend-flow --case flow-test1-user-role-actions --variant baseline --variant candidate --runs 3
```
Write official history snapshots for frontend variants:
```bash
cd ai_evals
bun run cli -- compare --surface frontend-app --case app-test1-counter-create --variant baseline --variant candidate --runs 3 --write-history
```
Inspect the tracked history:
```bash
@@ -157,6 +201,15 @@ official snapshot per compared variant into `ai_evals/history/` and rebuilds:
The compare output also includes tool usage and invoked skills as diagnostics.
For frontend surfaces, the benchmark CLI shells into a frontend-native Vitest
adapter so the evaluation runs with the frontend project's own module
resolution and test environment, while still producing the same benchmark
result shape as the CLI surface.
By default, the repo only ships `baseline` frontend variants. To compare
distinct prompt variants or write official history snapshots, add a second
manifest under `ai_evals/variants/frontend/<surface>/`.
True efficiency metrics such as latency, token usage, and cost are planned, but
the current CLI does not emit them yet. Until that lands, use `compare`
primarily to answer "did this skill bundle produce better artifacts on the same
@@ -193,7 +246,7 @@ instead of the generated default.
Later iterations should add:
- frontend adapters
- variant cleanup and diff helpers
- token and cost metrics in compare output
- richer history views and filtering
- frontend `script` surface
ai_evals/cli/index.ts
+314 -13
View File
@@ -24,6 +24,15 @@ import {
loadLatestHistoryRollup,
loadSummaryHistory
} from "../history/writer.mjs";
import {
loadFrontendCases,
loadFrontendVariants,
type FrontendSurfaceName,
} from "../adapters/frontend/manifests";
import {
runFrontendBenchmarkAdapter,
type FrontendAdapterPayload,
} from "../adapters/frontend/runtime";
type CommandName =
| "run"
@@ -32,7 +41,7 @@ type CommandName =
| "snapshot-variant"
| "compare"
| "history";
type SurfaceName = "cli";
type SurfaceName = "cli" | FrontendSurfaceName;
interface ParsedArgs {
command: CommandName;
@@ -62,6 +71,8 @@ interface AttemptSummary {
requiredFailedChecks: string[];
expectedFiles: Array<{ path: string; exists: boolean }>;
pathSignature: string;
judgeScore: number | null;
error: string | null;
}
interface AggregateMetrics {
@@ -77,6 +88,9 @@ interface AggregateMetrics {
distinctSkillsInvoked: string[];
distinctToolsUsed: string[];
requiredFailureCounts: Array<{ name: string; count: number }>;
judgeScoreMean: number | null;
judgeScoreMedian: number | null;
judgeScoreP10: number | null;
}
interface CompareCaseResult {
@@ -99,6 +113,9 @@ interface CompareCaseResult {
interface CompareVariantResult {
label: string;
variant: string;
provider: string;
model: string;
judgeModel: string | null;
totalCases: number;
fullyPassedCases: number;
totalRuns: number;
@@ -113,6 +130,9 @@ interface CompareVariantResult {
distinctSkillsInvoked: string[];
distinctToolsUsed: string[];
requiredFailureCounts: Array<{ name: string; count: number }>;
judgeScoreMean: number | null;
judgeScoreMedian: number | null;
judgeScoreP10: number | null;
caseResults: CompareCaseResult[];
}
@@ -170,6 +190,30 @@ async function handleListCases(args: ParsedArgs) {
}
return;
}
case "frontend-flow":
case "frontend-app": {
const cases = await loadFrontendCases(surface);
const payload = {
surface,
cases: cases.map((entry) => ({
id: entry.id,
description: entry.title ?? null
}))
};
if (args.json) {
process.stdout.write(JSON.stringify(payload, null, 2) + "\n");
return;
}
process.stdout.write(`Surface: ${surface}\n`);
for (const entry of payload.cases) {
process.stdout.write(
`- ${entry.id}${entry.description ? `: ${entry.description}` : ""}\n`
);
}
return;
}
default:
assertNever(surface);
}
@@ -202,6 +246,30 @@ async function handleListVariants(args: ParsedArgs) {
}
return;
}
case "frontend-flow":
case "frontend-app": {
const variants = await loadFrontendVariants(surface);
const payload = {
surface,
variants: variants.map((entry) => ({
id: entry.id,
description: entry.description ?? null
}))
};
if (args.json) {
process.stdout.write(JSON.stringify(payload, null, 2) + "\n");
return;
}
process.stdout.write(`Surface: ${surface}\n`);
for (const entry of payload.variants) {
process.stdout.write(
`- ${entry.id}${entry.description ? `: ${entry.description}` : ""}\n`
);
}
return;
}
default:
assertNever(surface);
}
@@ -261,6 +329,30 @@ async function handleRun(args: ParsedArgs) {
}
return;
}
case "frontend-flow":
case "frontend-app": {
if (args.keepWorkspace) {
throw new Error("--keep-workspace is not supported for frontend benchmark surfaces");
}
const variantId = requireSingleVariantId(args.variantIds, "baseline");
const payload = await buildFrontendRunPayload({
surface,
caseId,
variantId,
runs: args.runs
});
if (args.json) {
process.stdout.write(JSON.stringify(payload, null, 2) + "\n");
} else {
printRunSummary(payload);
}
if (!payload.passed) {
process.exitCode = 1;
}
return;
}
default:
assertNever(surface);
}
@@ -295,6 +387,9 @@ async function handleSnapshotVariant(args: ParsedArgs) {
return;
}
case "frontend-flow":
case "frontend-app":
throw new Error("snapshot-variant is currently supported for the cli surface only");
default:
assertNever(surface);
}
@@ -367,6 +462,9 @@ async function handleCompare(args: ParsedArgs) {
internalVariantResults.push({
label: labeledVariant.label,
variant: labeledVariant.variant.id,
provider: CLI_BENCHMARK_PROVIDER,
model: CLI_BENCHMARK_MODEL,
judgeModel: null,
totalCases: caseResults.length,
fullyPassedCases: caseResults.filter(
(entry) => entry.passedRuns === entry.totalRuns
@@ -383,6 +481,9 @@ async function handleCompare(args: ParsedArgs) {
distinctSkillsInvoked: aggregate.distinctSkillsInvoked,
distinctToolsUsed: aggregate.distinctToolsUsed,
requiredFailureCounts: aggregate.requiredFailureCounts,
judgeScoreMean: aggregate.judgeScoreMean,
judgeScoreMedian: aggregate.judgeScoreMedian,
judgeScoreP10: aggregate.judgeScoreP10,
caseResults,
allAttempts
});
@@ -414,6 +515,53 @@ async function handleCompare(args: ParsedArgs) {
return;
}
case "frontend-flow":
case "frontend-app": {
if (args.variantIds.length < 2) {
throw new Error("compare requires at least two --variant values");
}
if (args.writeHistory && new Set(args.variantIds).size !== args.variantIds.length) {
throw new Error(
"--write-history requires distinct variant ids so official snapshots stay unambiguous"
);
}
const internalVariantResults = await buildFrontendCompareResults({
surface,
caseIds: args.caseIds,
variantIds: args.variantIds,
runs: args.runs
});
const historyWrites = args.writeHistory
? await writeCompareHistorySnapshots({
surface,
runs: args.runs,
variants: internalVariantResults,
historyDir: args.historyDir
})
: [];
const payload = {
command: "compare",
surface,
caseIds:
args.caseIds.length === 0
? internalVariantResults[0]?.caseResults.map((entry) => entry.caseId) ?? []
: args.caseIds,
runs: args.runs,
variants: internalVariantResults.map(({ allAttempts: _allAttempts, ...variant }) => variant),
historyWrites
};
if (args.json) {
process.stdout.write(JSON.stringify(payload, null, 2) + "\n");
} else {
printCompareSummary(payload);
}
return;
}
default:
assertNever(surface);
}
@@ -475,6 +623,137 @@ async function handleHistory(args: ParsedArgs) {
}
}
async function buildFrontendRunPayload(input: {
surface: FrontendSurfaceName;
caseId: string;
variantId: string;
runs: number;
}) {
const compareResults = await buildFrontendCompareResults({
surface: input.surface,
caseIds: [input.caseId],
variantIds: [input.variantId],
runs: input.runs
});
const variant = compareResults[0];
return {
command: "run",
surface: input.surface,
variant: variant.variant,
caseId: input.caseId,
runs: input.runs,
passed: variant.passedRuns === variant.totalRuns,
passedRuns: variant.passedRuns,
totalRuns: variant.totalRuns,
passRate: variant.passRate,
workspaceKept: false,
workspaceDir: null,
metrics: {
averageDurationMs: variant.averageDurationMs,
averageAssistantMessages: variant.averageAssistantMessages,
averageToolCalls: variant.averageToolCalls,
averageSkillInvocations: variant.averageSkillInvocations
},
distinctSkillsInvoked: variant.distinctSkillsInvoked,
distinctToolsUsed: variant.distinctToolsUsed,
requiredFailureCounts: variant.requiredFailureCounts,
attempts: variant.allAttempts
};
}
async function buildFrontendCompareResults(input: {
surface: FrontendSurfaceName;
caseIds: string[];
variantIds: string[];
runs: number;
}): Promise<Array<CompareVariantResult & { allAttempts: AttemptSummary[] }>> {
const adapterResult = await runFrontendBenchmarkAdapter({
surface: input.surface,
caseIds: input.caseIds,
variantIds: input.variantIds,
runs: input.runs
});
return adapterResult.variants.map((variant) => {
const caseResults = variant.caseResults.map((caseResult) => {
const attempts = caseResult.attempts.map(summarizeFrontendAttempt);
const aggregate = aggregateAttempts(attempts);
return {
caseId: caseResult.caseId,
totalRuns: aggregate.totalRuns,
passedRuns: aggregate.passedRuns,
passRate: aggregate.passRate,
averageDurationMs: aggregate.averageDurationMs,
medianDurationMs: aggregate.medianDurationMs,
latencyPerSuccessMs: aggregate.latencyPerSuccessMs,
averageAssistantMessages: aggregate.averageAssistantMessages,
averageToolCalls: aggregate.averageToolCalls,
averageSkillInvocations: aggregate.averageSkillInvocations,
pathConsistency: computePathConsistency(attempts),
distinctSkillsInvoked: aggregate.distinctSkillsInvoked,
distinctToolsUsed: aggregate.distinctToolsUsed,
requiredFailureCounts: aggregate.requiredFailureCounts
};
});
const allAttempts = variant.caseResults.flatMap((caseResult) =>
caseResult.attempts.map(summarizeFrontendAttempt)
);
const aggregate = aggregateAttempts(allAttempts);
return {
label: variant.variant,
variant: variant.variant,
provider: variant.provider,
model: variant.model,
judgeModel: variant.judgeModel,
totalCases: caseResults.length,
fullyPassedCases: caseResults.filter((entry) => entry.passedRuns === entry.totalRuns)
.length,
totalRuns: aggregate.totalRuns,
passedRuns: aggregate.passedRuns,
passRate: aggregate.passRate,
averageDurationMs: aggregate.averageDurationMs,
medianDurationMs: aggregate.medianDurationMs,
latencyPerSuccessMs: aggregate.latencyPerSuccessMs,
averageAssistantMessages: aggregate.averageAssistantMessages,
averageToolCalls: aggregate.averageToolCalls,
averageSkillInvocations: aggregate.averageSkillInvocations,
distinctSkillsInvoked: aggregate.distinctSkillsInvoked,
distinctToolsUsed: aggregate.distinctToolsUsed,
requiredFailureCounts: aggregate.requiredFailureCounts,
judgeScoreMean: aggregate.judgeScoreMean,
judgeScoreMedian: aggregate.judgeScoreMedian,
judgeScoreP10: aggregate.judgeScoreP10,
caseResults,
allAttempts
};
});
}
function summarizeFrontendAttempt(
attempt: FrontendAdapterPayload["variants"][number]["caseResults"][number]["attempts"][number]
): AttemptSummary {
const toolsUsed = uniqueStrings(attempt.toolsUsed);
return {
attempt: attempt.attempt,
passed: attempt.passed,
durationMs: attempt.durationMs,
assistantMessageCount: attempt.assistantMessageCount,
toolCallCount: attempt.toolCallCount,
skillInvocationCount: 0,
skillsInvoked: [],
toolsUsed,
checks: attempt.checks,
requiredFailedChecks: attempt.requiredFailedChecks,
expectedFiles: [],
pathSignature: buildPathSignature([], toolsUsed),
judgeScore: attempt.judgeScore,
error: attempt.error
};
}
async function runCaseAttempts(
evalCase: Awaited<ReturnType<typeof loadCliArtifactEvalCases>>[number],
variant: CliVariant,
@@ -534,13 +813,18 @@ function summarizeAttempt(
path: file.path,
exists: file.exists
})),
pathSignature: buildPathSignature(skillsInvoked, toolsUsed)
pathSignature: buildPathSignature(skillsInvoked, toolsUsed),
judgeScore: null,
error: result.error ?? null
};
}
function aggregateAttempts(attempts: AttemptSummary[]): AggregateMetrics {
const requiredFailureCounts = new Map<string, number>();
const successfulAttempts = attempts.filter((attempt) => attempt.passed);
const judgeScores = attempts
.map((attempt) => attempt.judgeScore)
.filter((score): score is number => typeof score === "number" && Number.isFinite(score));
for (const attempt of attempts) {
for (const failure of attempt.requiredFailedChecks) {
@@ -562,7 +846,10 @@ function aggregateAttempts(attempts: AttemptSummary[]): AggregateMetrics {
distinctToolsUsed: uniqueStrings(attempts.flatMap((attempt) => attempt.toolsUsed)),
requiredFailureCounts: [...requiredFailureCounts.entries()]
.sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]))
.map(([name, count]) => ({ name, count }))
.map(([name, count]) => ({ name, count })),
judgeScoreMean: judgeScores.length > 0 ? average(judgeScores) : null,
judgeScoreMedian: judgeScores.length > 0 ? median(judgeScores) : null,
judgeScoreP10: judgeScores.length > 0 ? percentile(judgeScores, 0.1) : null
};
}
@@ -684,7 +971,7 @@ function requireSurface(surface: string | undefined): SurfaceName {
if (!surface) {
throw new Error("Missing required --surface argument");
}
if (surface !== "cli") {
if (surface !== "cli" && surface !== "frontend-flow" && surface !== "frontend-app") {
throw new Error(`Unsupported surface for now: ${surface}`);
}
return surface;
@@ -947,18 +1234,18 @@ function buildOfficialRun(input: {
scoring_version: "cli-deterministic-v1",
surface: input.surface,
variant_name: input.variant.variant,
provider: CLI_BENCHMARK_PROVIDER,
model: CLI_BENCHMARK_MODEL,
judge_model: null,
provider: input.variant.provider,
model: input.variant.model,
judge_model: input.variant.judgeModel,
runs_per_case: input.runs,
case_count: input.variant.caseResults.length,
metrics: {
quality: {
pass_rate: input.variant.passRate,
deterministic_pass_rate: input.variant.passRate,
judge_score_mean: 0,
judge_score_median: 0,
judge_score_p10: 0,
judge_score_mean: input.variant.judgeScoreMean ?? 0,
judge_score_median: input.variant.judgeScoreMedian ?? 0,
judge_score_p10: input.variant.judgeScoreP10 ?? 0,
quality_score: qualityScore
},
reliability: {
@@ -1127,14 +1414,18 @@ function printHelp() {
[
"Usage:",
" cd ai_evals && bun run cli -- list-cases --surface cli [--json]",
" cd ai_evals && bun run cli -- list-cases --surface frontend-flow [--json]",
" cd ai_evals && bun run cli -- list-cases --surface frontend-app [--json]",
" cd ai_evals && bun run cli -- list-variants --surface cli [--json]",
" cd ai_evals && bun run cli -- list-variants --surface frontend-flow [--json]",
" cd ai_evals && bun run cli -- list-variants --surface frontend-app [--json]",
" cd ai_evals && bun run cli -- snapshot-variant --surface cli --variant <id> [--description <text>] [--json]",
" cd ai_evals && bun run cli -- run --surface cli --case <id> [--variant <id>] [--runs <n>] [--json] [--keep-workspace]",
" cd ai_evals && bun run cli -- compare --surface cli [--case <id> ...] [--variant <id> ...] [--runs <n>] [--write-history] [--history-dir <path>] [--json]",
" cd ai_evals && bun run cli -- run --surface <cli|frontend-flow|frontend-app> --case <id> [--variant <id>] [--runs <n>] [--json] [--keep-workspace]",
" cd ai_evals && bun run cli -- compare --surface <cli|frontend-flow|frontend-app> [--case <id> ...] [--variant <id> ...] [--runs <n>] [--write-history] [--history-dir <path>] [--json]",
" cd ai_evals && bun run cli -- history [--view latest|summary|surface|variant|model] [--limit <n>] [--history-dir <path>] [--json]",
"",
"Current support:",
" surfaces: cli"
" surfaces: cli, frontend-flow, frontend-app"
].join("\n") + "\n"
);
}
@@ -1165,6 +1456,16 @@ function median(values: number[]): number {
return sorted[middle];
}
function percentile(values: number[], ratio: number): number {
if (values.length === 0) {
return 0;
}
const sorted = [...values].sort((left, right) => left - right);
const index = Math.max(0, Math.min(sorted.length - 1, Math.floor((sorted.length - 1) * ratio)));
return sorted[index];
}
function uniqueStrings(values: string[]): string[] {
return [...new Set(values)].sort((left, right) => left.localeCompare(right));
}
ai_evals/variants/frontend/app/baseline.json
+12
View File
@@ -0,0 +1,12 @@
{
"id": "baseline",
"description": "Production frontend app chat prompt and tools.",
"provider": "anthropic",
"model": "claude-haiku-4-5-20251001",
"systemPrompt": {
"type": "default"
},
"tools": {
"type": "default"
}
}
ai_evals/variants/frontend/flow/baseline.json
+12
View File
@@ -0,0 +1,12 @@
{
"id": "baseline",
"description": "Production frontend flow chat prompt and tools.",
"provider": "anthropic",
"model": "claude-haiku-4-5-20251001",
"systemPrompt": {
"type": "default"
},
"tools": {
"type": "default"
}
}
docs/system-prompt-testing-status.md
+29 -14
View File
@@ -18,6 +18,7 @@ What is true today:
- there is one repo-level benchmark CLI under `ai_evals/cli`
- the CLI surface is the first real benchmark adapter behind it
- frontend flow and app are now exposed through that same benchmark CLI
- CLI cases are now skill-sensitive rather than only artifact-sensitive
- the benchmark harness and `wmill init` now share the same AI-guidance writer
- variants can be frozen as named snapshots through the benchmark CLI
@@ -28,7 +29,8 @@ What is true today:
What is not true yet:
- frontend is not yet exposed through the benchmark CLI
- frontend `script` is not yet exposed through the benchmark CLI
- frontend still has only baseline prompt variants wired by default
- CLI still needs broader case coverage and richer efficiency metrics
- token and cost metrics are not implemented
- no UI studio exists yet
@@ -87,10 +89,11 @@ Implemented:
- `snapshot-variant`
- `history`
- CLI adapter selection through `--surface cli`
- frontend adapter selection through `--surface frontend-flow|frontend-app`
Still missing:
- frontend adapter selection
- frontend `script` adapter selection
### Phase 3: Replace the CLI smoke suite with real artifact evaluation
@@ -164,6 +167,8 @@ Implemented:
- benchmark CLI `history --view latest|summary|surface|variant|model`
- official run snapshots generated from repeated CLI benchmark results
- history rollups rebuilt from real benchmark writes
- shared history writing path generalized so frontend surfaces can use the same official run format
- frontend history writes validated through `compare --write-history` on `frontend-app`
Still missing:
@@ -184,21 +189,26 @@ Planned:
Status:
- mostly not done
- partially done
Implemented:
- shared frontend case scaffolding exists
- current frontend flow/app evals were moved toward shared manifests earlier
- repo-level benchmark CLI support for `frontend-flow` and `frontend-app`
- frontend-native adapter runner that executes headless through the frontend Vitest environment
- repeated runs for frontend flow/app through the shared benchmark CLI
- frontend prompt variants loaded from files under `ai_evals/variants/frontend/`
- frontend run/compare payloads aligned to the shared benchmark result shape
- frontend judge score aggregation wired into compare output and official run generation
- frontend official history writes validated through the shared benchmark CLI
Still missing:
- frontend adapter behind the benchmark CLI
- scored frontend reliability tests
- frontend `script` support
- repeated runs
- shared result/history alignment
- prompt-variant workflow for frontend
- more than the baseline prompt variant for frontend flow/app
- broader frontend reliability coverage across more cases in routine use
- regular use of frontend history writes in routine benchmark workflows
### Phase 6: Add CI tiers
@@ -253,6 +263,9 @@ The most important implemented changes so far are:
- `bun-hello-flow`
- Made CLI evals skill-sensitive instead of allowing silent skill bypass
- Added repeated-run CLI benchmarking with aggregate reporting
- Added frontend flow/app benchmark adapters behind the shared benchmark CLI
- Added frontend run/compare support with shared aggregation and judge score reporting
- Validated frontend `run`, `compare`, and `compare --write-history` through the shared benchmark CLI
- Shared AI-guidance generation between:
- benchmark temp workspaces
- `wmill init`
@@ -264,8 +277,8 @@ The most important implemented changes so far are:
The highest-priority remaining work is:
1. Bring frontend behind the same benchmark CLI.
2. Expand the CLI case corpus to cover more real skill behavior.
1. Add the missing frontend `script` surface behind the same benchmark CLI.
2. Expand the frontend and CLI case corpora to cover more real prompt and skill behavior.
3. Add richer aggregate metrics:
- pass rate
- flake rate
@@ -283,19 +296,21 @@ The highest-priority remaining work is:
The best next implementation step is:
- start the frontend adapter behind the existing benchmark CLI
- add the frontend `script` adapter behind the existing benchmark CLI
Reason:
- the current CLI harness can now detect meaningful skill regressions, write official snapshots, and read them back
- that is a strong enough foundation to reuse for the frontend instead of delaying on CLI perfection
- the remaining CLI work should stay tracked as follow-up TODOs while the main implementation path moves to frontend coverage
- flow and app now already run through the shared benchmark CLI and can write official snapshots
- `script` is the main frontend gap left before the frontend surface is structurally complete
- the remaining CLI work should stay tracked as follow-up TODOs while the main implementation path moves to fuller frontend coverage
## Relevant Files
- Plan: [system-prompt-testing-plan.md](/home/farhad/windmill__worktrees/prompt-testing-plan/docs/system-prompt-testing-plan.md)
- Benchmark CLI: [index.ts](/home/farhad/windmill__worktrees/prompt-testing-plan/ai_evals/cli/index.ts)
- CLI adapter: [artifact-eval.ts](/home/farhad/windmill__worktrees/prompt-testing-plan/ai_evals/adapters/cli/artifact-eval.ts)
- Frontend adapter runtime: [runtime.ts](/home/farhad/windmill__worktrees/prompt-testing-plan/ai_evals/adapters/frontend/runtime.ts)
- Frontend benchmark runner: [frontendBenchmarkRunner.ts](/home/farhad/windmill__worktrees/prompt-testing-plan/frontend/src/lib/components/copilot/chat/__tests__/benchmarkCli/frontendBenchmarkRunner.ts)
- CLI variants: [variants.ts](/home/farhad/windmill__worktrees/prompt-testing-plan/ai_evals/adapters/cli/variants.ts)
- Shared guidance writer: [writer.ts](/home/farhad/windmill__worktrees/prompt-testing-plan/cli/src/guidance/writer.ts)
- CLI benchmark docs: [README.md](/home/farhad/windmill__worktrees/prompt-testing-plan/ai_evals/cli/README.md)
frontend/src/lib/components/copilot/chat/__tests__/app/appEvalComparison.ts
+1 -1
View File
@@ -121,7 +121,7 @@ Please evaluate how well the generated app:
try {
const response = await client.messages.create({
model: 'claude-sonnet-4-5-20250514',
model: 'claude-sonnet-4-6',
max_tokens: 2048,
system: APP_GENERATION_EVALUATOR_SYSTEM_PROMPT,
messages: [
frontend/src/lib/components/copilot/chat/__tests__/benchmarkCli/aiEvalsAdapter.test.ts
+48
View File
@@ -0,0 +1,48 @@
import { expect, it, vi } from 'vitest'
// @ts-ignore - Node.js fs/promises
import { mkdir, writeFile } from 'fs/promises'
// @ts-ignore - Node.js path
import { dirname, resolve } from 'path'
vi.mock('monaco-editor', () => ({
editor: {},
languages: {},
KeyCode: {},
Uri: {
parse: (value: string) => ({ toString: () => value })
},
MarkerSeverity: {
Error: 8,
Warning: 4,
Info: 2,
Hint: 1
}
}))
vi.mock('@codingame/monaco-vscode-standalone-typescript-language-features', () => ({
getTypeScriptWorker: async () => async () => ({}),
typescriptVersion: 'test'
}))
vi.mock('@codingame/monaco-vscode-languages-service-override', () => ({
default: () => ({})
}))
vi.mock('$lib/components/vscode', () => ({}))
const benchmarkOutputPath = process.env.WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH
const benchmarkIt = benchmarkOutputPath ? it : it.skip
benchmarkIt(
'runs the frontend benchmark adapter from environment input',
async () => {
const { runFrontendBenchmarkFromEnv } = await import('./frontendBenchmarkRunner')
const payload = await runFrontendBenchmarkFromEnv()
const absoluteOutputPath = resolve(benchmarkOutputPath!)
await mkdir(dirname(absoluteOutputPath), { recursive: true })
await writeFile(absoluteOutputPath, JSON.stringify(payload, null, 2) + '\n', 'utf8')
expect(payload.variants.length).toBeGreaterThan(0)
},
600_000
)
frontend/src/lib/components/copilot/chat/__tests__/benchmarkCli/frontendBenchmarkRunner.ts
+440
View File
@@ -0,0 +1,440 @@
// @ts-ignore - Node.js fs/promises
import { readFile } from 'fs/promises'
// @ts-ignore - Node.js path
import { dirname, join, resolve } from 'path'
// @ts-ignore - Node.js url
import { fileURLToPath } from 'url'
import type { AIProvider } from '$lib/gen/types.gen'
import { loadAppEvalCases, loadFlowEvalCases } from '../shared/evalCaseLoader'
import type { VariantConfig } from '../shared'
export type FrontendBenchmarkSurface = 'flow' | 'app'
interface VariantPromptConfigDefault {
type: 'default'
}
interface VariantPromptConfigDefaultWithCustom {
type: 'default-with-custom'
custom?: string
customPath?: string
}
interface VariantPromptConfigCustom {
type: 'custom'
content?: string
contentPath?: string
}
interface VariantToolsConfigDefault {
type: 'default'
}
interface VariantToolsConfigSubset {
type: 'subset'
include: string[]
}
export interface FrontendBenchmarkVariantManifest {
id: string
description?: string
provider?: AIProvider
model?: string
systemPrompt?:
| VariantPromptConfigDefault
| VariantPromptConfigDefaultWithCustom
| VariantPromptConfigCustom
tools?: VariantToolsConfigDefault | VariantToolsConfigSubset
}
export interface FrontendBenchmarkAttempt {
attempt: number
passed: boolean
durationMs: number
assistantMessageCount: number
toolCallCount: number
toolsUsed: string[]
checks: Array<{ name: string; passed: boolean; required?: boolean }>
requiredFailedChecks: string[]
judgeScore: number | null
judgeStatement: string | null
error: string | null
}
export interface FrontendBenchmarkCaseResult {
caseId: string
attempts: FrontendBenchmarkAttempt[]
}
export interface FrontendBenchmarkVariantResult {
variant: string
provider: AIProvider
model: string
judgeModel: string | null
caseResults: FrontendBenchmarkCaseResult[]
}
export interface FrontendBenchmarkPayload {
surface: FrontendBenchmarkSurface
runs: number
variants: FrontendBenchmarkVariantResult[]
}
const DEFAULT_MIN_JUDGE_SCORE = 80
const FRONTEND_JUDGE_MODEL = 'claude-sonnet-4-6'
export async function runFrontendBenchmarkFromEnv(): Promise<FrontendBenchmarkPayload> {
return runFrontendBenchmark({
surface: parseSurface(process.env.WMILL_FRONTEND_AI_EVAL_SURFACE),
caseIds: parseJsonStringArray(process.env.WMILL_FRONTEND_AI_EVAL_CASE_IDS, 'WMILL_FRONTEND_AI_EVAL_CASE_IDS'),
variantIds: parseJsonStringArray(
process.env.WMILL_FRONTEND_AI_EVAL_VARIANT_IDS,
'WMILL_FRONTEND_AI_EVAL_VARIANT_IDS'
),
runs: parsePositiveInteger(process.env.WMILL_FRONTEND_AI_EVAL_RUNS, 'WMILL_FRONTEND_AI_EVAL_RUNS')
})
}
export async function runFrontendBenchmark(input: {
surface: FrontendBenchmarkSurface
caseIds: string[]
variantIds: string[]
runs: number
}): Promise<FrontendBenchmarkPayload> {
if (input.variantIds.length === 0) {
throw new Error('Frontend benchmark requires at least one variant id')
}
switch (input.surface) {
case 'flow':
return await runFlowBenchmark(input)
case 'app':
return await runAppBenchmark(input)
default:
throw new Error(`Unsupported frontend benchmark surface: ${String(input.surface)}`)
}
}
function resolveRepoRoot(): string {
const currentDir = dirname(fileURLToPath(import.meta.url))
const repoRoot = resolve(currentDir, '../../../../../../../..')
return repoRoot
}
async function runFlowBenchmark(input: {
surface: 'flow'
caseIds: string[]
variantIds: string[]
runs: number
}): Promise<FrontendBenchmarkPayload> {
const { runFlowEval } = await import('../flow/flowEvalRunner')
const allCases = loadFlowEvalCases()
const selectedCases =
input.caseIds.length === 0
? allCases
: input.caseIds.map((caseId) => {
const testCase = allCases.find((entry) => entry.id === caseId)
if (!testCase) {
throw new Error(`Unknown frontend flow case: ${caseId}`)
}
return testCase
})
const variants = await Promise.all(
input.variantIds.map((variantId) => loadFrontendVariant('flow', variantId))
)
return {
surface: input.surface,
runs: input.runs,
variants: await Promise.all(
variants.map(async (variant) => ({
variant: variant.id,
provider: variant.provider,
model: variant.model,
judgeModel: FRONTEND_JUDGE_MODEL,
caseResults: await Promise.all(
selectedCases.map(async (testCase) => ({
caseId: testCase.id,
attempts: await runRepeated(input.runs, async (attempt) => {
const startedAt = Date.now()
const result = await runFlowEval(
testCase.userPrompt,
getApiKeyForProvider(variant.provider),
{
initialModules: testCase.initialFlow?.value?.modules,
initialSchema: testCase.initialFlow?.schema,
expectedFlow: testCase.expectedFlow,
variant: variant.config,
provider: variant.provider,
model: variant.model
}
)
const minJudgeScore = testCase.minJudgeScore ?? DEFAULT_MIN_JUDGE_SCORE
const checks = [
{ name: 'chat run succeeded', passed: result.success, required: true },
{
name: 'judge evaluation succeeded',
passed: Boolean(result.evaluationResult?.success),
required: true
},
{
name: `judge score >= ${minJudgeScore}`,
passed: (result.evaluationResult?.resemblanceScore ?? 0) >= minJudgeScore,
required: true
}
]
return {
attempt,
passed: checks.every((check) => check.passed),
durationMs: Date.now() - startedAt,
assistantMessageCount: result.iterations,
toolCallCount: result.toolCallsCount,
toolsUsed: uniqueStrings(result.toolsCalled),
checks,
requiredFailedChecks: checks
.filter((check) => check.required !== false && !check.passed)
.map((check) => check.name),
judgeScore: result.evaluationResult?.resemblanceScore ?? null,
judgeStatement: result.evaluationResult?.statement ?? null,
error: result.error ?? result.evaluationResult?.error ?? null
} satisfies FrontendBenchmarkAttempt
})
}))
)
}))
)
}
}
async function runAppBenchmark(input: {
surface: 'app'
caseIds: string[]
variantIds: string[]
runs: number
}): Promise<FrontendBenchmarkPayload> {
const { runAppEval } = await import('../app/appEvalRunner')
const { loadAppFixtureForEval } = await import('../app/appFixtureLoader')
const allCases = loadAppEvalCases()
const selectedCases =
input.caseIds.length === 0
? allCases
: input.caseIds.map((caseId) => {
const testCase = allCases.find((entry) => entry.id === caseId)
if (!testCase) {
throw new Error(`Unknown frontend app case: ${caseId}`)
}
return testCase
})
const variants = await Promise.all(
input.variantIds.map((variantId) => loadFrontendVariant('app', variantId))
)
return {
surface: input.surface,
runs: input.runs,
variants: await Promise.all(
variants.map(async (variant) => ({
variant: variant.id,
provider: variant.provider,
model: variant.model,
judgeModel: FRONTEND_JUDGE_MODEL,
caseResults: await Promise.all(
selectedCases.map(async (testCase) => {
const fixture = testCase.initialAppFixturePath
? await loadAppFixtureForEval(testCase.initialAppFixturePath)
: { initialFrontend: {}, initialBackend: {} }
return {
caseId: testCase.id,
attempts: await runRepeated(input.runs, async (attempt) => {
const startedAt = Date.now()
const result = await runAppEval(
testCase.userPrompt,
getApiKeyForProvider(variant.provider),
{
...fixture,
variant: variant.config,
provider: variant.provider,
model: variant.model
}
)
const minJudgeScore = testCase.minJudgeScore ?? DEFAULT_MIN_JUDGE_SCORE
const checks = [
{ name: 'chat run succeeded', passed: result.success, required: true },
{
name: 'judge evaluation succeeded',
passed: Boolean(result.evaluationResult?.success),
required: true
},
{
name: `judge score >= ${minJudgeScore}`,
passed: (result.evaluationResult?.resemblanceScore ?? 0) >= minJudgeScore,
required: true
}
]
return {
attempt,
passed: checks.every((check) => check.passed),
durationMs: Date.now() - startedAt,
assistantMessageCount: result.iterations,
toolCallCount: result.toolCallsCount,
toolsUsed: uniqueStrings(result.toolsCalled),
checks,
requiredFailedChecks: checks
.filter((check) => check.required !== false && !check.passed)
.map((check) => check.name),
judgeScore: result.evaluationResult?.resemblanceScore ?? null,
judgeStatement: result.evaluationResult?.statement ?? null,
error: result.error ?? result.evaluationResult?.error ?? null
} satisfies FrontendBenchmarkAttempt
})
}
})
)
}))
)
}
}
async function loadFrontendVariant(
surface: FrontendBenchmarkSurface,
variantId: string
): Promise<{
id: string
provider: AIProvider
model: string
config: VariantConfig
}> {
const repoRoot = resolveRepoRoot()
const manifestPath = join(
repoRoot,
'ai_evals',
'variants',
'frontend',
surface,
`${variantId}.json`
)
const manifest = JSON.parse(
await readFile(manifestPath, 'utf8')
) as FrontendBenchmarkVariantManifest
if (!manifest.id) {
throw new Error(`Frontend variant manifest missing id: ${manifestPath}`)
}
return {
id: manifest.id,
provider: manifest.provider ?? 'anthropic',
model: manifest.model ?? 'claude-haiku-4-5-20251001',
config: {
name: manifest.id,
description: manifest.description,
systemPrompt: manifest.systemPrompt
? await resolveSystemPromptConfig(manifestPath, manifest.systemPrompt)
: { type: 'default' },
tools: manifest.tools ?? { type: 'default' }
}
}
}
async function resolveSystemPromptConfig(
manifestPath: string,
config: FrontendBenchmarkVariantManifest['systemPrompt']
): Promise<VariantConfig['systemPrompt']> {
if (!config || config.type === 'default') {
return { type: 'default' }
}
if (config.type === 'default-with-custom') {
const custom = config.custom ?? (config.customPath ? await readPromptFile(manifestPath, config.customPath) : undefined)
if (!custom) {
throw new Error(
`Frontend variant ${manifestPath} must provide systemPrompt.custom or systemPrompt.customPath`
)
}
return {
type: 'default-with-custom',
custom
}
}
const content =
config.content ?? (config.contentPath ? await readPromptFile(manifestPath, config.contentPath) : undefined)
if (!content) {
throw new Error(
`Frontend variant ${manifestPath} must provide systemPrompt.content or systemPrompt.contentPath`
)
}
return {
type: 'custom',
content
}
}
async function readPromptFile(manifestPath: string, relativePath: string): Promise<string> {
return await readFile(join(dirname(manifestPath), relativePath), 'utf8')
}
function getApiKeyForProvider(provider: AIProvider): string {
if (provider === 'anthropic') {
const apiKey = process.env.ANTHROPIC_API_KEY
if (!apiKey) {
throw new Error('ANTHROPIC_API_KEY is required for frontend benchmark runs')
}
return apiKey
}
if (provider === 'openai') {
const apiKey = process.env.OPENAI_API_KEY
if (!apiKey) {
throw new Error('OPENAI_API_KEY is required for frontend benchmark runs')
}
return apiKey
}
throw new Error(`Unsupported frontend benchmark provider: ${provider}`)
}
async function runRepeated<T>(runs: number, fn: (attempt: number) => Promise<T>): Promise<T[]> {
const results: T[] = []
for (let attempt = 1; attempt <= runs; attempt += 1) {
results.push(await fn(attempt))
}
return results
}
function parseJsonStringArray(value: string | undefined, envName: string): string[] {
if (!value) {
throw new Error(`Missing required ${envName}`)
}
const parsed = JSON.parse(value) as unknown
if (!Array.isArray(parsed) || parsed.some((entry) => typeof entry !== 'string')) {
throw new Error(`${envName} must be a JSON string array`)
}
return parsed
}
function parseSurface(value: string | undefined): FrontendBenchmarkSurface {
if (value === 'flow' || value === 'app') {
return value
}
throw new Error('WMILL_FRONTEND_AI_EVAL_SURFACE must be "flow" or "app"')
}
function parsePositiveInteger(value: string | undefined, envName: string): number {
const parsed = Number.parseInt(value ?? '', 10)
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error(`${envName} must be a positive integer`)
}
return parsed
}
function uniqueStrings(values: string[]): string[] {
return [...new Set(values)].sort((left, right) => left.localeCompare(right))
}
frontend/src/lib/components/copilot/chat/__tests__/shared/baseLLMEvaluator.ts
+1 -1
View File
@@ -50,7 +50,7 @@ export async function evaluateWithLLM(params: EvaluateParams): Promise<Evaluatio
expectedOutput,
evaluatorSystemPrompt,
apiKey,
model = 'claude-sonnet-4-5-20250514'
model = 'claude-sonnet-4-6'
} = params
// @ts-ignore - process.env