From 239244872874e3449220741d6310b74756ccfb61 Mon Sep 17 00:00:00 2001 From: centdix Date: Wed, 1 Apr 2026 13:16:22 +0200 Subject: [PATCH] feat: add frontend benchmark surfaces Co-Authored-By: Claude Opus 4.5 --- ai_evals/README.md | 8 + ai_evals/adapters/frontend/manifests.ts | 51 ++ ai_evals/adapters/frontend/runtime.ts | 97 ++++ ai_evals/cli/README.md | 57 ++- ai_evals/cli/index.ts | 327 ++++++++++++- ai_evals/variants/frontend/app/baseline.json | 12 + ai_evals/variants/frontend/flow/baseline.json | 12 + docs/system-prompt-testing-status.md | 43 +- .../chat/__tests__/app/appEvalComparison.ts | 2 +- .../benchmarkCli/aiEvalsAdapter.test.ts | 48 ++ .../benchmarkCli/frontendBenchmarkRunner.ts | 440 ++++++++++++++++++ .../chat/__tests__/shared/baseLLMEvaluator.ts | 2 +- 12 files changed, 1068 insertions(+), 31 deletions(-) create mode 100644 ai_evals/adapters/frontend/manifests.ts create mode 100644 ai_evals/adapters/frontend/runtime.ts create mode 100644 ai_evals/variants/frontend/app/baseline.json create mode 100644 ai_evals/variants/frontend/flow/baseline.json create mode 100644 frontend/src/lib/components/copilot/chat/__tests__/benchmarkCli/aiEvalsAdapter.test.ts create mode 100644 frontend/src/lib/components/copilot/chat/__tests__/benchmarkCli/frontendBenchmarkRunner.ts diff --git a/ai_evals/README.md b/ai_evals/README.md index 988c4dd760..455ef6903d 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -56,9 +56,17 @@ Current usage: ```bash cd ai_evals bun run cli -- list-variants --surface cli +bun run cli -- list-variants --surface frontend-flow +bun run cli -- list-variants --surface frontend-app bun run cli -- snapshot-variant --surface cli --variant candidate bun run cli -- list-cases --surface cli +bun run cli -- list-cases --surface frontend-flow +bun run cli -- list-cases --surface frontend-app bun run cli -- run --surface cli --case bun-hello-script --variant baseline --runs 5 +bun run cli -- run --surface frontend-flow --case flow-test1-user-role-actions --variant baseline --runs 1 +bun run cli -- run --surface frontend-app --case app-test1-counter-create --variant baseline --runs 1 +bun run cli -- compare --surface frontend-flow --case flow-test1-user-role-actions --variant baseline --variant baseline --runs 1 +bun run cli -- compare --surface frontend-app --case app-test1-counter-create --variant baseline --variant baseline --runs 1 bun run cli -- compare --surface cli --case bun-hello-script --variant baseline --variant baseline --runs 5 bun run cli -- compare --surface cli --case bun-hello-script --variant baseline-frozen --variant candidate --runs 5 --write-history bun run cli -- history --view latest diff --git a/ai_evals/adapters/frontend/manifests.ts b/ai_evals/adapters/frontend/manifests.ts new file mode 100644 index 0000000000..b2375f3c0e --- /dev/null +++ b/ai_evals/adapters/frontend/manifests.ts @@ -0,0 +1,51 @@ +import { readdir, readFile } from 'node:fs/promises' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +export type FrontendSurfaceName = 'frontend-flow' | 'frontend-app' + +interface FrontendCaseManifest { + id: string + title: string +} + +interface FrontendVariantManifest { + id: string + description?: string +} + +const REPO_ROOT = fileURLToPath(new URL('../../../', import.meta.url)) + +export async function loadFrontendCases(surface: FrontendSurfaceName): Promise { + const raw = await readFile( + path.join(REPO_ROOT, 'ai_evals', 'cases', 'frontend', `${surfaceToManifestName(surface)}.json`), + 'utf8' + ) + return JSON.parse(raw) as FrontendCaseManifest[] +} + +export async function loadFrontendVariants( + surface: FrontendSurfaceName +): Promise { + const variantsDir = path.join( + REPO_ROOT, + 'ai_evals', + 'variants', + 'frontend', + surfaceToManifestName(surface) + ) + const filenames = (await readdir(variantsDir)) + .filter((entry) => entry.endsWith('.json')) + .sort((left, right) => left.localeCompare(right)) + + return await Promise.all( + filenames.map(async (filename) => { + const raw = await readFile(path.join(variantsDir, filename), 'utf8') + return JSON.parse(raw) as FrontendVariantManifest + }) + ) +} + +function surfaceToManifestName(surface: FrontendSurfaceName): 'flow' | 'app' { + return surface === 'frontend-flow' ? 'flow' : 'app' +} diff --git a/ai_evals/adapters/frontend/runtime.ts b/ai_evals/adapters/frontend/runtime.ts new file mode 100644 index 0000000000..0a2ec17f6b --- /dev/null +++ b/ai_evals/adapters/frontend/runtime.ts @@ -0,0 +1,97 @@ +import { execFile as execFileCallback } from 'node:child_process' +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { promisify } from 'node:util' +import { fileURLToPath } from 'node:url' + +const execFile = promisify(execFileCallback) +const REPO_ROOT = fileURLToPath(new URL('../../../', import.meta.url)) +const FRONTEND_DIR = path.join(REPO_ROOT, 'frontend') +const FRONTEND_BENCHMARK_TEST = 'src/lib/components/copilot/chat/__tests__/benchmarkCli/aiEvalsAdapter.test.ts' + +export type FrontendSurfaceName = 'frontend-flow' | 'frontend-app' + +export interface FrontendAdapterAttempt { + attempt: number + passed: boolean + durationMs: number + assistantMessageCount: number + toolCallCount: number + toolsUsed: string[] + checks: Array<{ name: string; passed: boolean; required?: boolean }> + requiredFailedChecks: string[] + judgeScore: number | null + judgeStatement: string | null + error: string | null +} + +export interface FrontendAdapterCaseResult { + caseId: string + attempts: FrontendAdapterAttempt[] +} + +export interface FrontendAdapterVariantResult { + variant: string + provider: string + model: string + judgeModel: string | null + caseResults: FrontendAdapterCaseResult[] +} + +export interface FrontendAdapterPayload { + surface: 'flow' | 'app' + runs: number + variants: FrontendAdapterVariantResult[] +} + +export async function runFrontendBenchmarkAdapter(input: { + surface: FrontendSurfaceName + caseIds: string[] + variantIds: string[] + runs: number +}): Promise { + const tempDir = await mkdtemp(path.join(tmpdir(), 'wmill-frontend-benchmark-')) + const outputPath = path.join(tempDir, 'result.json') + + try { + await execFile( + path.join(FRONTEND_DIR, 'node_modules', '.bin', 'vitest'), + [ + 'run', + FRONTEND_BENCHMARK_TEST, + '--project', + 'server', + '--config', + 'vite.config.js' + ], + { + cwd: FRONTEND_DIR, + env: { + ...process.env, + WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH: outputPath, + WMILL_FRONTEND_AI_EVAL_SURFACE: frontendSurfaceToAdapterSurface(input.surface), + WMILL_FRONTEND_AI_EVAL_CASE_IDS: JSON.stringify(input.caseIds), + WMILL_FRONTEND_AI_EVAL_VARIANT_IDS: JSON.stringify(input.variantIds), + WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs) + }, + maxBuffer: 10 * 1024 * 1024 + } + ) + + const raw = await readFile(outputPath, 'utf8') + return JSON.parse(raw) as FrontendAdapterPayload + } catch (error) { + const executionError = error as Error & { stdout?: string; stderr?: string } + const details = [executionError.message, executionError.stdout, executionError.stderr] + .filter(Boolean) + .join('\n') + throw new Error(`Frontend benchmark adapter failed:\n${details}`) + } finally { + await rm(tempDir, { recursive: true, force: true }) + } +} + +function frontendSurfaceToAdapterSurface(surface: FrontendSurfaceName): 'flow' | 'app' { + return surface === 'frontend-flow' ? 'flow' : 'app' +} diff --git a/ai_evals/cli/README.md b/ai_evals/cli/README.md index 92849c7646..982174e23a 100644 --- a/ai_evals/cli/README.md +++ b/ai_evals/cli/README.md @@ -11,7 +11,7 @@ The current implementation is intentionally small: - `compare` command - `history` command - `list-cases` and `list-variants` discovery commands -- `cli` surface only +- `cli`, `frontend-flow`, and `frontend-app` surfaces This is the benchmark entrypoint for prompt and artifact evaluation. @@ -35,6 +35,14 @@ cd ai_evals bun run cli -- list-cases --surface cli ``` +List available frontend cases: + +```bash +cd ai_evals +bun run cli -- list-cases --surface frontend-flow +bun run cli -- list-cases --surface frontend-app +``` + List available CLI variants: ```bash @@ -42,6 +50,14 @@ cd ai_evals bun run cli -- list-variants --surface cli ``` +List available frontend variants: + +```bash +cd ai_evals +bun run cli -- list-variants --surface frontend-flow +bun run cli -- list-variants --surface frontend-app +``` + Snapshot the current guidance bundle into a named CLI variant: ```bash @@ -56,6 +72,20 @@ cd ai_evals bun run cli -- run --surface cli --case bun-hello-script --variant baseline ``` +Run one frontend flow case: + +```bash +cd ai_evals +bun run cli -- run --surface frontend-flow --case flow-test1-user-role-actions --variant baseline --runs 1 +``` + +Run one frontend app case: + +```bash +cd ai_evals +bun run cli -- run --surface frontend-app --case app-test1-counter-create --variant baseline --runs 1 +``` + Keep the temp workspace for inspection: ```bash @@ -91,6 +121,20 @@ cd ai_evals bun run cli -- compare --surface cli --case bun-hello-script --variant baseline-frozen --variant candidate --runs 5 --write-history ``` +Compare frontend prompt variants: + +```bash +cd ai_evals +bun run cli -- compare --surface frontend-flow --case flow-test1-user-role-actions --variant baseline --variant candidate --runs 3 +``` + +Write official history snapshots for frontend variants: + +```bash +cd ai_evals +bun run cli -- compare --surface frontend-app --case app-test1-counter-create --variant baseline --variant candidate --runs 3 --write-history +``` + Inspect the tracked history: ```bash @@ -157,6 +201,15 @@ official snapshot per compared variant into `ai_evals/history/` and rebuilds: The compare output also includes tool usage and invoked skills as diagnostics. +For frontend surfaces, the benchmark CLI shells into a frontend-native Vitest +adapter so the evaluation runs with the frontend project's own module +resolution and test environment, while still producing the same benchmark +result shape as the CLI surface. + +By default, the repo only ships `baseline` frontend variants. To compare +distinct prompt variants or write official history snapshots, add a second +manifest under `ai_evals/variants/frontend//`. + True efficiency metrics such as latency, token usage, and cost are planned, but the current CLI does not emit them yet. Until that lands, use `compare` primarily to answer "did this skill bundle produce better artifacts on the same @@ -193,7 +246,7 @@ instead of the generated default. Later iterations should add: -- frontend adapters - variant cleanup and diff helpers - token and cost metrics in compare output - richer history views and filtering +- frontend `script` surface diff --git a/ai_evals/cli/index.ts b/ai_evals/cli/index.ts index 0ece159f44..3bb086b429 100644 --- a/ai_evals/cli/index.ts +++ b/ai_evals/cli/index.ts @@ -24,6 +24,15 @@ import { loadLatestHistoryRollup, loadSummaryHistory } from "../history/writer.mjs"; +import { + loadFrontendCases, + loadFrontendVariants, + type FrontendSurfaceName, +} from "../adapters/frontend/manifests"; +import { + runFrontendBenchmarkAdapter, + type FrontendAdapterPayload, +} from "../adapters/frontend/runtime"; type CommandName = | "run" @@ -32,7 +41,7 @@ type CommandName = | "snapshot-variant" | "compare" | "history"; -type SurfaceName = "cli"; +type SurfaceName = "cli" | FrontendSurfaceName; interface ParsedArgs { command: CommandName; @@ -62,6 +71,8 @@ interface AttemptSummary { requiredFailedChecks: string[]; expectedFiles: Array<{ path: string; exists: boolean }>; pathSignature: string; + judgeScore: number | null; + error: string | null; } interface AggregateMetrics { @@ -77,6 +88,9 @@ interface AggregateMetrics { distinctSkillsInvoked: string[]; distinctToolsUsed: string[]; requiredFailureCounts: Array<{ name: string; count: number }>; + judgeScoreMean: number | null; + judgeScoreMedian: number | null; + judgeScoreP10: number | null; } interface CompareCaseResult { @@ -99,6 +113,9 @@ interface CompareCaseResult { interface CompareVariantResult { label: string; variant: string; + provider: string; + model: string; + judgeModel: string | null; totalCases: number; fullyPassedCases: number; totalRuns: number; @@ -113,6 +130,9 @@ interface CompareVariantResult { distinctSkillsInvoked: string[]; distinctToolsUsed: string[]; requiredFailureCounts: Array<{ name: string; count: number }>; + judgeScoreMean: number | null; + judgeScoreMedian: number | null; + judgeScoreP10: number | null; caseResults: CompareCaseResult[]; } @@ -170,6 +190,30 @@ async function handleListCases(args: ParsedArgs) { } return; } + case "frontend-flow": + case "frontend-app": { + const cases = await loadFrontendCases(surface); + const payload = { + surface, + cases: cases.map((entry) => ({ + id: entry.id, + description: entry.title ?? null + })) + }; + + if (args.json) { + process.stdout.write(JSON.stringify(payload, null, 2) + "\n"); + return; + } + + process.stdout.write(`Surface: ${surface}\n`); + for (const entry of payload.cases) { + process.stdout.write( + `- ${entry.id}${entry.description ? `: ${entry.description}` : ""}\n` + ); + } + return; + } default: assertNever(surface); } @@ -202,6 +246,30 @@ async function handleListVariants(args: ParsedArgs) { } return; } + case "frontend-flow": + case "frontend-app": { + const variants = await loadFrontendVariants(surface); + const payload = { + surface, + variants: variants.map((entry) => ({ + id: entry.id, + description: entry.description ?? null + })) + }; + + if (args.json) { + process.stdout.write(JSON.stringify(payload, null, 2) + "\n"); + return; + } + + process.stdout.write(`Surface: ${surface}\n`); + for (const entry of payload.variants) { + process.stdout.write( + `- ${entry.id}${entry.description ? `: ${entry.description}` : ""}\n` + ); + } + return; + } default: assertNever(surface); } @@ -261,6 +329,30 @@ async function handleRun(args: ParsedArgs) { } return; } + case "frontend-flow": + case "frontend-app": { + if (args.keepWorkspace) { + throw new Error("--keep-workspace is not supported for frontend benchmark surfaces"); + } + const variantId = requireSingleVariantId(args.variantIds, "baseline"); + const payload = await buildFrontendRunPayload({ + surface, + caseId, + variantId, + runs: args.runs + }); + + if (args.json) { + process.stdout.write(JSON.stringify(payload, null, 2) + "\n"); + } else { + printRunSummary(payload); + } + + if (!payload.passed) { + process.exitCode = 1; + } + return; + } default: assertNever(surface); } @@ -295,6 +387,9 @@ async function handleSnapshotVariant(args: ParsedArgs) { return; } + case "frontend-flow": + case "frontend-app": + throw new Error("snapshot-variant is currently supported for the cli surface only"); default: assertNever(surface); } @@ -367,6 +462,9 @@ async function handleCompare(args: ParsedArgs) { internalVariantResults.push({ label: labeledVariant.label, variant: labeledVariant.variant.id, + provider: CLI_BENCHMARK_PROVIDER, + model: CLI_BENCHMARK_MODEL, + judgeModel: null, totalCases: caseResults.length, fullyPassedCases: caseResults.filter( (entry) => entry.passedRuns === entry.totalRuns @@ -383,6 +481,9 @@ async function handleCompare(args: ParsedArgs) { distinctSkillsInvoked: aggregate.distinctSkillsInvoked, distinctToolsUsed: aggregate.distinctToolsUsed, requiredFailureCounts: aggregate.requiredFailureCounts, + judgeScoreMean: aggregate.judgeScoreMean, + judgeScoreMedian: aggregate.judgeScoreMedian, + judgeScoreP10: aggregate.judgeScoreP10, caseResults, allAttempts }); @@ -414,6 +515,53 @@ async function handleCompare(args: ParsedArgs) { return; } + case "frontend-flow": + case "frontend-app": { + if (args.variantIds.length < 2) { + throw new Error("compare requires at least two --variant values"); + } + if (args.writeHistory && new Set(args.variantIds).size !== args.variantIds.length) { + throw new Error( + "--write-history requires distinct variant ids so official snapshots stay unambiguous" + ); + } + + const internalVariantResults = await buildFrontendCompareResults({ + surface, + caseIds: args.caseIds, + variantIds: args.variantIds, + runs: args.runs + }); + + const historyWrites = args.writeHistory + ? await writeCompareHistorySnapshots({ + surface, + runs: args.runs, + variants: internalVariantResults, + historyDir: args.historyDir + }) + : []; + + const payload = { + command: "compare", + surface, + caseIds: + args.caseIds.length === 0 + ? internalVariantResults[0]?.caseResults.map((entry) => entry.caseId) ?? [] + : args.caseIds, + runs: args.runs, + variants: internalVariantResults.map(({ allAttempts: _allAttempts, ...variant }) => variant), + historyWrites + }; + + if (args.json) { + process.stdout.write(JSON.stringify(payload, null, 2) + "\n"); + } else { + printCompareSummary(payload); + } + + return; + } default: assertNever(surface); } @@ -475,6 +623,137 @@ async function handleHistory(args: ParsedArgs) { } } +async function buildFrontendRunPayload(input: { + surface: FrontendSurfaceName; + caseId: string; + variantId: string; + runs: number; +}) { + const compareResults = await buildFrontendCompareResults({ + surface: input.surface, + caseIds: [input.caseId], + variantIds: [input.variantId], + runs: input.runs + }); + const variant = compareResults[0]; + + return { + command: "run", + surface: input.surface, + variant: variant.variant, + caseId: input.caseId, + runs: input.runs, + passed: variant.passedRuns === variant.totalRuns, + passedRuns: variant.passedRuns, + totalRuns: variant.totalRuns, + passRate: variant.passRate, + workspaceKept: false, + workspaceDir: null, + metrics: { + averageDurationMs: variant.averageDurationMs, + averageAssistantMessages: variant.averageAssistantMessages, + averageToolCalls: variant.averageToolCalls, + averageSkillInvocations: variant.averageSkillInvocations + }, + distinctSkillsInvoked: variant.distinctSkillsInvoked, + distinctToolsUsed: variant.distinctToolsUsed, + requiredFailureCounts: variant.requiredFailureCounts, + attempts: variant.allAttempts + }; +} + +async function buildFrontendCompareResults(input: { + surface: FrontendSurfaceName; + caseIds: string[]; + variantIds: string[]; + runs: number; +}): Promise> { + const adapterResult = await runFrontendBenchmarkAdapter({ + surface: input.surface, + caseIds: input.caseIds, + variantIds: input.variantIds, + runs: input.runs + }); + + return adapterResult.variants.map((variant) => { + const caseResults = variant.caseResults.map((caseResult) => { + const attempts = caseResult.attempts.map(summarizeFrontendAttempt); + const aggregate = aggregateAttempts(attempts); + return { + caseId: caseResult.caseId, + totalRuns: aggregate.totalRuns, + passedRuns: aggregate.passedRuns, + passRate: aggregate.passRate, + averageDurationMs: aggregate.averageDurationMs, + medianDurationMs: aggregate.medianDurationMs, + latencyPerSuccessMs: aggregate.latencyPerSuccessMs, + averageAssistantMessages: aggregate.averageAssistantMessages, + averageToolCalls: aggregate.averageToolCalls, + averageSkillInvocations: aggregate.averageSkillInvocations, + pathConsistency: computePathConsistency(attempts), + distinctSkillsInvoked: aggregate.distinctSkillsInvoked, + distinctToolsUsed: aggregate.distinctToolsUsed, + requiredFailureCounts: aggregate.requiredFailureCounts + }; + }); + + const allAttempts = variant.caseResults.flatMap((caseResult) => + caseResult.attempts.map(summarizeFrontendAttempt) + ); + const aggregate = aggregateAttempts(allAttempts); + + return { + label: variant.variant, + variant: variant.variant, + provider: variant.provider, + model: variant.model, + judgeModel: variant.judgeModel, + totalCases: caseResults.length, + fullyPassedCases: caseResults.filter((entry) => entry.passedRuns === entry.totalRuns) + .length, + totalRuns: aggregate.totalRuns, + passedRuns: aggregate.passedRuns, + passRate: aggregate.passRate, + averageDurationMs: aggregate.averageDurationMs, + medianDurationMs: aggregate.medianDurationMs, + latencyPerSuccessMs: aggregate.latencyPerSuccessMs, + averageAssistantMessages: aggregate.averageAssistantMessages, + averageToolCalls: aggregate.averageToolCalls, + averageSkillInvocations: aggregate.averageSkillInvocations, + distinctSkillsInvoked: aggregate.distinctSkillsInvoked, + distinctToolsUsed: aggregate.distinctToolsUsed, + requiredFailureCounts: aggregate.requiredFailureCounts, + judgeScoreMean: aggregate.judgeScoreMean, + judgeScoreMedian: aggregate.judgeScoreMedian, + judgeScoreP10: aggregate.judgeScoreP10, + caseResults, + allAttempts + }; + }); +} + +function summarizeFrontendAttempt( + attempt: FrontendAdapterPayload["variants"][number]["caseResults"][number]["attempts"][number] +): AttemptSummary { + const toolsUsed = uniqueStrings(attempt.toolsUsed); + return { + attempt: attempt.attempt, + passed: attempt.passed, + durationMs: attempt.durationMs, + assistantMessageCount: attempt.assistantMessageCount, + toolCallCount: attempt.toolCallCount, + skillInvocationCount: 0, + skillsInvoked: [], + toolsUsed, + checks: attempt.checks, + requiredFailedChecks: attempt.requiredFailedChecks, + expectedFiles: [], + pathSignature: buildPathSignature([], toolsUsed), + judgeScore: attempt.judgeScore, + error: attempt.error + }; +} + async function runCaseAttempts( evalCase: Awaited>[number], variant: CliVariant, @@ -534,13 +813,18 @@ function summarizeAttempt( path: file.path, exists: file.exists })), - pathSignature: buildPathSignature(skillsInvoked, toolsUsed) + pathSignature: buildPathSignature(skillsInvoked, toolsUsed), + judgeScore: null, + error: result.error ?? null }; } function aggregateAttempts(attempts: AttemptSummary[]): AggregateMetrics { const requiredFailureCounts = new Map(); const successfulAttempts = attempts.filter((attempt) => attempt.passed); + const judgeScores = attempts + .map((attempt) => attempt.judgeScore) + .filter((score): score is number => typeof score === "number" && Number.isFinite(score)); for (const attempt of attempts) { for (const failure of attempt.requiredFailedChecks) { @@ -562,7 +846,10 @@ function aggregateAttempts(attempts: AttemptSummary[]): AggregateMetrics { distinctToolsUsed: uniqueStrings(attempts.flatMap((attempt) => attempt.toolsUsed)), requiredFailureCounts: [...requiredFailureCounts.entries()] .sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0])) - .map(([name, count]) => ({ name, count })) + .map(([name, count]) => ({ name, count })), + judgeScoreMean: judgeScores.length > 0 ? average(judgeScores) : null, + judgeScoreMedian: judgeScores.length > 0 ? median(judgeScores) : null, + judgeScoreP10: judgeScores.length > 0 ? percentile(judgeScores, 0.1) : null }; } @@ -684,7 +971,7 @@ function requireSurface(surface: string | undefined): SurfaceName { if (!surface) { throw new Error("Missing required --surface argument"); } - if (surface !== "cli") { + if (surface !== "cli" && surface !== "frontend-flow" && surface !== "frontend-app") { throw new Error(`Unsupported surface for now: ${surface}`); } return surface; @@ -947,18 +1234,18 @@ function buildOfficialRun(input: { scoring_version: "cli-deterministic-v1", surface: input.surface, variant_name: input.variant.variant, - provider: CLI_BENCHMARK_PROVIDER, - model: CLI_BENCHMARK_MODEL, - judge_model: null, + provider: input.variant.provider, + model: input.variant.model, + judge_model: input.variant.judgeModel, runs_per_case: input.runs, case_count: input.variant.caseResults.length, metrics: { quality: { pass_rate: input.variant.passRate, deterministic_pass_rate: input.variant.passRate, - judge_score_mean: 0, - judge_score_median: 0, - judge_score_p10: 0, + judge_score_mean: input.variant.judgeScoreMean ?? 0, + judge_score_median: input.variant.judgeScoreMedian ?? 0, + judge_score_p10: input.variant.judgeScoreP10 ?? 0, quality_score: qualityScore }, reliability: { @@ -1127,14 +1414,18 @@ function printHelp() { [ "Usage:", " cd ai_evals && bun run cli -- list-cases --surface cli [--json]", + " cd ai_evals && bun run cli -- list-cases --surface frontend-flow [--json]", + " cd ai_evals && bun run cli -- list-cases --surface frontend-app [--json]", " cd ai_evals && bun run cli -- list-variants --surface cli [--json]", + " cd ai_evals && bun run cli -- list-variants --surface frontend-flow [--json]", + " cd ai_evals && bun run cli -- list-variants --surface frontend-app [--json]", " cd ai_evals && bun run cli -- snapshot-variant --surface cli --variant [--description ] [--json]", - " cd ai_evals && bun run cli -- run --surface cli --case [--variant ] [--runs ] [--json] [--keep-workspace]", - " cd ai_evals && bun run cli -- compare --surface cli [--case ...] [--variant ...] [--runs ] [--write-history] [--history-dir ] [--json]", + " cd ai_evals && bun run cli -- run --surface --case [--variant ] [--runs ] [--json] [--keep-workspace]", + " cd ai_evals && bun run cli -- compare --surface [--case ...] [--variant ...] [--runs ] [--write-history] [--history-dir ] [--json]", " cd ai_evals && bun run cli -- history [--view latest|summary|surface|variant|model] [--limit ] [--history-dir ] [--json]", "", "Current support:", - " surfaces: cli" + " surfaces: cli, frontend-flow, frontend-app" ].join("\n") + "\n" ); } @@ -1165,6 +1456,16 @@ function median(values: number[]): number { return sorted[middle]; } +function percentile(values: number[], ratio: number): number { + if (values.length === 0) { + return 0; + } + + const sorted = [...values].sort((left, right) => left - right); + const index = Math.max(0, Math.min(sorted.length - 1, Math.floor((sorted.length - 1) * ratio))); + return sorted[index]; +} + function uniqueStrings(values: string[]): string[] { return [...new Set(values)].sort((left, right) => left.localeCompare(right)); } diff --git a/ai_evals/variants/frontend/app/baseline.json b/ai_evals/variants/frontend/app/baseline.json new file mode 100644 index 0000000000..e0b2c6957b --- /dev/null +++ b/ai_evals/variants/frontend/app/baseline.json @@ -0,0 +1,12 @@ +{ + "id": "baseline", + "description": "Production frontend app chat prompt and tools.", + "provider": "anthropic", + "model": "claude-haiku-4-5-20251001", + "systemPrompt": { + "type": "default" + }, + "tools": { + "type": "default" + } +} diff --git a/ai_evals/variants/frontend/flow/baseline.json b/ai_evals/variants/frontend/flow/baseline.json new file mode 100644 index 0000000000..c82efa01b4 --- /dev/null +++ b/ai_evals/variants/frontend/flow/baseline.json @@ -0,0 +1,12 @@ +{ + "id": "baseline", + "description": "Production frontend flow chat prompt and tools.", + "provider": "anthropic", + "model": "claude-haiku-4-5-20251001", + "systemPrompt": { + "type": "default" + }, + "tools": { + "type": "default" + } +} diff --git a/docs/system-prompt-testing-status.md b/docs/system-prompt-testing-status.md index f4901a7479..49e921cf63 100644 --- a/docs/system-prompt-testing-status.md +++ b/docs/system-prompt-testing-status.md @@ -18,6 +18,7 @@ What is true today: - there is one repo-level benchmark CLI under `ai_evals/cli` - the CLI surface is the first real benchmark adapter behind it +- frontend flow and app are now exposed through that same benchmark CLI - CLI cases are now skill-sensitive rather than only artifact-sensitive - the benchmark harness and `wmill init` now share the same AI-guidance writer - variants can be frozen as named snapshots through the benchmark CLI @@ -28,7 +29,8 @@ What is true today: What is not true yet: -- frontend is not yet exposed through the benchmark CLI +- frontend `script` is not yet exposed through the benchmark CLI +- frontend still has only baseline prompt variants wired by default - CLI still needs broader case coverage and richer efficiency metrics - token and cost metrics are not implemented - no UI studio exists yet @@ -87,10 +89,11 @@ Implemented: - `snapshot-variant` - `history` - CLI adapter selection through `--surface cli` +- frontend adapter selection through `--surface frontend-flow|frontend-app` Still missing: -- frontend adapter selection +- frontend `script` adapter selection ### Phase 3: Replace the CLI smoke suite with real artifact evaluation @@ -164,6 +167,8 @@ Implemented: - benchmark CLI `history --view latest|summary|surface|variant|model` - official run snapshots generated from repeated CLI benchmark results - history rollups rebuilt from real benchmark writes +- shared history writing path generalized so frontend surfaces can use the same official run format +- frontend history writes validated through `compare --write-history` on `frontend-app` Still missing: @@ -184,21 +189,26 @@ Planned: Status: -- mostly not done +- partially done Implemented: - shared frontend case scaffolding exists - current frontend flow/app evals were moved toward shared manifests earlier +- repo-level benchmark CLI support for `frontend-flow` and `frontend-app` +- frontend-native adapter runner that executes headless through the frontend Vitest environment +- repeated runs for frontend flow/app through the shared benchmark CLI +- frontend prompt variants loaded from files under `ai_evals/variants/frontend/` +- frontend run/compare payloads aligned to the shared benchmark result shape +- frontend judge score aggregation wired into compare output and official run generation +- frontend official history writes validated through the shared benchmark CLI Still missing: -- frontend adapter behind the benchmark CLI -- scored frontend reliability tests - frontend `script` support -- repeated runs -- shared result/history alignment -- prompt-variant workflow for frontend +- more than the baseline prompt variant for frontend flow/app +- broader frontend reliability coverage across more cases in routine use +- regular use of frontend history writes in routine benchmark workflows ### Phase 6: Add CI tiers @@ -253,6 +263,9 @@ The most important implemented changes so far are: - `bun-hello-flow` - Made CLI evals skill-sensitive instead of allowing silent skill bypass - Added repeated-run CLI benchmarking with aggregate reporting +- Added frontend flow/app benchmark adapters behind the shared benchmark CLI +- Added frontend run/compare support with shared aggregation and judge score reporting +- Validated frontend `run`, `compare`, and `compare --write-history` through the shared benchmark CLI - Shared AI-guidance generation between: - benchmark temp workspaces - `wmill init` @@ -264,8 +277,8 @@ The most important implemented changes so far are: The highest-priority remaining work is: -1. Bring frontend behind the same benchmark CLI. -2. Expand the CLI case corpus to cover more real skill behavior. +1. Add the missing frontend `script` surface behind the same benchmark CLI. +2. Expand the frontend and CLI case corpora to cover more real prompt and skill behavior. 3. Add richer aggregate metrics: - pass rate - flake rate @@ -283,19 +296,21 @@ The highest-priority remaining work is: The best next implementation step is: -- start the frontend adapter behind the existing benchmark CLI +- add the frontend `script` adapter behind the existing benchmark CLI Reason: -- the current CLI harness can now detect meaningful skill regressions, write official snapshots, and read them back -- that is a strong enough foundation to reuse for the frontend instead of delaying on CLI perfection -- the remaining CLI work should stay tracked as follow-up TODOs while the main implementation path moves to frontend coverage +- flow and app now already run through the shared benchmark CLI and can write official snapshots +- `script` is the main frontend gap left before the frontend surface is structurally complete +- the remaining CLI work should stay tracked as follow-up TODOs while the main implementation path moves to fuller frontend coverage ## Relevant Files - Plan: [system-prompt-testing-plan.md](/home/farhad/windmill__worktrees/prompt-testing-plan/docs/system-prompt-testing-plan.md) - Benchmark CLI: [index.ts](/home/farhad/windmill__worktrees/prompt-testing-plan/ai_evals/cli/index.ts) - CLI adapter: [artifact-eval.ts](/home/farhad/windmill__worktrees/prompt-testing-plan/ai_evals/adapters/cli/artifact-eval.ts) +- Frontend adapter runtime: [runtime.ts](/home/farhad/windmill__worktrees/prompt-testing-plan/ai_evals/adapters/frontend/runtime.ts) +- Frontend benchmark runner: [frontendBenchmarkRunner.ts](/home/farhad/windmill__worktrees/prompt-testing-plan/frontend/src/lib/components/copilot/chat/__tests__/benchmarkCli/frontendBenchmarkRunner.ts) - CLI variants: [variants.ts](/home/farhad/windmill__worktrees/prompt-testing-plan/ai_evals/adapters/cli/variants.ts) - Shared guidance writer: [writer.ts](/home/farhad/windmill__worktrees/prompt-testing-plan/cli/src/guidance/writer.ts) - CLI benchmark docs: [README.md](/home/farhad/windmill__worktrees/prompt-testing-plan/ai_evals/cli/README.md) diff --git a/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalComparison.ts b/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalComparison.ts index e6c795d445..41d86b2bf0 100644 --- a/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalComparison.ts +++ b/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalComparison.ts @@ -121,7 +121,7 @@ Please evaluate how well the generated app: try { const response = await client.messages.create({ - model: 'claude-sonnet-4-5-20250514', + model: 'claude-sonnet-4-6', max_tokens: 2048, system: APP_GENERATION_EVALUATOR_SYSTEM_PROMPT, messages: [ diff --git a/frontend/src/lib/components/copilot/chat/__tests__/benchmarkCli/aiEvalsAdapter.test.ts b/frontend/src/lib/components/copilot/chat/__tests__/benchmarkCli/aiEvalsAdapter.test.ts new file mode 100644 index 0000000000..e3aa4b151a --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/__tests__/benchmarkCli/aiEvalsAdapter.test.ts @@ -0,0 +1,48 @@ +import { expect, it, vi } from 'vitest' +// @ts-ignore - Node.js fs/promises +import { mkdir, writeFile } from 'fs/promises' +// @ts-ignore - Node.js path +import { dirname, resolve } from 'path' + +vi.mock('monaco-editor', () => ({ + editor: {}, + languages: {}, + KeyCode: {}, + Uri: { + parse: (value: string) => ({ toString: () => value }) + }, + MarkerSeverity: { + Error: 8, + Warning: 4, + Info: 2, + Hint: 1 + } +})) + +vi.mock('@codingame/monaco-vscode-standalone-typescript-language-features', () => ({ + getTypeScriptWorker: async () => async () => ({}), + typescriptVersion: 'test' +})) + +vi.mock('@codingame/monaco-vscode-languages-service-override', () => ({ + default: () => ({}) +})) + +vi.mock('$lib/components/vscode', () => ({})) + +const benchmarkOutputPath = process.env.WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH +const benchmarkIt = benchmarkOutputPath ? it : it.skip + +benchmarkIt( + 'runs the frontend benchmark adapter from environment input', + async () => { + const { runFrontendBenchmarkFromEnv } = await import('./frontendBenchmarkRunner') + const payload = await runFrontendBenchmarkFromEnv() + const absoluteOutputPath = resolve(benchmarkOutputPath!) + await mkdir(dirname(absoluteOutputPath), { recursive: true }) + await writeFile(absoluteOutputPath, JSON.stringify(payload, null, 2) + '\n', 'utf8') + + expect(payload.variants.length).toBeGreaterThan(0) + }, + 600_000 +) diff --git a/frontend/src/lib/components/copilot/chat/__tests__/benchmarkCli/frontendBenchmarkRunner.ts b/frontend/src/lib/components/copilot/chat/__tests__/benchmarkCli/frontendBenchmarkRunner.ts new file mode 100644 index 0000000000..f29d3a7681 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/__tests__/benchmarkCli/frontendBenchmarkRunner.ts @@ -0,0 +1,440 @@ +// @ts-ignore - Node.js fs/promises +import { readFile } from 'fs/promises' +// @ts-ignore - Node.js path +import { dirname, join, resolve } from 'path' +// @ts-ignore - Node.js url +import { fileURLToPath } from 'url' +import type { AIProvider } from '$lib/gen/types.gen' +import { loadAppEvalCases, loadFlowEvalCases } from '../shared/evalCaseLoader' +import type { VariantConfig } from '../shared' + +export type FrontendBenchmarkSurface = 'flow' | 'app' + +interface VariantPromptConfigDefault { + type: 'default' +} + +interface VariantPromptConfigDefaultWithCustom { + type: 'default-with-custom' + custom?: string + customPath?: string +} + +interface VariantPromptConfigCustom { + type: 'custom' + content?: string + contentPath?: string +} + +interface VariantToolsConfigDefault { + type: 'default' +} + +interface VariantToolsConfigSubset { + type: 'subset' + include: string[] +} + +export interface FrontendBenchmarkVariantManifest { + id: string + description?: string + provider?: AIProvider + model?: string + systemPrompt?: + | VariantPromptConfigDefault + | VariantPromptConfigDefaultWithCustom + | VariantPromptConfigCustom + tools?: VariantToolsConfigDefault | VariantToolsConfigSubset +} + +export interface FrontendBenchmarkAttempt { + attempt: number + passed: boolean + durationMs: number + assistantMessageCount: number + toolCallCount: number + toolsUsed: string[] + checks: Array<{ name: string; passed: boolean; required?: boolean }> + requiredFailedChecks: string[] + judgeScore: number | null + judgeStatement: string | null + error: string | null +} + +export interface FrontendBenchmarkCaseResult { + caseId: string + attempts: FrontendBenchmarkAttempt[] +} + +export interface FrontendBenchmarkVariantResult { + variant: string + provider: AIProvider + model: string + judgeModel: string | null + caseResults: FrontendBenchmarkCaseResult[] +} + +export interface FrontendBenchmarkPayload { + surface: FrontendBenchmarkSurface + runs: number + variants: FrontendBenchmarkVariantResult[] +} + +const DEFAULT_MIN_JUDGE_SCORE = 80 +const FRONTEND_JUDGE_MODEL = 'claude-sonnet-4-6' + +export async function runFrontendBenchmarkFromEnv(): Promise { + return runFrontendBenchmark({ + surface: parseSurface(process.env.WMILL_FRONTEND_AI_EVAL_SURFACE), + caseIds: parseJsonStringArray(process.env.WMILL_FRONTEND_AI_EVAL_CASE_IDS, 'WMILL_FRONTEND_AI_EVAL_CASE_IDS'), + variantIds: parseJsonStringArray( + process.env.WMILL_FRONTEND_AI_EVAL_VARIANT_IDS, + 'WMILL_FRONTEND_AI_EVAL_VARIANT_IDS' + ), + runs: parsePositiveInteger(process.env.WMILL_FRONTEND_AI_EVAL_RUNS, 'WMILL_FRONTEND_AI_EVAL_RUNS') + }) +} + +export async function runFrontendBenchmark(input: { + surface: FrontendBenchmarkSurface + caseIds: string[] + variantIds: string[] + runs: number +}): Promise { + if (input.variantIds.length === 0) { + throw new Error('Frontend benchmark requires at least one variant id') + } + + switch (input.surface) { + case 'flow': + return await runFlowBenchmark(input) + case 'app': + return await runAppBenchmark(input) + default: + throw new Error(`Unsupported frontend benchmark surface: ${String(input.surface)}`) + } +} + +function resolveRepoRoot(): string { + const currentDir = dirname(fileURLToPath(import.meta.url)) + const repoRoot = resolve(currentDir, '../../../../../../../..') + return repoRoot +} + +async function runFlowBenchmark(input: { + surface: 'flow' + caseIds: string[] + variantIds: string[] + runs: number +}): Promise { + const { runFlowEval } = await import('../flow/flowEvalRunner') + const allCases = loadFlowEvalCases() + const selectedCases = + input.caseIds.length === 0 + ? allCases + : input.caseIds.map((caseId) => { + const testCase = allCases.find((entry) => entry.id === caseId) + if (!testCase) { + throw new Error(`Unknown frontend flow case: ${caseId}`) + } + return testCase + }) + + const variants = await Promise.all( + input.variantIds.map((variantId) => loadFrontendVariant('flow', variantId)) + ) + + return { + surface: input.surface, + runs: input.runs, + variants: await Promise.all( + variants.map(async (variant) => ({ + variant: variant.id, + provider: variant.provider, + model: variant.model, + judgeModel: FRONTEND_JUDGE_MODEL, + caseResults: await Promise.all( + selectedCases.map(async (testCase) => ({ + caseId: testCase.id, + attempts: await runRepeated(input.runs, async (attempt) => { + const startedAt = Date.now() + const result = await runFlowEval( + testCase.userPrompt, + getApiKeyForProvider(variant.provider), + { + initialModules: testCase.initialFlow?.value?.modules, + initialSchema: testCase.initialFlow?.schema, + expectedFlow: testCase.expectedFlow, + variant: variant.config, + provider: variant.provider, + model: variant.model + } + ) + + const minJudgeScore = testCase.minJudgeScore ?? DEFAULT_MIN_JUDGE_SCORE + const checks = [ + { name: 'chat run succeeded', passed: result.success, required: true }, + { + name: 'judge evaluation succeeded', + passed: Boolean(result.evaluationResult?.success), + required: true + }, + { + name: `judge score >= ${minJudgeScore}`, + passed: (result.evaluationResult?.resemblanceScore ?? 0) >= minJudgeScore, + required: true + } + ] + + return { + attempt, + passed: checks.every((check) => check.passed), + durationMs: Date.now() - startedAt, + assistantMessageCount: result.iterations, + toolCallCount: result.toolCallsCount, + toolsUsed: uniqueStrings(result.toolsCalled), + checks, + requiredFailedChecks: checks + .filter((check) => check.required !== false && !check.passed) + .map((check) => check.name), + judgeScore: result.evaluationResult?.resemblanceScore ?? null, + judgeStatement: result.evaluationResult?.statement ?? null, + error: result.error ?? result.evaluationResult?.error ?? null + } satisfies FrontendBenchmarkAttempt + }) + })) + ) + })) + ) + } +} + +async function runAppBenchmark(input: { + surface: 'app' + caseIds: string[] + variantIds: string[] + runs: number +}): Promise { + const { runAppEval } = await import('../app/appEvalRunner') + const { loadAppFixtureForEval } = await import('../app/appFixtureLoader') + const allCases = loadAppEvalCases() + const selectedCases = + input.caseIds.length === 0 + ? allCases + : input.caseIds.map((caseId) => { + const testCase = allCases.find((entry) => entry.id === caseId) + if (!testCase) { + throw new Error(`Unknown frontend app case: ${caseId}`) + } + return testCase + }) + + const variants = await Promise.all( + input.variantIds.map((variantId) => loadFrontendVariant('app', variantId)) + ) + + return { + surface: input.surface, + runs: input.runs, + variants: await Promise.all( + variants.map(async (variant) => ({ + variant: variant.id, + provider: variant.provider, + model: variant.model, + judgeModel: FRONTEND_JUDGE_MODEL, + caseResults: await Promise.all( + selectedCases.map(async (testCase) => { + const fixture = testCase.initialAppFixturePath + ? await loadAppFixtureForEval(testCase.initialAppFixturePath) + : { initialFrontend: {}, initialBackend: {} } + + return { + caseId: testCase.id, + attempts: await runRepeated(input.runs, async (attempt) => { + const startedAt = Date.now() + const result = await runAppEval( + testCase.userPrompt, + getApiKeyForProvider(variant.provider), + { + ...fixture, + variant: variant.config, + provider: variant.provider, + model: variant.model + } + ) + + const minJudgeScore = testCase.minJudgeScore ?? DEFAULT_MIN_JUDGE_SCORE + const checks = [ + { name: 'chat run succeeded', passed: result.success, required: true }, + { + name: 'judge evaluation succeeded', + passed: Boolean(result.evaluationResult?.success), + required: true + }, + { + name: `judge score >= ${minJudgeScore}`, + passed: (result.evaluationResult?.resemblanceScore ?? 0) >= minJudgeScore, + required: true + } + ] + + return { + attempt, + passed: checks.every((check) => check.passed), + durationMs: Date.now() - startedAt, + assistantMessageCount: result.iterations, + toolCallCount: result.toolCallsCount, + toolsUsed: uniqueStrings(result.toolsCalled), + checks, + requiredFailedChecks: checks + .filter((check) => check.required !== false && !check.passed) + .map((check) => check.name), + judgeScore: result.evaluationResult?.resemblanceScore ?? null, + judgeStatement: result.evaluationResult?.statement ?? null, + error: result.error ?? result.evaluationResult?.error ?? null + } satisfies FrontendBenchmarkAttempt + }) + } + }) + ) + })) + ) + } +} + +async function loadFrontendVariant( + surface: FrontendBenchmarkSurface, + variantId: string +): Promise<{ + id: string + provider: AIProvider + model: string + config: VariantConfig +}> { + const repoRoot = resolveRepoRoot() + const manifestPath = join( + repoRoot, + 'ai_evals', + 'variants', + 'frontend', + surface, + `${variantId}.json` + ) + const manifest = JSON.parse( + await readFile(manifestPath, 'utf8') + ) as FrontendBenchmarkVariantManifest + + if (!manifest.id) { + throw new Error(`Frontend variant manifest missing id: ${manifestPath}`) + } + + return { + id: manifest.id, + provider: manifest.provider ?? 'anthropic', + model: manifest.model ?? 'claude-haiku-4-5-20251001', + config: { + name: manifest.id, + description: manifest.description, + systemPrompt: manifest.systemPrompt + ? await resolveSystemPromptConfig(manifestPath, manifest.systemPrompt) + : { type: 'default' }, + tools: manifest.tools ?? { type: 'default' } + } + } +} + +async function resolveSystemPromptConfig( + manifestPath: string, + config: FrontendBenchmarkVariantManifest['systemPrompt'] +): Promise { + if (!config || config.type === 'default') { + return { type: 'default' } + } + + if (config.type === 'default-with-custom') { + const custom = config.custom ?? (config.customPath ? await readPromptFile(manifestPath, config.customPath) : undefined) + if (!custom) { + throw new Error( + `Frontend variant ${manifestPath} must provide systemPrompt.custom or systemPrompt.customPath` + ) + } + return { + type: 'default-with-custom', + custom + } + } + + const content = + config.content ?? (config.contentPath ? await readPromptFile(manifestPath, config.contentPath) : undefined) + if (!content) { + throw new Error( + `Frontend variant ${manifestPath} must provide systemPrompt.content or systemPrompt.contentPath` + ) + } + return { + type: 'custom', + content + } +} + +async function readPromptFile(manifestPath: string, relativePath: string): Promise { + return await readFile(join(dirname(manifestPath), relativePath), 'utf8') +} + +function getApiKeyForProvider(provider: AIProvider): string { + if (provider === 'anthropic') { + const apiKey = process.env.ANTHROPIC_API_KEY + if (!apiKey) { + throw new Error('ANTHROPIC_API_KEY is required for frontend benchmark runs') + } + return apiKey + } + + if (provider === 'openai') { + const apiKey = process.env.OPENAI_API_KEY + if (!apiKey) { + throw new Error('OPENAI_API_KEY is required for frontend benchmark runs') + } + return apiKey + } + + throw new Error(`Unsupported frontend benchmark provider: ${provider}`) +} + +async function runRepeated(runs: number, fn: (attempt: number) => Promise): Promise { + const results: T[] = [] + for (let attempt = 1; attempt <= runs; attempt += 1) { + results.push(await fn(attempt)) + } + return results +} + +function parseJsonStringArray(value: string | undefined, envName: string): string[] { + if (!value) { + throw new Error(`Missing required ${envName}`) + } + const parsed = JSON.parse(value) as unknown + if (!Array.isArray(parsed) || parsed.some((entry) => typeof entry !== 'string')) { + throw new Error(`${envName} must be a JSON string array`) + } + return parsed +} + +function parseSurface(value: string | undefined): FrontendBenchmarkSurface { + if (value === 'flow' || value === 'app') { + return value + } + throw new Error('WMILL_FRONTEND_AI_EVAL_SURFACE must be "flow" or "app"') +} + +function parsePositiveInteger(value: string | undefined, envName: string): number { + const parsed = Number.parseInt(value ?? '', 10) + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`${envName} must be a positive integer`) + } + return parsed +} + +function uniqueStrings(values: string[]): string[] { + return [...new Set(values)].sort((left, right) => left.localeCompare(right)) +} diff --git a/frontend/src/lib/components/copilot/chat/__tests__/shared/baseLLMEvaluator.ts b/frontend/src/lib/components/copilot/chat/__tests__/shared/baseLLMEvaluator.ts index bd7bd06d44..5f6e513a7e 100644 --- a/frontend/src/lib/components/copilot/chat/__tests__/shared/baseLLMEvaluator.ts +++ b/frontend/src/lib/components/copilot/chat/__tests__/shared/baseLLMEvaluator.ts @@ -50,7 +50,7 @@ export async function evaluateWithLLM(params: EvaluateParams): Promise