From 1b664f5525c20c8af5ee0955a2314769ea8f3a8d Mon Sep 17 00:00:00 2001 From: centdix Date: Tue, 7 Apr 2026 16:23:07 +0200 Subject: [PATCH] feat: add deterministic benchmark validators Co-Authored-By: Claude Opus 4.5 --- ai_evals/adapters/cli/artifact-eval.ts | 64 +-- ai_evals/adapters/frontend/benchmarkRunner.ts | 123 ++--- .../adapters/frontend/core/evalCaseLoader.ts | 4 +- ai_evals/adapters/shared/validators.ts | 473 ++++++++++++++++++ 4 files changed, 539 insertions(+), 125 deletions(-) create mode 100644 ai_evals/adapters/shared/validators.ts diff --git a/ai_evals/adapters/cli/artifact-eval.ts b/ai_evals/adapters/cli/artifact-eval.ts index e3d7246ba2..7b4bc3797a 100644 --- a/ai_evals/adapters/cli/artifact-eval.ts +++ b/ai_evals/adapters/cli/artifact-eval.ts @@ -7,10 +7,13 @@ import { loadEvalCases, type CliExpectedFileCheck } from "../shared/evalCases"; +import { + validateCliArtifact, + type BenchmarkCheck +} from "../shared/validators"; import { runPromptAndCapture, type PromptRunResult, - wasSkillInvoked } from "./runtime"; import type { CliVariant } from "./variants"; @@ -26,12 +29,7 @@ export interface CliArtifactEvalCase { expectedFiles: ExpectedFile[]; } -export interface ArtifactCheck { - name: string; - passed: boolean; - required?: boolean; - details?: string; -} +export type ArtifactCheck = BenchmarkCheck; export interface FileArtifactResult { path: string; @@ -178,48 +176,12 @@ function buildChecks( run: PromptRunResult, fileResults: FileArtifactResult[] ): ArtifactCheck[] { - const checks: ArtifactCheck[] = []; - - if (evalCase.expectedSkill) { - checks.push({ - name: `invokes ${evalCase.expectedSkill}`, - passed: wasSkillInvoked(run, evalCase.expectedSkill), - required: true, - details: `skills invoked: ${run.skillsInvoked.join(", ")}` - }); - } - - for (const expectedOutput of evalCase.expectedOutputSubstrings ?? []) { - checks.push({ - name: `mentions '${expectedOutput}' in assistant output`, - passed: run.output.includes(expectedOutput), - required: true - }); - } - - for (const expectedFile of evalCase.expectedFiles) { - const fileResult = fileResults.find((entry) => entry.path === expectedFile.path); - const content = fileResult?.content ?? ""; - - checks.push({ - name: `creates ${expectedFile.path}`, - passed: Boolean(fileResult?.exists) - }); - - for (const requiredSnippet of expectedFile.mustContain ?? []) { - checks.push({ - name: `${expectedFile.path} contains '${requiredSnippet}'`, - passed: content.includes(requiredSnippet) - }); - } - - for (const forbiddenSnippet of expectedFile.mustNotContain ?? []) { - checks.push({ - name: `${expectedFile.path} avoids '${forbiddenSnippet}'`, - passed: !content.includes(forbiddenSnippet) - }); - } - } - - return checks; + return validateCliArtifact({ + assistantOutput: run.output, + skillsInvoked: run.skillsInvoked, + expectedSkill: evalCase.expectedSkill, + expectedOutputSubstrings: evalCase.expectedOutputSubstrings, + expectedFiles: evalCase.expectedFiles, + fileResults + }); } diff --git a/ai_evals/adapters/frontend/benchmarkRunner.ts b/ai_evals/adapters/frontend/benchmarkRunner.ts index 2e8f2aeed7..742d938788 100644 --- a/ai_evals/adapters/frontend/benchmarkRunner.ts +++ b/ai_evals/adapters/frontend/benchmarkRunner.ts @@ -7,6 +7,16 @@ import { fileURLToPath } from 'url' import type { AIProvider } from '$lib/gen/types.gen' import { loadAppEvalCases, loadFlowEvalCases, loadScriptEvalCases } from './core/evalCaseLoader' import type { VariantConfig } from './core/shared' +import { + allRequiredChecksPassed, + buildJudgeChecks, + getRequiredFailedChecks, + requiredCheck, + validateAppArtifact, + validateFlowArtifact, + validateScriptArtifact, + type BenchmarkCheck +} from '../shared/validators' export type FrontendBenchmarkSurface = 'flow' | 'app' | 'script' @@ -54,7 +64,7 @@ export interface FrontendBenchmarkAttempt { assistantMessageCount: number toolCallCount: number toolsUsed: string[] - checks: Array<{ name: string; passed: boolean; required?: boolean }> + checks: BenchmarkCheck[] requiredFailedChecks: string[] judgeScore: number | null judgeStatement: string | null @@ -175,17 +185,19 @@ async function runFlowBenchmark(input: { const minJudgeScore = testCase.minJudgeScore ?? DEFAULT_MIN_JUDGE_SCORE const checks = [ - { name: 'chat run succeeded', passed: result.success, required: true }, - { - name: 'judge evaluation succeeded', - passed: Boolean(result.evaluationResult?.success), - required: true - }, - { - name: `judge score >= ${minJudgeScore}`, - passed: (result.evaluationResult?.resemblanceScore ?? 0) >= minJudgeScore, - required: true - } + requiredCheck('chat run succeeded', result.success, result.error), + ...validateFlowArtifact({ + generatedFlow: { + summary: result.flow.summary, + value: { modules: result.flow.value.modules }, + schema: result.flow.schema + }, + expectedFlow: testCase.expectedFlow + }), + ...buildJudgeChecks({ + evaluationResult: result.evaluationResult, + minJudgeScore + }) ] return { @@ -196,9 +208,7 @@ async function runFlowBenchmark(input: { toolCallCount: result.toolCallsCount, toolsUsed: uniqueStrings(result.toolsCalled), checks, - requiredFailedChecks: checks - .filter((check) => check.required !== false && !check.passed) - .map((check) => check.name), + requiredFailedChecks: getRequiredFailedChecks(checks), judgeScore: result.evaluationResult?.resemblanceScore ?? null, judgeStatement: result.evaluationResult?.statement ?? null, error: result.error ?? result.evaluationResult?.error ?? null @@ -267,17 +277,20 @@ async function runAppBenchmark(input: { const minJudgeScore = testCase.minJudgeScore ?? DEFAULT_MIN_JUDGE_SCORE const checks = [ - { name: 'chat run succeeded', passed: result.success, required: true }, - { - name: 'judge evaluation succeeded', - passed: Boolean(result.evaluationResult?.success), - required: true - }, - { - name: `judge score >= ${minJudgeScore}`, - passed: (result.evaluationResult?.resemblanceScore ?? 0) >= minJudgeScore, - required: true - } + requiredCheck('chat run succeeded', result.success, result.error), + ...validateAppArtifact({ + generatedApp: result.files, + initialApp: testCase.initialAppFixturePath + ? { + frontend: fixture.initialFrontend, + backend: fixture.initialBackend + } + : undefined + }), + ...buildJudgeChecks({ + evaluationResult: result.evaluationResult, + minJudgeScore + }) ] return { @@ -288,9 +301,7 @@ async function runAppBenchmark(input: { toolCallCount: result.toolCallsCount, toolsUsed: uniqueStrings(result.toolsCalled), checks, - requiredFailedChecks: checks - .filter((check) => check.required !== false && !check.passed) - .map((check) => check.name), + requiredFailedChecks: getRequiredFailedChecks(checks), judgeScore: result.evaluationResult?.resemblanceScore ?? null, judgeStatement: result.evaluationResult?.statement ?? null, error: result.error ?? result.evaluationResult?.error ?? null @@ -355,40 +366,16 @@ async function runScriptBenchmark(input: { const minJudgeScore = testCase.minJudgeScore ?? DEFAULT_MIN_JUDGE_SCORE const checks = [ - { name: 'chat run succeeded', passed: result.success, required: true }, - { - name: 'script exports entrypoint', - passed: - /export\s+(async\s+)?function\s+(main|preprocessor)\s*\(/.test( - result.script.code - ), - required: true - }, - { - name: 'edit_code used', - passed: result.toolsCalled.includes('edit_code'), - required: true - }, - { - name: 'get_lint_errors used', - passed: result.toolsCalled.includes('get_lint_errors'), - required: true - }, - { - name: 'test_run_script used', - passed: result.toolsCalled.includes('test_run_script'), - required: false - }, - { - name: 'judge evaluation succeeded', - passed: Boolean(result.evaluationResult?.success), - required: true - }, - { - name: `judge score >= ${minJudgeScore}`, - passed: (result.evaluationResult?.resemblanceScore ?? 0) >= minJudgeScore, - required: true - } + requiredCheck('chat run succeeded', result.success, result.error), + ...validateScriptArtifact({ + generatedScript: result.script, + expectedScript: testCase.expectedScript, + initialScript: testCase.initialScript + }), + ...buildJudgeChecks({ + evaluationResult: result.evaluationResult, + minJudgeScore + }) ] return { @@ -399,9 +386,7 @@ async function runScriptBenchmark(input: { toolCallCount: result.toolCallsCount, toolsUsed: uniqueStrings(result.toolsCalled), checks, - requiredFailedChecks: checks - .filter((check) => check.required !== false && !check.passed) - .map((check) => check.name), + requiredFailedChecks: getRequiredFailedChecks(checks), judgeScore: result.evaluationResult?.resemblanceScore ?? null, judgeStatement: result.evaluationResult?.statement ?? null, error: result.error ?? result.evaluationResult?.error ?? null @@ -550,9 +535,3 @@ function parsePositiveInteger(value: string | undefined, envName: string): numbe function uniqueStrings(values: string[]): string[] { return [...new Set(values)].sort((left, right) => left.localeCompare(right)) } - -function allRequiredChecksPassed( - checks: Array<{ name: string; passed: boolean; required?: boolean }> -): boolean { - return checks.every((check) => check.required === false || check.passed) -} diff --git a/ai_evals/adapters/frontend/core/evalCaseLoader.ts b/ai_evals/adapters/frontend/core/evalCaseLoader.ts index b342fc8d63..bcd0233de4 100644 --- a/ai_evals/adapters/frontend/core/evalCaseLoader.ts +++ b/ai_evals/adapters/frontend/core/evalCaseLoader.ts @@ -30,7 +30,7 @@ export interface ScriptEvalFixture { code: string; lang: ScriptLang | "bunnative"; path: string; - args?: Record; + args: Record; } export interface ScriptEvalCaseManifest { @@ -84,6 +84,6 @@ function toScriptEvalFixture(fixture: EvalScriptFixture): ScriptEvalFixture { code: fixture.code, lang: fixture.lang as ScriptLang | "bunnative", path: fixture.path, - args: fixture.args as Record | undefined + args: (fixture.args as Record | undefined) ?? {} }; } diff --git a/ai_evals/adapters/shared/validators.ts b/ai_evals/adapters/shared/validators.ts new file mode 100644 index 0000000000..f9b6c0c395 --- /dev/null +++ b/ai_evals/adapters/shared/validators.ts @@ -0,0 +1,473 @@ +import ts from "typescript"; + +export interface BenchmarkCheck { + name: string; + passed: boolean; + required?: boolean; + details?: string; +} + +interface ScriptLikeArtifact { + code: string; + lang: string; + path: string; +} + +interface FlowLikeArtifact { + value?: { + modules?: Array>; + }; + schema?: Record; +} + +interface AppLikeFiles { + frontend: Record; + backend: Record; +} + +interface AppLikeBackendRunnable { + type?: string; + name?: string; + path?: string; + inlineScript?: { + language?: string; + content?: string; + }; +} + +interface CliExpectedFileCheck { + path: string; + mustContain?: string[]; + mustNotContain?: string[]; +} + +interface CliFileArtifactResult { + path: string; + exists: boolean; + content?: string; +} + +const TS_LIKE_LANGUAGES = new Set(["bun", "deno", "nativets", "bunnative", "ts", "typescript"]); + +export function requiredCheck( + name: string, + passed: boolean, + details?: string +): BenchmarkCheck { + return { + name, + passed, + required: true, + ...(details ? { details } : {}) + }; +} + +export function optionalCheck( + name: string, + passed: boolean, + details?: string +): BenchmarkCheck { + return { + name, + passed, + required: false, + ...(details ? { details } : {}) + }; +} + +export function allRequiredChecksPassed(checks: BenchmarkCheck[]): boolean { + return checks.every((check) => check.required === false || check.passed); +} + +export function getRequiredFailedChecks(checks: BenchmarkCheck[]): string[] { + return checks + .filter((check) => check.required !== false && !check.passed) + .map((check) => check.name); +} + +export function buildJudgeChecks(input: { + evaluationResult: + | { + success: boolean; + resemblanceScore: number; + error?: string; + } + | undefined; + minJudgeScore: number; +}): BenchmarkCheck[] { + return [ + requiredCheck( + "judge evaluation succeeded", + Boolean(input.evaluationResult?.success), + input.evaluationResult?.error + ), + requiredCheck( + `judge score >= ${input.minJudgeScore}`, + (input.evaluationResult?.resemblanceScore ?? 0) >= input.minJudgeScore, + `score=${input.evaluationResult?.resemblanceScore ?? 0}` + ) + ]; +} + +export function validateCliArtifact(input: { + assistantOutput: string; + skillsInvoked: string[]; + expectedSkill?: string; + expectedOutputSubstrings?: string[]; + expectedFiles: CliExpectedFileCheck[]; + fileResults: CliFileArtifactResult[]; +}): BenchmarkCheck[] { + const checks: BenchmarkCheck[] = []; + + if (input.expectedSkill) { + checks.push( + requiredCheck( + `invokes ${input.expectedSkill}`, + input.skillsInvoked.includes(input.expectedSkill), + `skills invoked: ${input.skillsInvoked.join(", ")}` + ) + ); + } + + for (const expectedOutput of input.expectedOutputSubstrings ?? []) { + checks.push( + requiredCheck( + `mentions '${expectedOutput}' in assistant output`, + input.assistantOutput.includes(expectedOutput) + ) + ); + } + + for (const expectedFile of input.expectedFiles) { + const fileResult = input.fileResults.find((entry) => entry.path === expectedFile.path); + const content = fileResult?.content ?? ""; + + checks.push( + requiredCheck(`creates ${expectedFile.path}`, Boolean(fileResult?.exists)) + ); + + for (const requiredSnippet of expectedFile.mustContain ?? []) { + checks.push( + requiredCheck( + `${expectedFile.path} contains '${requiredSnippet}'`, + content.includes(requiredSnippet) + ) + ); + } + + for (const forbiddenSnippet of expectedFile.mustNotContain ?? []) { + checks.push( + requiredCheck( + `${expectedFile.path} avoids '${forbiddenSnippet}'`, + !content.includes(forbiddenSnippet) + ) + ); + } + } + + return checks; +} + +export function validateScriptArtifact(input: { + generatedScript: ScriptLikeArtifact; + expectedScript: ScriptLikeArtifact; + initialScript?: ScriptLikeArtifact; +}): BenchmarkCheck[] { + const lintErrors = getScriptLintErrors(input.generatedScript.code, input.generatedScript.lang); + const normalizedGenerated = normalizeText(input.generatedScript.code); + const normalizedInitial = input.initialScript + ? normalizeText(input.initialScript.code) + : null; + + return [ + requiredCheck( + "script path matches expected", + input.generatedScript.path === input.expectedScript.path, + `expected ${input.expectedScript.path}, got ${input.generatedScript.path}` + ), + requiredCheck( + "script language matches expected", + input.generatedScript.lang === input.expectedScript.lang, + `expected ${input.expectedScript.lang}, got ${input.generatedScript.lang}` + ), + requiredCheck( + "script exports entrypoint", + hasSupportedEntrypoint(input.generatedScript.code) + ), + requiredCheck( + "script has no syntax errors", + lintErrors.length === 0, + lintErrors.join(" | ") + ), + ...(normalizedInitial === null + ? [] + : [ + requiredCheck( + "script differs from initial input", + normalizedGenerated !== normalizedInitial + ) + ]) + ]; +} + +export function validateFlowArtifact(input: { + generatedFlow: FlowLikeArtifact; + expectedFlow: FlowLikeArtifact; +}): BenchmarkCheck[] { + const generatedModules = getFlowModules(input.generatedFlow); + const expectedModules = getFlowModules(input.expectedFlow); + const generatedTypes = collectFlowModuleTypes(generatedModules); + const expectedTypes = collectFlowModuleTypes(expectedModules); + const missingTypes = [...expectedTypes].filter((type) => !generatedTypes.has(type)); + const generatedTopLevelIds = getTopLevelFlowModuleIds(input.generatedFlow); + const expectedTopLevelIds = getTopLevelFlowModuleIds(input.expectedFlow); + const missingTopLevelIds = expectedTopLevelIds.filter( + (id) => !generatedTopLevelIds.includes(id) + ); + const expectedSchemaType = getSchemaRootType(input.expectedFlow.schema); + const generatedSchemaType = getSchemaRootType(input.generatedFlow.schema); + + return [ + requiredCheck("flow has modules", generatedModules.length > 0), + requiredCheck( + "flow includes expected module types", + missingTypes.length === 0, + missingTypes.length > 0 ? `missing types: ${missingTypes.join(", ")}` : undefined + ), + ...(expectedSchemaType + ? [ + requiredCheck( + "flow schema root type matches expected", + generatedSchemaType === expectedSchemaType, + `expected ${expectedSchemaType}, got ${generatedSchemaType ?? "(missing)"}` + ) + ] + : []), + optionalCheck( + "flow includes expected top-level step ids", + missingTopLevelIds.length === 0, + missingTopLevelIds.length > 0 + ? `missing ids: ${missingTopLevelIds.join(", ")}` + : undefined + ) + ]; +} + +export function validateAppArtifact(input: { + generatedApp: AppLikeFiles; + initialApp?: AppLikeFiles; +}): BenchmarkCheck[] { + const frontendEntries = Object.entries(input.generatedApp.frontend ?? {}); + const emptyFrontendFiles = frontendEntries + .filter(([, content]) => normalizeText(content).length === 0) + .map(([path]) => path); + const backendReferenceKeys = collectBackendReferences( + frontendEntries.map(([, content]) => content) + ); + const missingBackendReferences = backendReferenceKeys.filter( + (key) => input.generatedApp.backend[key] === undefined + ); + const invalidInlineRunnables = Object.entries(input.generatedApp.backend ?? {}) + .filter(([, runnable]) => runnable.type === "inline") + .filter(([, runnable]) => !hasSupportedEntrypoint(runnable.inlineScript?.content ?? "")) + .map(([key]) => key); + const hasChangedFromInitial = input.initialApp + ? !appFilesEqual(input.generatedApp, input.initialApp) + : true; + + return [ + requiredCheck("app has frontend files", frontendEntries.length > 0), + requiredCheck( + "app has frontend entrypoint", + Object.keys(input.generatedApp.frontend ?? {}).some( + (filePath) => filePath === "/index.tsx" || filePath === "/index.jsx" + ) + ), + requiredCheck( + "frontend files are non-empty", + emptyFrontendFiles.length === 0, + emptyFrontendFiles.length > 0 + ? `empty files: ${emptyFrontendFiles.join(", ")}` + : undefined + ), + requiredCheck( + "frontend backend references resolve", + missingBackendReferences.length === 0, + missingBackendReferences.length > 0 + ? `missing runnables: ${missingBackendReferences.join(", ")}` + : undefined + ), + requiredCheck( + "inline backend runnables export entrypoint", + invalidInlineRunnables.length === 0, + invalidInlineRunnables.length > 0 + ? `invalid inline runnables: ${invalidInlineRunnables.join(", ")}` + : undefined + ), + ...(input.initialApp + ? [requiredCheck("app differs from initial input", hasChangedFromInitial)] + : []) + ]; +} + +function hasSupportedEntrypoint(code: string): boolean { + return ( + /export\s+(async\s+)?function\s+main\s*\(/.test(code) || + /export\s+(async\s+)?function\s+preprocessor\s*\(/.test(code) + ); +} + +function getScriptLintErrors(code: string, lang: string): string[] { + if (!TS_LIKE_LANGUAGES.has(lang)) { + return hasSupportedEntrypoint(code) + ? [] + : ["Script must export a main or preprocessor function."]; + } + + const output = ts.transpileModule(code, { + compilerOptions: { + target: ts.ScriptTarget.ES2022, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.Bundler, + noEmit: true, + allowJs: true, + checkJs: false, + strict: false, + skipLibCheck: true + }, + fileName: "script.ts", + reportDiagnostics: true + }); + + const diagnostics = (output.diagnostics ?? []).map((diagnostic) => + ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n") + ); + + if (!hasSupportedEntrypoint(code)) { + diagnostics.push("Script must export a main or preprocessor function."); + } + + return diagnostics; +} + +function getFlowModules(flow: FlowLikeArtifact): Array> { + const rootModules = Array.isArray(flow.value?.modules) ? flow.value.modules : []; + const collected: Array> = []; + + for (const module of rootModules) { + visitFlowModule(module, collected); + } + + return collected; +} + +function visitFlowModule( + module: Record, + collected: Array> +): void { + collected.push(module); + + const value = asRecord(module.value); + const nestedModules = Array.isArray(value?.modules) ? value.modules : []; + for (const nested of nestedModules) { + if (isRecord(nested)) { + visitFlowModule(nested, collected); + } + } + + const branches = Array.isArray(value?.branches) ? value.branches : []; + for (const branch of branches) { + const branchRecord = asRecord(branch); + const branchModules = Array.isArray(branchRecord?.modules) + ? branchRecord.modules + : []; + for (const nested of branchModules) { + if (isRecord(nested)) { + visitFlowModule(nested, collected); + } + } + } + + const defaultModules = Array.isArray(value?.default) ? value.default : []; + for (const nested of defaultModules) { + if (isRecord(nested)) { + visitFlowModule(nested, collected); + } + } +} + +function collectFlowModuleTypes( + modules: Array> +): Set { + const types = new Set(); + for (const module of modules) { + const value = asRecord(module.value); + if (typeof value?.type === "string") { + types.add(value.type); + } + } + return types; +} + +function getTopLevelFlowModuleIds(flow: FlowLikeArtifact): string[] { + const rootModules = Array.isArray(flow.value?.modules) ? flow.value.modules : []; + return rootModules + .map((module) => (isRecord(module) && typeof module.id === "string" ? module.id : null)) + .filter((id): id is string => id !== null); +} + +function getSchemaRootType(schema: Record | undefined): string | null { + return typeof schema?.type === "string" ? schema.type : null; +} + +function collectBackendReferences(frontendContents: string[]): string[] { + const references = new Set(); + const backendCallPattern = /backend\.([A-Za-z0-9_]+)\s*\(/g; + + for (const content of frontendContents) { + for (const match of content.matchAll(backendCallPattern)) { + const key = match[1]; + if (key) { + references.add(key); + } + } + } + + return [...references].sort((left, right) => left.localeCompare(right)); +} + +function appFilesEqual(left: AppLikeFiles, right: AppLikeFiles): boolean { + return stableStringify(left) === stableStringify(right); +} + +function stableStringify(value: unknown): string { + return JSON.stringify(sortJsonValue(value)); +} + +function sortJsonValue(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(sortJsonValue); + } + + if (!isRecord(value)) { + return value; + } + + return Object.fromEntries( + Object.entries(value) + .sort((left, right) => left[0].localeCompare(right[0])) + .map(([key, nestedValue]) => [key, sortJsonValue(nestedValue)]) + ); +} + +function normalizeText(value: string): string { + return value.replace(/\r\n/g, "\n").trim(); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function asRecord(value: unknown): Record | null { + return isRecord(value) ? value : null; +}