diff --git a/ai_evals/.gitignore b/ai_evals/.gitignore index 2eea525d88..9263598939 100644 --- a/ai_evals/.gitignore +++ b/ai_evals/.gitignore @@ -1 +1,2 @@ -.env \ No newline at end of file +.env +results/ diff --git a/ai_evals/README.md b/ai_evals/README.md index 1d9882ff0f..9547088810 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -1,61 +1,69 @@ # AI Evals -Repo-level benchmark suite for the Windmill CLI guidance and the frontend AI -chat surfaces. +Minimal benchmark runner for the four Windmill AI generation modes: -## What It Is For +- `cli` +- `flow` +- `script` +- `app` -The default workflow is: +Each case is just: -1. run the benchmark on the current checkout -2. save the local result JSON -3. make your change -4. run again -5. diff the two result files -6. optionally promote the good run into official history +- a `prompt` +- an optional `initial` fixture +- an optional `expected` fixture -This keeps local development focused on before/after comparisons instead of -named prompt variants. +Each attempt runs: -## Entry Point +1. the real production prompt/tool/guidance path +2. deterministic validation +3. LLM judging -Install once: +## Install ```bash cd ai_evals bun install ``` -Main commands: +Frontend runs also require frontend dependencies: + +```bash +cd frontend +bun install +``` + +## CLI + +List cases: ```bash cd ai_evals -bun run cli -- list-cases -bun run cli -- run --surface flow --runs 3 -bun run cli -- diff-results ai_evals/results/before.json ai_evals/results/after.json -bun run cli -- history --limit 10 +bun run cli -- cases +bun run cli -- cases flow ``` -## Surfaces +Run a mode: -- `cli`: benchmark the CLI skills / AGENTS / CLAUDE guidance path -- `flow`: benchmark frontend flow chat -- `app`: benchmark frontend app chat -- `script`: benchmark frontend script chat +```bash +cd ai_evals +bun run cli -- run flow +bun run cli -- run flow flow-test5-simple-modification --runs 3 +bun run cli -- run cli bun-hello-script +``` + +`run` always writes a JSON result file under `ai_evals/results/` unless you pass +`--output`. ## Layout -- `cli/`: benchmark CLI entrypoint -- `cases/`: eval manifests -- `fixtures/`: initial and expected frontend artifacts -- `history/`: tracked official benchmark history -- `results/`: local run outputs written by `run` and ignored by git +- `cases/`: one JSON file per mode +- `fixtures/`: initial and expected fixtures +- `core/`: shared case loading, validation, judging, and result writing +- `modes/`: one runner per mode ## Notes -- Frontend runs reuse the production frontend chat code through the Vitest - adapter under `ai_evals/adapters/frontend/`. -- CLI runs create an isolated workspace and write the current checkout's - guidance into it before running the benchmark prompt. -- Official history is separate from local experimentation. Use - `bun run cli -- promote-result ...` only for runs you want to preserve. +- Frontend modes reuse the production frontend chat code through the Vitest bridge. +- CLI mode creates an isolated workspace, writes the current checkout guidance into it, and benchmarks the real skills / AGENTS flow. +- Frontend progress streams live while the benchmark is running. diff --git a/ai_evals/adapters/cli/artifact-eval.ts b/ai_evals/adapters/cli/artifact-eval.ts deleted file mode 100644 index ac1a4e7bd8..0000000000 --- a/ai_evals/adapters/cli/artifact-eval.ts +++ /dev/null @@ -1,194 +0,0 @@ -import { existsSync } from "fs"; -import { mkdtemp, mkdir, readFile, rm, writeFile } from "fs/promises"; -import { tmpdir } from "os"; -import { dirname, join } from "path"; -import { writeAiGuidanceFiles } from "../../../cli/src/guidance/writer.ts"; -import { getGeneratedSkillsSource } from "./runtime"; -import { - loadEvalCases, - type CliExpectedFileCheck -} from "../shared/evalCases"; -import { - validateCliArtifact, - type BenchmarkCheck -} from "../shared/validators"; -import { - runPromptAndCapture, - type PromptRunResult, -} from "./runtime"; - -export type ExpectedFile = CliExpectedFileCheck; - -export interface CliArtifactEvalCase { - id: string; - description?: string; - prompt: string; - maxTurns?: number; - expectedSkill?: string; - expectedOutputSubstrings?: string[]; - expectedFiles: ExpectedFile[]; -} - -export type ArtifactCheck = BenchmarkCheck; - -export interface FileArtifactResult { - path: string; - exists: boolean; - content?: string; -} - -export interface CliArtifactEvalResult { - workspaceDir: string; - renderedPrompt: string; - run: PromptRunResult; - checks: ArtifactCheck[]; - expectedFiles: FileArtifactResult[]; - passed: boolean; - guidanceLabel: string; -} - -export interface CliGuidanceConfig { - label: string; - skillsSourcePath?: string; - agentsSourcePath?: string; - claudeSourcePath?: string; -} - -const CLAUDE_PROJECT_PREAMBLE = [ - "Follow the project instructions from AGENTS.md exactly.", - "Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.", - "Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.", - "Do not skip the Skill step." -].join(" "); - -export async function loadCliArtifactEvalCases(): Promise { - return loadEvalCases("cli").map((entry) => ({ - id: entry.id, - description: entry.title, - prompt: entry.userPrompt, - maxTurns: - typeof entry.workspaceContext.max_turns === "number" - ? entry.workspaceContext.max_turns - : undefined, - expectedSkill: entry.artifactChecks.expectedSkill, - expectedOutputSubstrings: entry.artifactChecks.expectedOutputSubstrings, - expectedFiles: entry.artifactChecks.expectedFiles - })); -} - -export async function runCliArtifactEvalCase( - evalCase: CliArtifactEvalCase, - options: { - guidance: CliGuidanceConfig; - } -): Promise { - const workspaceDir = await createIsolatedWorkspace(evalCase.id, options.guidance); - - try { - const renderedPrompt = await renderPrompt(evalCase.prompt, workspaceDir); - const run = await runPromptAndCapture( - renderedPrompt, - workspaceDir, - evalCase.maxTurns ?? 6 - ); - const fileResults = await collectExpectedFiles(workspaceDir, evalCase.expectedFiles); - const checks = buildChecks(evalCase, run, fileResults); - - return { - workspaceDir, - renderedPrompt, - run, - checks, - expectedFiles: fileResults, - passed: checks.every((check) => check.required === false || check.passed), - guidanceLabel: options.guidance.label - }; - } catch (error) { - if (!shouldKeepWorkspace()) { - await cleanupWorkspace(workspaceDir); - } - throw error; - } -} - -export async function cleanupWorkspace(workspaceDir: string): Promise { - await rm(workspaceDir, { recursive: true, force: true }); -} - -export function shouldKeepWorkspace(): boolean { - return process.env.WMILL_CLI_EVAL_KEEP_WORKSPACE === "1"; -} - -async function createIsolatedWorkspace( - caseId: string, - guidance: CliGuidanceConfig -): Promise { - const workspaceDir = await mkdtemp(join(tmpdir(), `wmill-cli-artifact-${caseId}-`)); - await mkdir(dirname(join(workspaceDir, ".claude", "skills")), { recursive: true }); - await writeAiGuidanceFiles({ - targetDir: workspaceDir, - nonDottedPaths: true, - overwriteProjectGuidance: true, - skillsSourcePath: guidance.skillsSourcePath ?? getGeneratedSkillsSource(), - agentsSourcePath: guidance.agentsSourcePath, - claudeSourcePath: guidance.claudeSourcePath, - }); - await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8"); - - return workspaceDir; -} - -async function renderPrompt(prompt: string, workspaceDir: string): Promise { - const renderedUserPrompt = prompt.replaceAll("{{workspace_root}}", workspaceDir); - const agentsInstructions = await readFile(join(workspaceDir, "AGENTS.md"), "utf8"); - - return [ - "# Project Instructions", - agentsInstructions.trim(), - "", - "# Benchmark Harness", - CLAUDE_PROJECT_PREAMBLE, - "", - "# User Request", - renderedUserPrompt - ].join("\n"); -} - -async function collectExpectedFiles( - workspaceDir: string, - expectedFiles: ExpectedFile[] -): Promise { - const results: FileArtifactResult[] = []; - - for (const expectedFile of expectedFiles) { - const absolutePath = join(workspaceDir, expectedFile.path); - const exists = existsSync(absolutePath); - if (!exists) { - results.push({ path: expectedFile.path, exists: false }); - continue; - } - - results.push({ - path: expectedFile.path, - exists: true, - content: await readFile(absolutePath, "utf8") - }); - } - - return results; -} - -function buildChecks( - evalCase: CliArtifactEvalCase, - run: PromptRunResult, - fileResults: FileArtifactResult[] -): ArtifactCheck[] { - return validateCliArtifact({ - assistantOutput: run.output, - skillsInvoked: run.skillsInvoked, - expectedSkill: evalCase.expectedSkill, - expectedOutputSubstrings: evalCase.expectedOutputSubstrings, - expectedFiles: evalCase.expectedFiles, - fileResults - }); -} diff --git a/ai_evals/adapters/frontend/benchmarkRunner.ts b/ai_evals/adapters/frontend/benchmarkRunner.ts index 36e7155fe5..5a8fae5215 100644 --- a/ai_evals/adapters/frontend/benchmarkRunner.ts +++ b/ai_evals/adapters/frontend/benchmarkRunner.ts @@ -1,452 +1,75 @@ -import type { AIProvider } from '$lib/gen/types.gen' -import { loadAppEvalCases, loadFlowEvalCases, loadScriptEvalCases } from './core/evalCaseLoader' -import type { VariantConfig } from './core/shared' -import { - allRequiredChecksPassed, - buildJudgeChecks, - getRequiredFailedChecks, - requiredCheck, - validateAppArtifact, - validateFlowArtifact, - validateScriptArtifact, - type BenchmarkCheck -} from '../shared/validators' +import { loadSelectedCases } from "../../core/cases"; +import { buildRunResult } from "../../core/results"; +import { runSuite } from "../../core/runSuite"; +import type { BenchmarkRunResult, ModeRunner } from "../../core/types"; +import { emitFrontendBenchmarkProgress } from "./progress"; +import { createAppModeRunner } from "../../modes/app"; +import { createFlowModeRunner } from "../../modes/flow"; +import { createScriptModeRunner } from "../../modes/script"; +import { DEFAULT_JUDGE_MODEL } from "../../core/judge"; +import { getFrontendRunModelLabel } from "../../modes/frontendCommon"; -export type FrontendBenchmarkSurface = 'flow' | 'app' | 'script' +export type FrontendBenchmarkMode = "flow" | "app" | "script"; -export interface FrontendBenchmarkConfig { - provider?: AIProvider - model?: string - systemPrompt?: { - mode: 'append' | 'replace' - content: string - } +export async function runFrontendBenchmarkFromEnv(): Promise { + const mode = parseMode(process.env.WMILL_FRONTEND_AI_EVAL_MODE); + const caseIds = parseOptionalJsonStringArray(process.env.WMILL_FRONTEND_AI_EVAL_CASE_IDS); + const runs = parsePositiveInteger(process.env.WMILL_FRONTEND_AI_EVAL_RUNS, "WMILL_FRONTEND_AI_EVAL_RUNS"); + const emitProgress = process.env.WMILL_FRONTEND_AI_EVAL_PROGRESS === "1"; + + const selectedCases = await loadSelectedCases(mode, caseIds); + const modeRunner = getModeRunner(mode); + const caseResults = await runSuite({ + modeRunner, + cases: selectedCases, + runs, + runModel: getFrontendRunModelLabel(), + judgeModel: DEFAULT_JUDGE_MODEL, + onProgress: emitProgress ? (event) => emitFrontendBenchmarkProgress(event) : undefined, + }); + + return buildRunResult({ + mode, + runs, + runModel: getFrontendRunModelLabel(), + judgeModel: DEFAULT_JUDGE_MODEL, + caseResults, + }); } -export interface FrontendBenchmarkAttempt { - attempt: number - passed: boolean - durationMs: number - assistantMessageCount: number - toolCallCount: number - toolsUsed: string[] - checks: BenchmarkCheck[] - requiredFailedChecks: string[] - judgeScore: number | null - judgeStatement: string | null - error: string | null +function getModeRunner(mode: FrontendBenchmarkMode): ModeRunner { + switch (mode) { + case "flow": + return createFlowModeRunner(); + case "app": + return createAppModeRunner(); + case "script": + return createScriptModeRunner(); + } } -export interface FrontendBenchmarkCaseResult { - caseId: string - attempts: FrontendBenchmarkAttempt[] -} - -export interface FrontendBenchmarkPayload { - surface: FrontendBenchmarkSurface - runs: number - provider: AIProvider - model: string - judgeModel: string | null - caseResults: FrontendBenchmarkCaseResult[] -} - -const DEFAULT_MIN_JUDGE_SCORE = 80 -const DEFAULT_PROVIDER: AIProvider = 'anthropic' -const DEFAULT_MODEL = 'claude-haiku-4-5-20251001' -const FRONTEND_JUDGE_MODEL = 'claude-sonnet-4-6' - -export async function runFrontendBenchmarkFromEnv(): Promise { - return runFrontendBenchmark({ - surface: parseSurface(process.env.WMILL_FRONTEND_AI_EVAL_SURFACE), - caseIds: parseOptionalJsonStringArray(process.env.WMILL_FRONTEND_AI_EVAL_CASE_IDS), - runs: parsePositiveInteger(process.env.WMILL_FRONTEND_AI_EVAL_RUNS, 'WMILL_FRONTEND_AI_EVAL_RUNS'), - config: parseConfig(process.env.WMILL_FRONTEND_AI_EVAL_CONFIG) - }) -} - -export async function runFrontendBenchmark(input: { - surface: FrontendBenchmarkSurface - caseIds: string[] - runs: number - config?: FrontendBenchmarkConfig -}): Promise { - switch (input.surface) { - case 'flow': - return await runFlowBenchmark({ - surface: 'flow', - caseIds: input.caseIds, - runs: input.runs, - config: input.config - }) - case 'app': - return await runAppBenchmark({ - surface: 'app', - caseIds: input.caseIds, - runs: input.runs, - config: input.config - }) - case 'script': - return await runScriptBenchmark({ - surface: 'script', - caseIds: input.caseIds, - runs: input.runs, - config: input.config - }) - default: - throw new Error(`Unsupported frontend benchmark surface: ${String(input.surface)}`) - } -} - -async function runFlowBenchmark(input: { - surface: 'flow' - caseIds: string[] - runs: number - config?: FrontendBenchmarkConfig -}): Promise { - const { runFlowEval } = await import('./core/flow/flowEvalRunner') - const allCases = loadFlowEvalCases() - const selectedCases = resolveCases(allCases, input.caseIds, 'frontend flow') - const resolvedConfig = resolveConfig(input.config) - - return { - surface: input.surface, - runs: input.runs, - provider: resolvedConfig.provider, - model: resolvedConfig.model, - judgeModel: FRONTEND_JUDGE_MODEL, - caseResults: await Promise.all( - selectedCases.map(async (testCase) => ({ - caseId: testCase.id, - attempts: await runRepeated(input.runs, async (attempt) => { - const startedAt = Date.now() - const result = await runFlowEval( - testCase.userPrompt, - getApiKeyForProvider(resolvedConfig.provider), - { - initialModules: testCase.initialFlow?.value?.modules, - initialSchema: testCase.initialFlow?.schema, - expectedFlow: testCase.expectedFlow as unknown as Parameters< - typeof runFlowEval - >[2]['expectedFlow'], - variant: resolvedConfig.variant, - provider: resolvedConfig.provider, - model: resolvedConfig.model - } - ) - - const minJudgeScore = testCase.minJudgeScore ?? DEFAULT_MIN_JUDGE_SCORE - const checks = [ - requiredCheck('chat run succeeded', result.success, result.error), - ...validateFlowArtifact({ - generatedFlow: { - value: { modules: result.flow.value.modules }, - schema: result.flow.schema - }, - expectedFlow: testCase.expectedFlow - }), - ...buildJudgeChecks({ - evaluationResult: result.evaluationResult, - minJudgeScore - }) - ] - - return { - attempt, - passed: allRequiredChecksPassed(checks), - durationMs: Date.now() - startedAt, - assistantMessageCount: result.iterations, - toolCallCount: result.toolCallsCount, - toolsUsed: uniqueStrings(result.toolsCalled), - checks, - requiredFailedChecks: getRequiredFailedChecks(checks), - judgeScore: result.evaluationResult?.resemblanceScore ?? null, - judgeStatement: result.evaluationResult?.statement ?? null, - error: result.error ?? result.evaluationResult?.error ?? null - } satisfies FrontendBenchmarkAttempt - }) - })) - ) - } -} - -async function runAppBenchmark(input: { - surface: 'app' - caseIds: string[] - runs: number - config?: FrontendBenchmarkConfig -}): Promise { - const { runAppEval } = await import('./core/app/appEvalRunner') - const { loadAppFixtureForEval } = await import('./core/app/appFixtureLoader') - const allCases = loadAppEvalCases() - const selectedCases = resolveCases(allCases, input.caseIds, 'frontend app') - const resolvedConfig = resolveConfig(input.config) - - return { - surface: input.surface, - runs: input.runs, - provider: resolvedConfig.provider, - model: resolvedConfig.model, - judgeModel: FRONTEND_JUDGE_MODEL, - caseResults: await Promise.all( - selectedCases.map(async (testCase) => { - const fixture = testCase.initialAppFixturePath - ? await loadAppFixtureForEval(testCase.initialAppFixturePath) - : { initialFrontend: {}, initialBackend: {} } - - return { - caseId: testCase.id, - attempts: await runRepeated(input.runs, async (attempt) => { - const startedAt = Date.now() - const result = await runAppEval( - testCase.userPrompt, - getApiKeyForProvider(resolvedConfig.provider), - { - ...fixture, - variant: resolvedConfig.variant, - provider: resolvedConfig.provider, - model: resolvedConfig.model - } - ) - - const minJudgeScore = testCase.minJudgeScore ?? DEFAULT_MIN_JUDGE_SCORE - const checks = [ - requiredCheck('chat run succeeded', result.success, result.error), - ...validateAppArtifact({ - generatedApp: result.files, - initialApp: testCase.initialAppFixturePath - ? { - frontend: fixture.initialFrontend, - backend: fixture.initialBackend - } - : undefined - }), - ...buildJudgeChecks({ - evaluationResult: result.evaluationResult, - minJudgeScore - }) - ] - - return { - attempt, - passed: allRequiredChecksPassed(checks), - durationMs: Date.now() - startedAt, - assistantMessageCount: result.iterations, - toolCallCount: result.toolCallsCount, - toolsUsed: uniqueStrings(result.toolsCalled), - checks, - requiredFailedChecks: getRequiredFailedChecks(checks), - judgeScore: result.evaluationResult?.resemblanceScore ?? null, - judgeStatement: result.evaluationResult?.statement ?? null, - error: result.error ?? result.evaluationResult?.error ?? null - } satisfies FrontendBenchmarkAttempt - }) - } - }) - ) - } -} - -async function runScriptBenchmark(input: { - surface: 'script' - caseIds: string[] - runs: number - config?: FrontendBenchmarkConfig -}): Promise { - const { runScriptEval } = await import('./core/script/scriptEvalRunner') - const allCases = loadScriptEvalCases() - const selectedCases = resolveCases(allCases, input.caseIds, 'frontend script') - const resolvedConfig = resolveConfig(input.config) - - return { - surface: input.surface, - runs: input.runs, - provider: resolvedConfig.provider, - model: resolvedConfig.model, - judgeModel: FRONTEND_JUDGE_MODEL, - caseResults: await Promise.all( - selectedCases.map(async (testCase) => ({ - caseId: testCase.id, - attempts: await runRepeated(input.runs, async (attempt) => { - const startedAt = Date.now() - const result = await runScriptEval( - testCase.userPrompt, - getApiKeyForProvider(resolvedConfig.provider), - { - initialScript: testCase.initialScript ?? testCase.expectedScript, - expectedScript: testCase.expectedScript, - variant: resolvedConfig.variant, - provider: resolvedConfig.provider, - model: resolvedConfig.model - } - ) - - const minJudgeScore = testCase.minJudgeScore ?? DEFAULT_MIN_JUDGE_SCORE - const checks = [ - requiredCheck('chat run succeeded', result.success, result.error), - ...validateScriptArtifact({ - generatedScript: result.script, - expectedScript: testCase.expectedScript, - initialScript: testCase.initialScript - }), - ...buildJudgeChecks({ - evaluationResult: result.evaluationResult, - minJudgeScore - }) - ] - - return { - attempt, - passed: allRequiredChecksPassed(checks), - durationMs: Date.now() - startedAt, - assistantMessageCount: result.iterations, - toolCallCount: result.toolCallsCount, - toolsUsed: uniqueStrings(result.toolsCalled), - checks, - requiredFailedChecks: getRequiredFailedChecks(checks), - judgeScore: result.evaluationResult?.resemblanceScore ?? null, - judgeStatement: result.evaluationResult?.statement ?? null, - error: result.error ?? result.evaluationResult?.error ?? null - } satisfies FrontendBenchmarkAttempt - }) - })) - ) - } -} - -function resolveConfig(config?: FrontendBenchmarkConfig): { - provider: AIProvider - model: string - variant: VariantConfig | undefined -} { - const provider = config?.provider ?? DEFAULT_PROVIDER - const model = config?.model ?? DEFAULT_MODEL - - if (!config?.systemPrompt) { - return { - provider, - model, - variant: undefined - } - } - - return { - provider, - model, - variant: { - name: config.systemPrompt.mode === 'replace' ? 'custom-system-prompt' : 'appended-system-prompt', - systemPrompt: - config.systemPrompt.mode === 'replace' - ? { type: 'custom', content: config.systemPrompt.content } - : { type: 'default-with-custom', custom: config.systemPrompt.content }, - tools: { type: 'default' }, - model - } - } -} - -function resolveCases( - allCases: T[], - caseIds: string[], - surfaceLabel: string -): T[] { - if (caseIds.length === 0) { - return allCases - } - - return caseIds.map((caseId) => { - const testCase = allCases.find((entry) => entry.id === caseId) - if (!testCase) { - throw new Error(`Unknown ${surfaceLabel} case: ${caseId}`) - } - return testCase - }) -} - -function parseConfig(value: string | undefined): FrontendBenchmarkConfig | undefined { - if (!value) { - return undefined - } - - const parsed = JSON.parse(value) as FrontendBenchmarkConfig - if (!parsed || typeof parsed !== 'object') { - throw new Error('WMILL_FRONTEND_AI_EVAL_CONFIG must be a JSON object') - } - - if (parsed.provider && parsed.provider !== 'anthropic' && parsed.provider !== 'openai') { - throw new Error('WMILL_FRONTEND_AI_EVAL_CONFIG.provider must be "anthropic" or "openai"') - } - - if (parsed.systemPrompt) { - const systemPrompt = parsed.systemPrompt - if ( - (systemPrompt.mode !== 'append' && systemPrompt.mode !== 'replace') || - typeof systemPrompt.content !== 'string' || - systemPrompt.content.trim().length === 0 - ) { - throw new Error( - 'WMILL_FRONTEND_AI_EVAL_CONFIG.systemPrompt must include mode "append" or "replace" and non-empty content' - ) - } - } - - return parsed +function parseMode(value: string | undefined): FrontendBenchmarkMode { + if (value === "flow" || value === "app" || value === "script") { + return value; + } + throw new Error(`Unsupported frontend benchmark mode: ${String(value)}`); } function parseOptionalJsonStringArray(value: string | undefined): string[] { - if (!value) { - return [] - } - const parsed = JSON.parse(value) as unknown - if (!Array.isArray(parsed) || parsed.some((entry) => typeof entry !== 'string')) { - throw new Error('WMILL_FRONTEND_AI_EVAL_CASE_IDS must be a JSON string array') - } - return parsed -} - -function parseSurface(value: string | undefined): FrontendBenchmarkSurface { - if (value === 'flow' || value === 'app' || value === 'script') { - return value - } - throw new Error('WMILL_FRONTEND_AI_EVAL_SURFACE must be "flow", "app", or "script"') + if (!value) { + return []; + } + const parsed = JSON.parse(value) as unknown; + if (!Array.isArray(parsed) || parsed.some((entry) => typeof entry !== "string")) { + throw new Error("WMILL_FRONTEND_AI_EVAL_CASE_IDS must be a JSON string array"); + } + return parsed; } function parsePositiveInteger(value: string | undefined, envName: string): number { - const parsed = Number.parseInt(value ?? '', 10) - if (!Number.isInteger(parsed) || parsed < 1) { - throw new Error(`${envName} must be a positive integer`) - } - return parsed -} - -function getApiKeyForProvider(provider: AIProvider): string { - if (provider === 'anthropic') { - const apiKey = process.env.ANTHROPIC_API_KEY - if (!apiKey) { - throw new Error('ANTHROPIC_API_KEY is required for frontend benchmark runs') - } - return apiKey - } - - if (provider === 'openai') { - const apiKey = process.env.OPENAI_API_KEY - if (!apiKey) { - throw new Error('OPENAI_API_KEY is required for frontend benchmark runs') - } - return apiKey - } - - throw new Error(`Unsupported frontend benchmark provider: ${provider}`) -} - -async function runRepeated(runs: number, fn: (attempt: number) => Promise): Promise { - const results: T[] = [] - for (let attempt = 1; attempt <= runs; attempt += 1) { - results.push(await fn(attempt)) - } - return results -} - -function uniqueStrings(values: string[]): string[] { - return [...new Set(values)].sort((left, right) => left.localeCompare(right)) + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${envName} must be a positive integer`); + } + return parsed; } diff --git a/ai_evals/adapters/frontend/core/app/appEvalComparison.ts b/ai_evals/adapters/frontend/core/app/appEvalComparison.ts deleted file mode 100644 index 008be3985d..0000000000 --- a/ai_evals/adapters/frontend/core/app/appEvalComparison.ts +++ /dev/null @@ -1,174 +0,0 @@ -import Anthropic from '@anthropic-ai/sdk' -import type { - AppFiles, - BackendRunnable -} from '../../../../../frontend/src/lib/components/copilot/chat/app/core' -import { BASE_EVALUATOR_RESPONSE_FORMAT } from '../shared' -import type { EvaluationResult } from '../shared' - -/** - * Expected app structure for evaluation. - */ -export interface ExpectedApp { - frontend: Record - backend: Record -} - -/** - * Initial app state for evaluation context. - */ -export interface InitialApp { - frontend: Record - backend: Record -} - -/** - * System prompt for evaluating app generation without a reference expected app. - * Evaluates based on user request fulfillment and appropriate modifications to initial state. - */ -const APP_GENERATION_EVALUATOR_SYSTEM_PROMPT = `You are an expert evaluator for Windmill Raw App definitions. Your task is to evaluate a generated app based on: -1. The original user request/prompt -2. The initial app state (if any) - this is what the app looked like before the AI made changes - -## Windmill Raw App Context -- Raw Apps consist of frontend files and backend runnables -- Frontend files are TypeScript/JavaScript files bundled with esbuild (entrypoint: index.tsx) -- Backend runnables can be: inline scripts (TypeScript/Python), workspace scripts, workspace flows, or hub scripts -- Frontend calls backend using \`await backend.(args...)\` -- Each backend runnable has a key (identifier), name (description), type, and configuration - -## Backend Runnable Types -- **inline**: Custom code with \`inlineScript.language\` and \`inlineScript.content\` -- **script**: Workspace script reference with \`path\` -- **flow**: Workspace flow reference with \`path\` -- **hubscript**: Hub script reference with \`path\` - -## Evaluation Criteria -1. **User Request Fulfillment**: Does the generated app address ALL requirements from the user's original prompt? - - Are all requested features implemented? - - Does the frontend UI match the requirements? - - Are the correct backend runnables created? -2. **Appropriate Modifications** (if initial app was provided): - - Were the changes made relevant to the user's request? - - Was existing functionality preserved where appropriate? - - Were only necessary changes made (no unnecessary removals or additions)? -3. **Frontend Structure**: Are the frontend files correctly organized and implemented? - - Is the code valid TypeScript/JavaScript? - - Are components properly structured? - - Are backend calls correctly made? -4. **Backend Structure**: Are the backend runnables correctly configured? - - Do inline scripts have proper main functions? - - Are types and paths correct for non-inline runnables? -5. **Integration**: Does the frontend correctly call the backend? - - Are the runnable keys correctly referenced? - - Are arguments passed correctly? -6. **Code Quality**: Is the code functionally correct and well-structured? - -## Important Notes -- Focus on whether the user's request was fulfilled, not on stylistic preferences -- If an initial app was provided, evaluate the appropriateness of the changes made -- For new apps (no initial state), evaluate completeness and correctness -- Extra helper functions or slightly different approaches can still score high if they accomplish the goal - -${BASE_EVALUATOR_RESPONSE_FORMAT}` - -/** - * Evaluates how well a generated app fulfills the user's request, considering any initial app state. - * Uses Anthropic API directly. - */ -export async function evaluateAppGeneration( - userPrompt: string, - generatedApp: AppFiles, - initialApp?: InitialApp -): Promise { - // @ts-ignore - const apiKey = process.env.ANTHROPIC_API_KEY - if (!apiKey) { - return { - success: false, - resemblanceScore: 0, - statement: 'No API key available for evaluation', - error: 'ANTHROPIC_API_KEY not set' - } - } - - const client = new Anthropic({ apiKey }) - - let userMessage = `## User's Original Request -${userPrompt} - -` - - if (initialApp) { - userMessage += `## Initial App State (before AI modifications) -\`\`\`json -${JSON.stringify(initialApp, null, 2)} -\`\`\` - -` - } else { - userMessage += `## Initial App State -No initial app was provided - this is a new app created from scratch. - -` - } - - userMessage += `## Generated App -\`\`\`json -${JSON.stringify(generatedApp, null, 2)} -\`\`\` - -Please evaluate how well the generated app: -1. Fulfills ALL requirements from the user's original request -2. ${initialApp ? 'Makes appropriate modifications to the initial app state' : 'Implements a complete and correct new app'}` - - try { - const response = await client.messages.create({ - model: 'claude-sonnet-4-6', - max_tokens: 2048, - system: APP_GENERATION_EVALUATOR_SYSTEM_PROMPT, - messages: [ - { role: 'user', content: userMessage } - ], - temperature: 0 - }) - - const textBlock = response.content.find((block) => block.type === 'text') - const content = textBlock?.text - if (!content) { - return { - success: false, - resemblanceScore: 0, - statement: 'No response from evaluator', - error: 'Empty response from LLM' - } - } - - // Parse JSON response - handle potential markdown code blocks - let jsonContent = content.trim() - if (jsonContent.startsWith('```')) { - jsonContent = jsonContent.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '') - } - - const parsed = JSON.parse(jsonContent) as { - resemblanceScore: number - statement: string - missingRequirements?: string[] - } - - return { - success: true, - resemblanceScore: Math.max(0, Math.min(100, Math.round(parsed.resemblanceScore))), - statement: parsed.statement, - missingRequirements: parsed.missingRequirements ?? [] - } - } catch (err) { - const errorMessage = err instanceof Error ? err.message : String(err) - return { - success: false, - resemblanceScore: 0, - statement: 'Evaluation failed', - error: errorMessage - } - } -} diff --git a/ai_evals/adapters/frontend/core/app/appEvalRunner.ts b/ai_evals/adapters/frontend/core/app/appEvalRunner.ts index f8c70ab735..ea086bf979 100644 --- a/ai_evals/adapters/frontend/core/app/appEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/app/appEvalRunner.ts @@ -11,60 +11,29 @@ import { prepareAppSystemMessage, prepareAppUserMessage } from '../../../../../frontend/src/lib/components/copilot/chat/app/core' +import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared' import { createAppFileHelpers } from './fileHelpers' -import { evaluateAppGeneration, type InitialApp } from './appEvalComparison' -import { - runEval, - resolveSystemPrompt, - resolveTools, - resolveModel, - type VariantConfig, - type BaseEvalResult, - type EvaluationResult, - type Tool, - type VariantDefaults -} from '../shared' +import { runEval } from '../shared' import type { AIProvider } from '$lib/gen/types.gen' -// Re-export for convenience -export type { InitialApp } from './appEvalComparison' - -/** - * App-specific evaluation result. - */ -export interface AppEvalResult extends BaseEvalResult { - /** Alias for output to maintain API compatibility */ +export interface AppEvalResult { + success: boolean files: AppFiles + error?: string + assistantMessageCount: number + toolCallCount: number + toolsUsed: string[] } -/** - * Options for running an app evaluation. - */ export interface AppEvalOptions { initialFrontend?: Record initialBackend?: Record model?: string - customSystemPrompt?: string maxIterations?: number - variant?: VariantConfig - /** Whether to evaluate the generated app with LLM. Default: true. Set to false to skip evaluation. */ - evaluateWithLLM?: boolean - /** AI provider (inferred from model name if omitted) */ provider?: AIProvider workspaceRoot?: string } -/** - * App-specific variant defaults. - */ -const appDefaults: VariantDefaults = { - prepareSystemMessage: prepareAppSystemMessage, - tools: getAppTools() as Tool[] -} - -/** - * Runs an app chat evaluation using the shared chat loop (same code path as production). - */ export async function runAppEval( userPrompt: string, apiKey: string, @@ -80,14 +49,9 @@ export async function runAppEval( ) try { - const variantName = options?.variant?.name ?? 'baseline' - const systemMessage = resolveSystemPrompt( - options?.variant, - appDefaults, - options?.customSystemPrompt - ) - const { tools } = resolveTools(options?.variant, appDefaults) - const model = resolveModel(options?.variant, options?.model) + const systemMessage = prepareAppSystemMessage() + const tools = getAppTools() as ProductionTool[] + const model = options?.model ?? 'claude-haiku-4-5-20251001' const userMessage = prepareAppUserMessage(userPrompt, helpers.getSelectedContext()) const rawResult = await runEval({ @@ -106,24 +70,13 @@ export async function runAppEval( } }) - let evaluationResult: EvaluationResult | undefined - if (options?.evaluateWithLLM !== false) { - const generatedApp = getFiles() - const initialApp: InitialApp | undefined = - options?.initialFrontend || options?.initialBackend - ? { - frontend: options.initialFrontend ?? {}, - backend: options.initialBackend ?? {} - } - : undefined - evaluationResult = await evaluateAppGeneration(userPrompt, generatedApp, initialApp) - } - return { - ...rawResult, - variantName, files: rawResult.output, - evaluationResult + success: rawResult.success, + error: rawResult.error, + assistantMessageCount: rawResult.iterations, + toolCallCount: rawResult.toolCallsCount, + toolsUsed: rawResult.toolsCalled } } finally { await cleanup() diff --git a/ai_evals/adapters/frontend/core/evalCaseLoader.ts b/ai_evals/adapters/frontend/core/evalCaseLoader.ts deleted file mode 100644 index bcd0233de4..0000000000 --- a/ai_evals/adapters/frontend/core/evalCaseLoader.ts +++ /dev/null @@ -1,89 +0,0 @@ -import type { ScriptLang } from "$lib/gen/types.gen"; -import { - loadEvalCases, - type EvalScriptFixture -} from "../../shared/evalCases"; - -export interface FlowEvalCaseManifest { - id: string; - title: string; - userPrompt: string; - minJudgeScore?: number; -} - -export interface FlowEvalCase extends FlowEvalCaseManifest { - expectedFlow: Record; - initialFlow?: Record; -} - -export interface AppEvalCaseManifest { - id: string; - title: string; - userPrompt: string; - initialAppFixturePath?: string; - minJudgeScore?: number; -} - -export interface AppEvalCase extends AppEvalCaseManifest {} - -export interface ScriptEvalFixture { - code: string; - lang: ScriptLang | "bunnative"; - path: string; - args: Record; -} - -export interface ScriptEvalCaseManifest { - id: string; - title: string; - userPrompt: string; - minJudgeScore?: number; -} - -export interface ScriptEvalCase extends ScriptEvalCaseManifest { - expectedScript: ScriptEvalFixture; - initialScript?: ScriptEvalFixture; -} - -export function loadFlowEvalCases(): FlowEvalCase[] { - return loadEvalCases("frontend-flow").map((testCase) => ({ - id: testCase.id, - title: testCase.title, - userPrompt: testCase.userPrompt, - minJudgeScore: testCase.judgeRubric.minScore, - expectedFlow: testCase.artifactChecks.expectedFlow, - initialFlow: testCase.initialState.initialFlow as Record | undefined - })); -} - -export function loadAppEvalCases(): AppEvalCase[] { - return loadEvalCases("frontend-app").map((testCase) => ({ - id: testCase.id, - title: testCase.title, - userPrompt: testCase.userPrompt, - initialAppFixturePath: testCase.initialState.initialAppFixturePath, - minJudgeScore: testCase.judgeRubric.minScore - })); -} - -export function loadScriptEvalCases(): ScriptEvalCase[] { - return loadEvalCases("frontend-script").map((testCase) => ({ - id: testCase.id, - title: testCase.title, - userPrompt: testCase.userPrompt, - minJudgeScore: testCase.judgeRubric.minScore, - expectedScript: toScriptEvalFixture(testCase.artifactChecks.expectedScript), - initialScript: testCase.initialState.initialScript - ? toScriptEvalFixture(testCase.initialState.initialScript) - : undefined - })); -} - -function toScriptEvalFixture(fixture: EvalScriptFixture): ScriptEvalFixture { - return { - code: fixture.code, - lang: fixture.lang as ScriptLang | "bunnative", - path: fixture.path, - args: (fixture.args as Record | undefined) ?? {} - }; -} diff --git a/ai_evals/adapters/frontend/core/flow/flowEvalComparison.ts b/ai_evals/adapters/frontend/core/flow/flowEvalComparison.ts deleted file mode 100644 index 4c2b41d577..0000000000 --- a/ai_evals/adapters/frontend/core/flow/flowEvalComparison.ts +++ /dev/null @@ -1,68 +0,0 @@ -import type { FlowModule } from '$lib/gen' -import { evaluateWithLLM, BASE_EVALUATOR_RESPONSE_FORMAT } from '../shared' -import type { EvaluationResult } from '../shared' - -/** - * Expected flow structure for evaluation. - */ -export interface ExpectedFlow { - summary?: string - value: { - modules: FlowModule[] - } - schema?: Record -} - -/** - * Flow-specific evaluator system prompt. - */ -const FLOW_EVALUATOR_SYSTEM_PROMPT = `You are an expert evaluator for Windmill flow definitions. Your task is to evaluate a generated flow against: -1. The original user request/prompt -2. An expected reference flow - -## Windmill Flow Context -- Flows consist of modules (steps) that execute sequentially -- Module types include: rawscript, forloopflow, branchone, branchall, script, flow, aiagent -- Each module has an id, value (containing type and config), and may have input_transforms -- input_transforms connect modules using expressions like "results.previous_step". Valid input_transforms are: static, javascript. Valid variables in javascript expressions are: results, flow_input, flow_input.iter.value (for forloopflow), flow_input.iter.index (for forloopflow). -- forloopflow contains nested modules that execute per iteration with access to flow_input.iter.value -- branchone executes first matching branch, branchall executes all matching branches -- Branches have conditional expressions (expr) that determine execution -- aiagent modules contain tools array with tool definitions - -## Evaluation Criteria -1. **User Request Fulfillment**: Does the generated flow address ALL requirements from the user's original prompt? - - Are all requested steps present? - - Are the requested features implemented (loops, branches, specific logic)? - - Does the schema match what the user requested for inputs? -2. **Structure**: Are the module types and nesting structure appropriate for the task? -3. **Logic**: Does the flow accomplish the intended logical task? -4. **Connections**: Are input_transforms connecting data correctly between steps? -5. **Completeness**: Are all required steps present with no major omissions? -6. **Code Quality**: Is the code functionally correct (exact syntax doesn't need to match)? - -## Important Notes -- Minor differences in variable names, code formatting, or exact wording are acceptable -- Focus on functional equivalence, not character-by-character matching -- The generated flow should achieve the same outcome as described in the user request -- Extra helper steps or slightly different approaches can still score high if they accomplish the goal -- If the user requested specific module types (like aiagent), verify they are used correctly - -${BASE_EVALUATOR_RESPONSE_FORMAT}` - -/** - * Evaluates how well a generated flow matches an expected flow and user request using an LLM. - * Returns a resemblance score (0-100), a qualitative statement, and any missing requirements. - */ -export async function evaluateFlowComparison( - generatedFlow: ExpectedFlow, - expectedFlow: ExpectedFlow, - userPrompt: string -): Promise { - return evaluateWithLLM({ - userPrompt, - generatedOutput: generatedFlow, - expectedOutput: expectedFlow, - evaluatorSystemPrompt: FLOW_EVALUATOR_SYSTEM_PROMPT - }) -} diff --git a/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts b/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts index 312410bfc4..272308b0b3 100644 --- a/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts @@ -10,58 +10,34 @@ import { prepareFlowUserMessage, type FlowAIChatHelpers } from '../../../../../frontend/src/lib/components/copilot/chat/flow/core' +import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared' import { createFlowFileHelpers } from './fileHelpers' -import { evaluateFlowComparison, type ExpectedFlow } from './flowEvalComparison' -import { - runEval, - resolveSystemPrompt, - resolveTools, - resolveModel, - type VariantConfig, - type BaseEvalResult, - type EvaluationResult, - type Tool, - type VariantDefaults -} from '../shared' +import { runEval } from '../shared' -// Re-export for convenience -export type { ExpectedFlow } from './flowEvalComparison' - -/** - * Flow-specific evaluation result. - */ -export interface FlowEvalResult extends BaseEvalResult { - /** Alias for output to maintain API compatibility */ - flow: ExtendedOpenFlow +export interface FlowFixture { + value?: { + modules?: FlowModule[] + } + schema?: Record +} + +export interface FlowEvalResult { + success: boolean + flow: ExtendedOpenFlow + error?: string + assistantMessageCount: number + toolCallCount: number + toolsUsed: string[] } -/** - * Options for running a flow evaluation. - */ export interface FlowEvalOptions { - initialModules?: FlowModule[] - initialSchema?: Record + initialFlow?: FlowFixture model?: string - customSystemPrompt?: string maxIterations?: number - variant?: VariantConfig - expectedFlow?: ExpectedFlow - /** AI provider (inferred from model name if omitted) */ provider?: AIProvider workspaceRoot?: string } -/** - * Flow-specific variant defaults. - */ -const flowDefaults: VariantDefaults = { - prepareSystemMessage: prepareFlowSystemMessage, - tools: flowTools as Tool[] -} - -/** - * Runs a flow chat evaluation using the shared chat loop (same code path as production). - */ export async function runFlowEval( userPrompt: string, apiKey: string, @@ -71,20 +47,15 @@ export async function runFlowEval( options?.workspaceRoot ?? (await mkdtemp(join(tmpdir(), 'wmill-frontend-flow-benchmark-'))) const { helpers, getFlow, cleanup } = await createFlowFileHelpers( - options?.initialModules ?? [], - options?.initialSchema, + options?.initialFlow?.value?.modules ?? [], + options?.initialFlow?.schema, workspaceRoot ) try { - const variantName = options?.variant?.name ?? 'baseline' - const systemMessage = resolveSystemPrompt( - options?.variant, - flowDefaults, - options?.customSystemPrompt - ) - const { tools } = resolveTools(options?.variant, flowDefaults) - const model = resolveModel(options?.variant, options?.model) + const systemMessage = prepareFlowSystemMessage() + const tools = flowTools as ProductionTool[] + const model = options?.model ?? 'claude-haiku-4-5-20251001' const userMessage = prepareFlowUserMessage(userPrompt, helpers.getFlowAndSelectedId(), []) const rawResult = await runEval({ @@ -103,25 +74,13 @@ export async function runFlowEval( } }) - let evaluationResult: EvaluationResult | undefined - if (options?.expectedFlow) { - const generatedFlow = getFlow() - evaluationResult = await evaluateFlowComparison( - { - summary: generatedFlow.summary, - value: { modules: generatedFlow.value.modules }, - schema: generatedFlow.schema - }, - options.expectedFlow, - userPrompt - ) - } - return { - ...rawResult, - variantName, flow: rawResult.output, - evaluationResult + success: rawResult.success, + error: rawResult.error, + assistantMessageCount: rawResult.iterations, + toolCallCount: rawResult.toolCallsCount, + toolsUsed: rawResult.toolsCalled } } finally { await cleanup() diff --git a/ai_evals/adapters/frontend/core/script/scriptEvalComparison.ts b/ai_evals/adapters/frontend/core/script/scriptEvalComparison.ts deleted file mode 100644 index 3481bcaead..0000000000 --- a/ai_evals/adapters/frontend/core/script/scriptEvalComparison.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { evaluateWithLLM, BASE_EVALUATOR_RESPONSE_FORMAT } from '../shared' -import type { EvaluationResult } from '../shared' -import type { ScriptEvalState } from './fileHelpers' - -const SCRIPT_EVALUATOR_SYSTEM_PROMPT = `You are an expert evaluator for Windmill script generation. - -Your task is to evaluate a generated script against: -1. The original user request/prompt -2. An expected reference script - -## Windmill Script Context -- Scripts should usually export a \`main\` function unless the request clearly requires a \`preprocessor\` -- The generated output may contain extra imports, helper functions, or comments -- Minor differences in formatting, variable names, or extra safe guards are acceptable -- Focus on whether the code fulfills the requested behavior and has the correct overall structure - -## Evaluation Criteria -1. **User Request Fulfillment**: Does the generated script address the requested behavior? -2. **Entrypoint Correctness**: Does it export the right main entrypoint and shape? -3. **Functional Equivalence**: Does it implement the same logic as the expected script, even if syntax differs? -4. **Windmill Fit**: Is the script appropriate for a Windmill script context? - -${BASE_EVALUATOR_RESPONSE_FORMAT}` - -export async function evaluateScriptComparison( - generatedScript: ScriptEvalState, - expectedScript: ScriptEvalState, - userPrompt: string -): Promise { - return evaluateWithLLM({ - userPrompt, - generatedOutput: generatedScript, - expectedOutput: expectedScript, - evaluatorSystemPrompt: SCRIPT_EVALUATOR_SYSTEM_PROMPT - }) -} diff --git a/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts b/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts index b958cb5640..ae1cccbb97 100644 --- a/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts @@ -9,31 +9,23 @@ import { prepareScriptUserMessage, type ScriptChatHelpers } from '../../../../../frontend/src/lib/components/copilot/chat/script/core' +import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared' import { createScriptFileHelpers, type ScriptEvalState } from './fileHelpers' -import { evaluateScriptComparison } from './scriptEvalComparison' -import { - runEval, - resolveSystemPrompt, - resolveTools, - resolveModel, - type VariantConfig, - type BaseEvalResult, - type EvaluationResult, - type Tool, - type VariantDefaults -} from '../shared' +import { runEval } from '../shared' -export interface ScriptEvalResult extends BaseEvalResult { +export interface ScriptEvalResult { + success: boolean script: ScriptEvalState + error?: string + assistantMessageCount: number + toolCallCount: number + toolsUsed: string[] } export interface ScriptEvalOptions { initialScript: ScriptEvalState model?: string - customSystemPrompt?: string maxIterations?: number - variant?: VariantConfig - expectedScript?: ScriptEvalState provider?: AIProvider workspaceRoot?: string } @@ -64,31 +56,19 @@ export async function runScriptEval( ) try { - const variantName = options.variant?.name ?? 'baseline' - const model = resolveModel(options.variant, options.model) + const model = options.model ?? 'claude-haiku-4-5-20251001' const modelProvider = resolveModelProvider(model, options.provider) const selectedContext: ContextElement[] = [] - const scriptDefaults: VariantDefaults = { - prepareSystemMessage: (customPrompt?: string) => - prepareScriptSystemMessage( - modelProvider, - options.initialScript.lang, - {}, - customPrompt ?? options.customSystemPrompt - ), - tools: prepareScriptTools( - modelProvider, - options.initialScript.lang, - selectedContext - ) as Tool[] - } - - const systemMessage = resolveSystemPrompt( - options.variant, - scriptDefaults, - options.customSystemPrompt + const systemMessage = prepareScriptSystemMessage( + modelProvider, + options.initialScript.lang, + {} ) - const { tools } = resolveTools(options.variant, scriptDefaults) + const tools = prepareScriptTools( + modelProvider, + options.initialScript.lang, + selectedContext + ) as ProductionTool[] const userMessage = prepareScriptUserMessage(userPrompt, selectedContext) const rawResult = await runEval({ @@ -107,20 +87,13 @@ export async function runScriptEval( } }) - let evaluationResult: EvaluationResult | undefined - if (options.expectedScript) { - evaluationResult = await evaluateScriptComparison( - getScript(), - options.expectedScript, - userPrompt - ) - } - return { - ...rawResult, - variantName, script: rawResult.output, - evaluationResult + success: rawResult.success, + error: rawResult.error, + assistantMessageCount: rawResult.iterations, + toolCallCount: rawResult.toolCallsCount, + toolsUsed: rawResult.toolsCalled } } finally { await cleanup() diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 67c2351faa..55c3941483 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -5,29 +5,13 @@ import type { ChatCompletionSystemMessageParam } from 'openai/resources/chat/completions.mjs' import type { AIProvider, AIProviderModel } from '$lib/gen/types.gen' -import type { TokenUsage, ToolCallDetail, EvalRunnerOptions } from './types' -import type { Tool } from './baseVariants' +import type { TokenUsage, ToolCallDetail, EvalRunnerOptions, RawEvalResult } from './types' import { runChatLoop, type ChatClients } from '../../../../../frontend/src/lib/components/copilot/chat/chatLoop' import type { Tool as ProductionTool, ToolCallbacks } from '../../../../../frontend/src/lib/components/copilot/chat/shared' -/** - * Result from a single eval run (before domain-specific evaluation). - */ -export interface RawEvalResult { - success: boolean - output: TOutput - error?: string - tokenUsage: TokenUsage - toolCallsCount: number - toolsCalled: string[] - toolCallDetails: ToolCallDetail[] - iterations: number - messages: ChatCompletionMessageParam[] -} - /** * Parameters for running a base evaluation. */ @@ -41,7 +25,7 @@ export interface RunEvalParams { /** Tool definitions for the LLM API (unused — derived from tools) */ toolDefs?: unknown /** Full tool implementations for execution */ - tools: Tool[] + tools: ProductionTool[] /** Domain-specific helpers for tool execution */ helpers: THelpers /** API key for the provider */ @@ -60,12 +44,12 @@ function createEvalClients(provider: AIProvider, apiKey: string): ChatClients { return { openai: new OpenAI({ apiKey: 'unused' }), anthropic: new Anthropic({ apiKey }) - } + } as ChatClients } return { openai: new OpenAI({ apiKey }), anthropic: new Anthropic({ apiKey: 'unused' }) - } + } as ChatClients } /** @@ -131,7 +115,7 @@ export async function runEval( } return tool.fn(p) } - })) as ProductionTool[] + })) // No-op callbacks for eval const callbacks: ToolCallbacks & { diff --git a/ai_evals/adapters/frontend/core/shared/baseLLMEvaluator.ts b/ai_evals/adapters/frontend/core/shared/baseLLMEvaluator.ts deleted file mode 100644 index 5f6e513a7e..0000000000 --- a/ai_evals/adapters/frontend/core/shared/baseLLMEvaluator.ts +++ /dev/null @@ -1,135 +0,0 @@ -import Anthropic from '@anthropic-ai/sdk' -import type { EvaluationResult } from './types' - -/** - * Parameters for LLM-based evaluation. - */ -export interface EvaluateParams { - /** The user's original request/prompt */ - userPrompt: string - /** The generated output to evaluate */ - generatedOutput: unknown - /** The expected/reference output */ - expectedOutput: unknown - /** Domain-specific system prompt for the evaluator */ - evaluatorSystemPrompt: string - /** Anthropic API key for evaluation */ - apiKey?: string - /** Model to use for evaluation (default: 'claude-sonnet-4-5-20250514') */ - model?: string -} - -/** - * Base evaluator system prompt template. - * Domain-specific evaluators should build on this structure. - */ -export const BASE_EVALUATOR_RESPONSE_FORMAT = ` -## Response Format -You MUST respond with valid JSON only, no additional text: -{ - "resemblanceScore": <0-100 integer>, - "statement": "", - "missingRequirements": [""] -} - -Score guidelines: -- 90-100: Fully addresses user request, functionally equivalent to expected output -- 70-89: Addresses most user requirements, same overall structure with minor differences -- 50-69: Partially addresses user request, achieves similar goal but different approach -- 30-49: Missing significant requirements from user request -- 0-29: Does not address user request or significantly incorrect` - -/** - * Evaluates how well a generated output matches an expected output using an LLM. - * Uses Anthropic API directly instead of OpenRouter. - */ -export async function evaluateWithLLM(params: EvaluateParams): Promise { - const { - userPrompt, - generatedOutput, - expectedOutput, - evaluatorSystemPrompt, - apiKey, - model = 'claude-sonnet-4-6' - } = params - - // @ts-ignore - process.env - const anthropicKey = apiKey ?? process.env.ANTHROPIC_API_KEY - if (!anthropicKey) { - return { - success: false, - resemblanceScore: 0, - statement: 'No API key available for evaluation', - error: 'ANTHROPIC_API_KEY not set and no apiKey provided' - } - } - - const client = new Anthropic({ apiKey: anthropicKey }) - - const userMessage = `## User's Original Request -${userPrompt} - -## Expected Reference Output -\`\`\`json -${JSON.stringify(expectedOutput, null, 2)} -\`\`\` - -## Generated Output -\`\`\`json -${JSON.stringify(generatedOutput, null, 2)} -\`\`\` - -Please evaluate how well the generated output: -1. Fulfills ALL requirements from the user's original request -2. Matches the structure and logic of the expected reference output` - - try { - const response = await client.messages.create({ - model, - max_tokens: 2048, - system: evaluatorSystemPrompt, - messages: [ - { role: 'user', content: userMessage } - ], - temperature: 0 - }) - - const textBlock = response.content.find((block) => block.type === 'text') - const content = textBlock?.text - if (!content) { - return { - success: false, - resemblanceScore: 0, - statement: 'No response from evaluator', - error: 'Empty response from LLM' - } - } - - // Parse JSON response - handle potential markdown code blocks - let jsonContent = content.trim() - if (jsonContent.startsWith('```')) { - jsonContent = jsonContent.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '') - } - - const parsed = JSON.parse(jsonContent) as { - resemblanceScore: number - statement: string - missingRequirements?: string[] - } - - return { - success: true, - resemblanceScore: Math.max(0, Math.min(100, Math.round(parsed.resemblanceScore))), - statement: parsed.statement, - missingRequirements: parsed.missingRequirements ?? [] - } - } catch (err) { - const errorMessage = err instanceof Error ? err.message : String(err) - return { - success: false, - resemblanceScore: 0, - statement: 'Evaluation failed', - error: errorMessage - } - } -} diff --git a/ai_evals/adapters/frontend/core/shared/baseVariants.ts b/ai_evals/adapters/frontend/core/shared/baseVariants.ts deleted file mode 100644 index 26d9bf57cc..0000000000 --- a/ai_evals/adapters/frontend/core/shared/baseVariants.ts +++ /dev/null @@ -1,108 +0,0 @@ -import type { - ChatCompletionFunctionTool, - ChatCompletionSystemMessageParam -} from 'openai/resources/chat/completions.mjs' -import type { ChatCompletionTool } from 'openai/resources/chat/completions.mjs' -import type { VariantConfig } from './types' - -/** - * Generic tool interface that matches the structure used across chat modules. - */ -export interface Tool { - def: ChatCompletionFunctionTool - fn: (params: { - args: Record - workspace: string - helpers: THelpers - toolCallbacks: { - setToolStatus: (...args: unknown[]) => void - removeToolStatus: (...args: unknown[]) => void - } - toolId: string - }) => Promise -} - -/** - * Domain-specific defaults for variant resolution. - */ -export interface VariantDefaults { - /** Function to prepare system message, optionally with custom prompt */ - prepareSystemMessage: (customPrompt?: string) => ChatCompletionSystemMessageParam - /** Available tools for the domain */ - tools: Tool[] -} - -/** - * Resolves system prompt from variant config. - * Returns the appropriate ChatCompletionSystemMessageParam based on config. - */ -export function resolveSystemPrompt( - variant: VariantConfig | undefined, - defaults: VariantDefaults, - fallbackCustomPrompt?: string -): ChatCompletionSystemMessageParam { - if (!variant?.systemPrompt || variant.systemPrompt.type === 'default') { - return defaults.prepareSystemMessage(fallbackCustomPrompt) - } - - if (variant.systemPrompt.type === 'default-with-custom') { - return defaults.prepareSystemMessage(variant.systemPrompt.custom) - } - - // type === 'custom' - return { - role: 'system', - content: variant.systemPrompt.content - } -} - -/** - * Resolves tools from variant config. - * Returns both the tool definitions (for API) and full tools (for execution). - */ -export function resolveTools( - variant: VariantConfig | undefined, - defaults: VariantDefaults -): { - toolDefs: ChatCompletionTool[] - tools: Tool[] -} { - if (!variant?.tools || variant.tools.type === 'default') { - return { - toolDefs: defaults.tools.map((t) => t.def), - tools: defaults.tools - } - } - - if (variant.tools.type === 'subset') { - const includeList = (variant.tools as { type: 'subset'; include: string[] }).include - const subset = defaults.tools.filter((t) => includeList.includes(t.def.function.name)) - return { - toolDefs: subset.map((t) => t.def), - tools: subset - } - } - - if (variant.tools.type === 'custom') { - // Custom tools are typed as unknown[] in base VariantConfig but domain-specific - // code should ensure they are the correct Tool type - const customTools = variant.tools.tools as Tool[] - return { - toolDefs: customTools.map((t) => t.def), - tools: customTools - } - } - - // Default fallback - return { - toolDefs: defaults.tools.map((t) => t.def), - tools: defaults.tools - } -} - -/** - * Resolves model from variant config with fallback. - */ -export function resolveModel(variant?: VariantConfig, fallback?: string): string { - return variant?.model ?? fallback ?? 'gpt-4o' -} diff --git a/ai_evals/adapters/frontend/core/shared/index.ts b/ai_evals/adapters/frontend/core/shared/index.ts index dc38a6390a..290abc8b0f 100644 --- a/ai_evals/adapters/frontend/core/shared/index.ts +++ b/ai_evals/adapters/frontend/core/shared/index.ts @@ -1,18 +1,3 @@ -export type { - TokenUsage, - ToolCallDetail, - EvaluationResult, - BaseEvalResult, - VariantConfig, - EvalRunnerOptions, - ToolCallbacks -} from './types' - -export type { Tool, VariantDefaults } from './baseVariants' -export { resolveSystemPrompt, resolveTools, resolveModel } from './baseVariants' - -export type { RawEvalResult, RunEvalParams } from './baseEvalRunner' +export type { TokenUsage, ToolCallDetail, EvalRunnerOptions, RawEvalResult } from './types' +export type { RunEvalParams } from './baseEvalRunner' export { runEval } from './baseEvalRunner' - -export type { EvaluateParams } from './baseLLMEvaluator' -export { evaluateWithLLM, BASE_EVALUATOR_RESPONSE_FORMAT } from './baseLLMEvaluator' diff --git a/ai_evals/adapters/frontend/core/shared/types.ts b/ai_evals/adapters/frontend/core/shared/types.ts index 61f7f1fd1f..4bc3a49b3c 100644 --- a/ai_evals/adapters/frontend/core/shared/types.ts +++ b/ai_evals/adapters/frontend/core/shared/types.ts @@ -1,39 +1,25 @@ import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs' import type { AIProvider } from '$lib/gen/types.gen' -/** - * Token usage tracking for LLM calls. - */ export interface TokenUsage { prompt: number completion: number total: number } -/** - * Details of a single tool call made during evaluation. - */ export interface ToolCallDetail { name: string arguments: Record } -/** - * Result of LLM-based comparison/evaluation. - */ -export interface EvaluationResult { - success: boolean - resemblanceScore: number - statement: string - missingRequirements?: string[] - error?: string +export interface EvalRunnerOptions { + maxIterations?: number + model?: string + workspace?: string + provider?: AIProvider } -/** - * Base evaluation result that can be extended for domain-specific outputs. - * @template TOutput The domain-specific output type (e.g., flow definition, app files) - */ -export interface BaseEvalResult { +export interface RawEvalResult { success: boolean output: TOutput error?: string @@ -42,66 +28,5 @@ export interface BaseEvalResult { toolsCalled: string[] toolCallDetails: ToolCallDetail[] iterations: number - variantName: string - evaluationResult?: EvaluationResult messages: ChatCompletionMessageParam[] } - -/** - * Base configuration for a variant in eval testing. - * Allows customizing system prompt, tools, and model for comparison. - * - * Note: Domain-specific variants may extend this with custom tool configurations. - * See flow/flowEvalVariants.ts for an example with custom tools. - */ -export interface VariantConfig { - name: string - description?: string - - /** System prompt configuration */ - systemPrompt?: - | { type: 'default' } - | { type: 'default-with-custom'; custom: string } - | { type: 'custom'; content: string } - - /** Tools configuration - basic types supported by shared code */ - tools?: - | { type: 'default' } - | { type: 'subset'; include: string[] } - | { type: 'custom'; tools: unknown[] } - - /** Model to use (default: 'gpt-4o') */ - model?: string -} - -/** - * Options for running an evaluation. - */ -export interface EvalRunnerOptions { - /** Maximum iterations for tool call loop (default: 20) */ - maxIterations?: number - /** Model to use for LLM calls */ - model?: string - /** Workspace ID for tool calls */ - workspace?: string - /** AI provider (inferred from model name if omitted) */ - provider?: AIProvider -} - -/** - * No-op tool callbacks for eval testing. - */ -export interface ToolCallbacks { - setToolStatus: (id: string, status: { content?: string; result?: string; error?: string }) => void - removeToolStatus: (id: string) => void -} - -/** - * Creates no-op tool callbacks for eval testing. - */ -export function createNoOpToolCallbacks(): ToolCallbacks { - return { - setToolStatus: () => {}, - removeToolStatus: () => {} - } -} diff --git a/ai_evals/adapters/frontend/progress.ts b/ai_evals/adapters/frontend/progress.ts new file mode 100644 index 0000000000..ddf3c3a827 --- /dev/null +++ b/ai_evals/adapters/frontend/progress.ts @@ -0,0 +1,101 @@ +export type FrontendBenchmarkProgressSurface = 'flow' | 'app' | 'script' + +export type FrontendBenchmarkProgressEvent = + | { + type: 'run-start' + surface: FrontendBenchmarkProgressSurface + totalCases: number + runs: number + concurrency: number + } + | { + type: 'attempt-start' + surface: FrontendBenchmarkProgressSurface + caseId: string + caseNumber: number + totalCases: number + attempt: number + runs: number + } + | { + type: 'attempt-finish' + surface: FrontendBenchmarkProgressSurface + caseId: string + caseNumber: number + totalCases: number + attempt: number + runs: number + passed: boolean + durationMs: number + judgeScore: number | null + error: string | null + } + +export const FRONTEND_BENCHMARK_PROGRESS_PREFIX = 'WMILL_FRONTEND_AI_EVAL_PROGRESS ' + +export function emitFrontendBenchmarkProgress(event: FrontendBenchmarkProgressEvent): void { + process.stderr.write( + `${FRONTEND_BENCHMARK_PROGRESS_PREFIX}${JSON.stringify(event)}\n` + ) +} + +export function parseFrontendBenchmarkProgressLine( + line: string +): FrontendBenchmarkProgressEvent | null { + if (!line.startsWith(FRONTEND_BENCHMARK_PROGRESS_PREFIX)) { + return null + } + + try { + const parsed = JSON.parse( + line.slice(FRONTEND_BENCHMARK_PROGRESS_PREFIX.length) + ) as FrontendBenchmarkProgressEvent + return parsed?.type ? parsed : null + } catch { + return null + } +} + +export function formatFrontendBenchmarkProgressEvent( + event: FrontendBenchmarkProgressEvent +): string { + switch (event.type) { + case 'run-start': + return `Running ${event.surface}: ${event.totalCases} cases x ${event.runs} run${event.runs === 1 ? '' : 's'}, concurrency ${event.concurrency}` + case 'attempt-start': + return `${formatCasePrefix(event.caseNumber, event.totalCases)} ${event.caseId} attempt ${event.attempt}/${event.runs}...` + case 'attempt-finish': { + const parts = [ + `${formatCasePrefix(event.caseNumber, event.totalCases)} ${event.caseId} attempt ${event.attempt}/${event.runs} ${event.passed ? 'pass' : 'fail'}`, + formatDuration(event.durationMs) + ] + if (event.judgeScore !== null) { + parts.push(`judge ${formatNumber(event.judgeScore)}`) + } + if (event.error) { + parts.push(truncateSingleLine(event.error, 120)) + } + return parts.join(' | ') + } + } +} + +function formatCasePrefix(caseNumber: number, totalCases: number): string { + return `[${caseNumber}/${totalCases}]` +} + +function formatDuration(durationMs: number): string { + return `${formatNumber(durationMs / 1000)}s` +} + +function formatNumber(value: number): string { + return Number.isInteger(value) ? String(value) : value.toFixed(1) +} + +function truncateSingleLine(value: string, maxLength: number): string { + const normalized = value.replace(/\s+/g, ' ').trim() + if (normalized.length <= maxLength) { + return normalized + } + return `${normalized.slice(0, Math.max(0, maxLength - 3))}...` +} diff --git a/ai_evals/adapters/frontend/runtime.ts b/ai_evals/adapters/frontend/runtime.ts index faf4f2f62f..386278afde 100644 --- a/ai_evals/adapters/frontend/runtime.ts +++ b/ai_evals/adapters/frontend/runtime.ts @@ -1,65 +1,30 @@ -import { execFile as execFileCallback } from 'node:child_process' +import { spawn } from 'node:child_process' import { mkdtemp, readFile, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' -import { promisify } from 'node:util' import { fileURLToPath } from 'node:url' +import { + formatFrontendBenchmarkProgressEvent, + parseFrontendBenchmarkProgressLine +} from './progress' +import type { BenchmarkRunResult } from '../../core/types' -const execFile = promisify(execFileCallback) const REPO_ROOT = fileURLToPath(new URL('../../../', import.meta.url)) const FRONTEND_DIR = path.join(REPO_ROOT, 'frontend') const FRONTEND_BENCHMARK_TEST = '../ai_evals/adapters/frontend/vitestAdapter.test.ts' -export type FrontendSurfaceName = 'frontend-flow' | 'frontend-app' | 'frontend-script' - -export interface FrontendBenchmarkConfig { - provider?: 'anthropic' | 'openai' - model?: string - systemPrompt?: { - mode: 'append' | 'replace' - content: string - } -} - -export interface FrontendAdapterAttempt { - attempt: number - passed: boolean - durationMs: number - assistantMessageCount: number - toolCallCount: number - toolsUsed: string[] - checks: Array<{ name: string; passed: boolean; required?: boolean }> - requiredFailedChecks: string[] - judgeScore: number | null - judgeStatement: string | null - error: string | null -} - -export interface FrontendAdapterCaseResult { - caseId: string - attempts: FrontendAdapterAttempt[] -} - -export interface FrontendAdapterPayload { - surface: 'flow' | 'app' | 'script' - runs: number - provider: string - model: string - judgeModel: string | null - caseResults: FrontendAdapterCaseResult[] -} +export type FrontendMode = 'flow' | 'app' | 'script' export async function runFrontendBenchmarkAdapter(input: { - surface: FrontendSurfaceName + mode: FrontendMode caseIds: string[] runs: number - config?: FrontendBenchmarkConfig -}): Promise { +}): Promise { const tempDir = await mkdtemp(path.join(tmpdir(), 'wmill-frontend-benchmark-')) const outputPath = path.join(tempDir, 'result.json') try { - await execFile( + await runVitestBenchmark( path.join(FRONTEND_DIR, 'node_modules', '.bin', 'vitest'), [ 'run', @@ -68,42 +33,124 @@ export async function runFrontendBenchmarkAdapter(input: { 'server', '--config', 'vite.config.js' - ], - { - cwd: FRONTEND_DIR, - env: { - ...process.env, - WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH: outputPath, - WMILL_FRONTEND_AI_EVAL_SURFACE: frontendSurfaceToAdapterSurface(input.surface), - WMILL_FRONTEND_AI_EVAL_CASE_IDS: JSON.stringify(input.caseIds), - WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs), - WMILL_FRONTEND_AI_EVAL_CONFIG: input.config - ? JSON.stringify(input.config) - : '' - }, - maxBuffer: 10 * 1024 * 1024 + ], + { + cwd: FRONTEND_DIR, + env: { + ...process.env, + BROWSERSLIST_IGNORE_OLD_DATA: '1', + WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH: outputPath, + WMILL_FRONTEND_AI_EVAL_MODE: input.mode, + WMILL_FRONTEND_AI_EVAL_CASE_IDS: JSON.stringify(input.caseIds), + WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs), + WMILL_FRONTEND_AI_EVAL_PROGRESS: '1' } - ) + } + ) const raw = await readFile(outputPath, 'utf8') - return JSON.parse(raw) as FrontendAdapterPayload + return JSON.parse(raw) as BenchmarkRunResult } catch (error) { - const executionError = error as Error & { stdout?: string; stderr?: string } - const details = [executionError.message, executionError.stdout, executionError.stderr] - .filter(Boolean) - .join('\n') - throw new Error(`Frontend benchmark adapter failed:\n${details}`) + throw new Error(`Frontend benchmark adapter failed:\n${toErrorMessage(error)}`) } finally { await rm(tempDir, { recursive: true, force: true }) } } -function frontendSurfaceToAdapterSurface(surface: FrontendSurfaceName): 'flow' | 'app' | 'script' { - if (surface === 'frontend-flow') { - return 'flow' +async function runVitestBenchmark( + command: string, + args: string[], + options: { + cwd: string + env: NodeJS.ProcessEnv } - if (surface === 'frontend-app') { - return 'app' - } - return 'script' +): Promise { + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env, + stdio: ['ignore', 'pipe', 'pipe'] + }) + + let stdout = '' + let stderr = '' + let stderrLineBuffer = '' + + child.stdout?.setEncoding('utf8') + child.stdout?.on('data', (chunk: string) => { + stdout += chunk + }) + + child.stderr?.setEncoding('utf8') + child.stderr?.on('data', (chunk: string) => { + stderrLineBuffer += chunk + const { remainder, passthrough } = drainProgressLines(stderrLineBuffer) + stderrLineBuffer = remainder + stderr += passthrough + }) + + await new Promise((resolve, reject) => { + child.once('error', reject) + child.once('close', (code) => { + if (stderrLineBuffer.length > 0) { + const { remainder, passthrough } = drainProgressLines(`${stderrLineBuffer}\n`) + stderrLineBuffer = remainder + stderr += passthrough + } + + if (code === 0) { + resolve() + return + } + + const details = [`vitest exited with code ${code}`, stdout, stderr].filter(Boolean).join('\n') + reject(new Error(details)) + }) + }) +} + +function drainProgressLines(buffer: string): { + remainder: string + passthrough: string +} { + let remainder = buffer + let passthrough = '' + + while (true) { + const newlineIndex = remainder.indexOf('\n') + if (newlineIndex === -1) { + return { remainder, passthrough } + } + + const line = remainder.slice(0, newlineIndex).replace(/\r$/, '') + remainder = remainder.slice(newlineIndex + 1) + + const progressEvent = parseFrontendBenchmarkProgressLine(line) + if (progressEvent) { + process.stderr.write(`${formatFrontendBenchmarkProgressEvent(progressEvent)}\n`) + continue + } + + if (shouldSuppressFrontendStderrLine(line)) { + continue + } + + passthrough += `${line}\n` + process.stderr.write(`${line}\n`) + } +} + +function shouldSuppressFrontendStderrLine(line: string): boolean { + return ( + line.startsWith('[baseline-browser-mapping] ') || + line.startsWith('Browserslist: browsers data (caniuse-lite) is ') || + line.includes('update-browserslist-db@latest') || + line.includes('update-db#readme') + ) +} + +function toErrorMessage(error: unknown): string { + if (error instanceof Error) { + return error.message + } + return String(error) } diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index c64be91e7e..c1785efd01 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -33,16 +33,16 @@ vi.mock('$lib/components/vscode', () => ({})) const benchmarkOutputPath = process.env.WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH const benchmarkIt = benchmarkOutputPath ? it : it.skip - benchmarkIt( - 'runs the frontend benchmark adapter from environment input', - async () => { - const { runFrontendBenchmarkFromEnv } = await import('./benchmarkRunner') - const payload = await runFrontendBenchmarkFromEnv() +benchmarkIt( + 'runs the frontend benchmark adapter from environment input', + async () => { + const { runFrontendBenchmarkFromEnv } = await import('./benchmarkRunner') + const payload = await runFrontendBenchmarkFromEnv() const absoluteOutputPath = resolve(benchmarkOutputPath!) await mkdir(dirname(absoluteOutputPath), { recursive: true }) await writeFile(absoluteOutputPath, JSON.stringify(payload, null, 2) + '\n', 'utf8') - expect(payload.caseResults.length).toBeGreaterThan(0) - }, - 600_000 - ) + expect(payload.cases.length).toBeGreaterThan(0) + }, + 600_000 +) diff --git a/ai_evals/adapters/shared/evalCases.ts b/ai_evals/adapters/shared/evalCases.ts deleted file mode 100644 index b66bb20a5f..0000000000 --- a/ai_evals/adapters/shared/evalCases.ts +++ /dev/null @@ -1,344 +0,0 @@ -import { readdirSync, readFileSync } from "node:fs"; -import { join, resolve, dirname } from "node:path"; -import { fileURLToPath } from "node:url"; - -export type EvalSurfaceName = - | "cli" - | "frontend-flow" - | "frontend-app" - | "frontend-script"; - -export interface EvalCaseSummary { - id: string; - surface: EvalSurfaceName; - title: string; - tags: string[]; -} - -export interface EvalJudgeRubric { - minScore?: number; -} - -export interface CliExpectedFileCheck { - path: string; - mustContain?: string[]; - mustNotContain?: string[]; -} - -interface RawCliExpectedFileCheck { - path: string; - must_contain?: string[]; - must_not_contain?: string[]; -} - -export interface EvalScriptFixture { - code: string; - lang: string; - path: string; - args?: Record; -} - -interface ResolvedEvalCaseBase { - id: string; - surface: EvalSurfaceName; - title: string; - userPrompt: string; - workspaceContext: Record; - judgeRubric: EvalJudgeRubric; - tags: string[]; -} - -export interface ResolvedCliEvalCase extends ResolvedEvalCaseBase { - surface: "cli"; - initialState: Record; - artifactChecks: { - expectedSkill?: string; - expectedOutputSubstrings: string[]; - expectedFiles: CliExpectedFileCheck[]; - }; -} - -export interface ResolvedFrontendFlowEvalCase extends ResolvedEvalCaseBase { - surface: "frontend-flow"; - initialState: { - initialFlow?: Record; - }; - artifactChecks: { - expectedFlow: Record; - }; -} - -export interface ResolvedFrontendAppEvalCase extends ResolvedEvalCaseBase { - surface: "frontend-app"; - initialState: { - initialAppFixturePath?: string; - }; - artifactChecks: Record; -} - -export interface ResolvedFrontendScriptEvalCase extends ResolvedEvalCaseBase { - surface: "frontend-script"; - initialState: { - initialScript?: EvalScriptFixture; - }; - artifactChecks: { - expectedScript: EvalScriptFixture; - }; -} - -export type ResolvedEvalCaseBySurface = { - cli: ResolvedCliEvalCase; - "frontend-flow": ResolvedFrontendFlowEvalCase; - "frontend-app": ResolvedFrontendAppEvalCase; - "frontend-script": ResolvedFrontendScriptEvalCase; -}; - -type RawJudgeRubric = { - min_score?: number; -}; - -type RawSharedEvalCase = { - id: string; - surface: EvalSurfaceName; - title: string; - user_prompt: string; - initial_state: Record; - workspace_context: Record; - artifact_checks: Record; - judge_rubric: RawJudgeRubric; - tags: string[]; -}; - -type RawCliEvalCase = RawSharedEvalCase & { - surface: "cli"; - artifact_checks: { - expected_skill?: string; - expected_output_substrings?: string[]; - expected_files?: RawCliExpectedFileCheck[]; - }; -}; - -type RawFrontendFlowEvalCase = RawSharedEvalCase & { - surface: "frontend-flow"; - initial_state: { - flow_path?: string; - }; - artifact_checks: { - expected_flow_path: string; - }; -}; - -type RawFrontendAppEvalCase = RawSharedEvalCase & { - surface: "frontend-app"; - initial_state: { - app_fixture_path?: string; - }; - artifact_checks: Record; -}; - -type RawFrontendScriptEvalCase = RawSharedEvalCase & { - surface: "frontend-script"; - initial_state: { - script_path?: string; - }; - artifact_checks: { - expected_script_path: string; - }; -}; - -type RawEvalCaseBySurface = { - cli: RawCliEvalCase; - "frontend-flow": RawFrontendFlowEvalCase; - "frontend-app": RawFrontendAppEvalCase; - "frontend-script": RawFrontendScriptEvalCase; -}; - -const REPO_ROOT = resolve( - dirname(fileURLToPath(import.meta.url)), - "../../.." -); - -export function loadEvalCaseSummaries(surface: EvalSurfaceName): EvalCaseSummary[] { - return loadRawEvalCases(surface).map((entry) => ({ - id: entry.id, - surface: entry.surface, - title: entry.title, - tags: [...(entry.tags ?? [])] - })); -} - -export function loadEvalCases( - surface: T -): Array { - return loadRawEvalCases(surface).map((entry) => - resolveEvalCase(entry) - ) as Array; -} - -function loadRawEvalCases( - surface: T -): Array { - const manifestPaths = getManifestPaths(surface); - const cases: Array = []; - - for (const manifestPath of manifestPaths) { - const parsed = JSON.parse(readFileSync(manifestPath, "utf8")) as Array< - RawEvalCaseBySurface[T] - >; - - if (!Array.isArray(parsed) || parsed.length === 0) { - throw new Error(`No eval cases found in ${manifestPath}`); - } - - for (const entry of parsed) { - if (entry.surface !== surface) { - throw new Error( - `Eval case ${entry.id} in ${manifestPath} declared surface ${entry.surface}, expected ${surface}` - ); - } - cases.push(entry); - } - } - - return cases; -} - -function resolveEvalCase( - entry: - | RawCliEvalCase - | RawFrontendFlowEvalCase - | RawFrontendAppEvalCase - | RawFrontendScriptEvalCase -): - | ResolvedCliEvalCase - | ResolvedFrontendFlowEvalCase - | ResolvedFrontendAppEvalCase - | ResolvedFrontendScriptEvalCase { - const base = { - id: entry.id, - surface: entry.surface, - title: entry.title, - userPrompt: entry.user_prompt, - workspaceContext: entry.workspace_context ?? {}, - judgeRubric: normalizeJudgeRubric(entry.judge_rubric), - tags: [...(entry.tags ?? [])] - }; - - switch (entry.surface) { - case "cli": - return { - ...base, - surface: "cli", - initialState: {}, - artifactChecks: { - expectedSkill: entry.artifact_checks.expected_skill, - expectedOutputSubstrings: - entry.artifact_checks.expected_output_substrings ?? [], - expectedFiles: (entry.artifact_checks.expected_files ?? []).map( - (file) => ({ - path: file.path, - mustContain: file.must_contain, - mustNotContain: file.must_not_contain - }) - ) - } - }; - case "frontend-flow": - return { - ...base, - surface: "frontend-flow", - initialState: { - initialFlow: entry.initial_state.flow_path - ? readRepoRelativeJson>( - entry.initial_state.flow_path - ) - : undefined - }, - artifactChecks: { - expectedFlow: readRepoRelativeJson>( - entry.artifact_checks.expected_flow_path - ) - } - }; - case "frontend-app": - return { - ...base, - surface: "frontend-app", - initialState: { - initialAppFixturePath: entry.initial_state.app_fixture_path - ? resolveRepoRelativePath(entry.initial_state.app_fixture_path) - : undefined - }, - artifactChecks: {} - }; - case "frontend-script": - return { - ...base, - surface: "frontend-script", - initialState: { - initialScript: entry.initial_state.script_path - ? readRepoRelativeJson( - entry.initial_state.script_path - ) - : undefined - }, - artifactChecks: { - expectedScript: readRepoRelativeJson( - entry.artifact_checks.expected_script_path - ) - } - }; - default: - return assertNever(entry); - } -} - -function getManifestPaths(surface: EvalSurfaceName): string[] { - if (surface === "cli") { - const cliCasesDir = join(REPO_ROOT, "ai_evals", "cases", "cli"); - return readdirSync(cliCasesDir) - .filter((entry) => entry.endsWith(".json")) - .sort((left, right) => left.localeCompare(right)) - .map((entry) => join(cliCasesDir, entry)); - } - - return [ - join( - REPO_ROOT, - "ai_evals", - "cases", - "frontend", - `${surfaceToFrontendManifestName(surface)}.json` - ) - ]; -} - -function surfaceToFrontendManifestName( - surface: Exclude -): "flow" | "app" | "script" { - if (surface === "frontend-flow") { - return "flow"; - } - if (surface === "frontend-app") { - return "app"; - } - return "script"; -} - -function normalizeJudgeRubric(value: RawJudgeRubric | undefined): EvalJudgeRubric { - return { - minScore: value?.min_score - }; -} - -function readRepoRelativeJson(relativePath: string): T { - return JSON.parse(readFileSync(resolveRepoRelativePath(relativePath), "utf8")) as T; -} - -function resolveRepoRelativePath(relativePath: string): string { - return join(REPO_ROOT, relativePath); -} - -function assertNever(value: never): never { - throw new Error(`Unexpected value: ${JSON.stringify(value)}`); -} diff --git a/ai_evals/adapters/shared/validators.ts b/ai_evals/adapters/shared/validators.ts deleted file mode 100644 index f9b6c0c395..0000000000 --- a/ai_evals/adapters/shared/validators.ts +++ /dev/null @@ -1,473 +0,0 @@ -import ts from "typescript"; - -export interface BenchmarkCheck { - name: string; - passed: boolean; - required?: boolean; - details?: string; -} - -interface ScriptLikeArtifact { - code: string; - lang: string; - path: string; -} - -interface FlowLikeArtifact { - value?: { - modules?: Array>; - }; - schema?: Record; -} - -interface AppLikeFiles { - frontend: Record; - backend: Record; -} - -interface AppLikeBackendRunnable { - type?: string; - name?: string; - path?: string; - inlineScript?: { - language?: string; - content?: string; - }; -} - -interface CliExpectedFileCheck { - path: string; - mustContain?: string[]; - mustNotContain?: string[]; -} - -interface CliFileArtifactResult { - path: string; - exists: boolean; - content?: string; -} - -const TS_LIKE_LANGUAGES = new Set(["bun", "deno", "nativets", "bunnative", "ts", "typescript"]); - -export function requiredCheck( - name: string, - passed: boolean, - details?: string -): BenchmarkCheck { - return { - name, - passed, - required: true, - ...(details ? { details } : {}) - }; -} - -export function optionalCheck( - name: string, - passed: boolean, - details?: string -): BenchmarkCheck { - return { - name, - passed, - required: false, - ...(details ? { details } : {}) - }; -} - -export function allRequiredChecksPassed(checks: BenchmarkCheck[]): boolean { - return checks.every((check) => check.required === false || check.passed); -} - -export function getRequiredFailedChecks(checks: BenchmarkCheck[]): string[] { - return checks - .filter((check) => check.required !== false && !check.passed) - .map((check) => check.name); -} - -export function buildJudgeChecks(input: { - evaluationResult: - | { - success: boolean; - resemblanceScore: number; - error?: string; - } - | undefined; - minJudgeScore: number; -}): BenchmarkCheck[] { - return [ - requiredCheck( - "judge evaluation succeeded", - Boolean(input.evaluationResult?.success), - input.evaluationResult?.error - ), - requiredCheck( - `judge score >= ${input.minJudgeScore}`, - (input.evaluationResult?.resemblanceScore ?? 0) >= input.minJudgeScore, - `score=${input.evaluationResult?.resemblanceScore ?? 0}` - ) - ]; -} - -export function validateCliArtifact(input: { - assistantOutput: string; - skillsInvoked: string[]; - expectedSkill?: string; - expectedOutputSubstrings?: string[]; - expectedFiles: CliExpectedFileCheck[]; - fileResults: CliFileArtifactResult[]; -}): BenchmarkCheck[] { - const checks: BenchmarkCheck[] = []; - - if (input.expectedSkill) { - checks.push( - requiredCheck( - `invokes ${input.expectedSkill}`, - input.skillsInvoked.includes(input.expectedSkill), - `skills invoked: ${input.skillsInvoked.join(", ")}` - ) - ); - } - - for (const expectedOutput of input.expectedOutputSubstrings ?? []) { - checks.push( - requiredCheck( - `mentions '${expectedOutput}' in assistant output`, - input.assistantOutput.includes(expectedOutput) - ) - ); - } - - for (const expectedFile of input.expectedFiles) { - const fileResult = input.fileResults.find((entry) => entry.path === expectedFile.path); - const content = fileResult?.content ?? ""; - - checks.push( - requiredCheck(`creates ${expectedFile.path}`, Boolean(fileResult?.exists)) - ); - - for (const requiredSnippet of expectedFile.mustContain ?? []) { - checks.push( - requiredCheck( - `${expectedFile.path} contains '${requiredSnippet}'`, - content.includes(requiredSnippet) - ) - ); - } - - for (const forbiddenSnippet of expectedFile.mustNotContain ?? []) { - checks.push( - requiredCheck( - `${expectedFile.path} avoids '${forbiddenSnippet}'`, - !content.includes(forbiddenSnippet) - ) - ); - } - } - - return checks; -} - -export function validateScriptArtifact(input: { - generatedScript: ScriptLikeArtifact; - expectedScript: ScriptLikeArtifact; - initialScript?: ScriptLikeArtifact; -}): BenchmarkCheck[] { - const lintErrors = getScriptLintErrors(input.generatedScript.code, input.generatedScript.lang); - const normalizedGenerated = normalizeText(input.generatedScript.code); - const normalizedInitial = input.initialScript - ? normalizeText(input.initialScript.code) - : null; - - return [ - requiredCheck( - "script path matches expected", - input.generatedScript.path === input.expectedScript.path, - `expected ${input.expectedScript.path}, got ${input.generatedScript.path}` - ), - requiredCheck( - "script language matches expected", - input.generatedScript.lang === input.expectedScript.lang, - `expected ${input.expectedScript.lang}, got ${input.generatedScript.lang}` - ), - requiredCheck( - "script exports entrypoint", - hasSupportedEntrypoint(input.generatedScript.code) - ), - requiredCheck( - "script has no syntax errors", - lintErrors.length === 0, - lintErrors.join(" | ") - ), - ...(normalizedInitial === null - ? [] - : [ - requiredCheck( - "script differs from initial input", - normalizedGenerated !== normalizedInitial - ) - ]) - ]; -} - -export function validateFlowArtifact(input: { - generatedFlow: FlowLikeArtifact; - expectedFlow: FlowLikeArtifact; -}): BenchmarkCheck[] { - const generatedModules = getFlowModules(input.generatedFlow); - const expectedModules = getFlowModules(input.expectedFlow); - const generatedTypes = collectFlowModuleTypes(generatedModules); - const expectedTypes = collectFlowModuleTypes(expectedModules); - const missingTypes = [...expectedTypes].filter((type) => !generatedTypes.has(type)); - const generatedTopLevelIds = getTopLevelFlowModuleIds(input.generatedFlow); - const expectedTopLevelIds = getTopLevelFlowModuleIds(input.expectedFlow); - const missingTopLevelIds = expectedTopLevelIds.filter( - (id) => !generatedTopLevelIds.includes(id) - ); - const expectedSchemaType = getSchemaRootType(input.expectedFlow.schema); - const generatedSchemaType = getSchemaRootType(input.generatedFlow.schema); - - return [ - requiredCheck("flow has modules", generatedModules.length > 0), - requiredCheck( - "flow includes expected module types", - missingTypes.length === 0, - missingTypes.length > 0 ? `missing types: ${missingTypes.join(", ")}` : undefined - ), - ...(expectedSchemaType - ? [ - requiredCheck( - "flow schema root type matches expected", - generatedSchemaType === expectedSchemaType, - `expected ${expectedSchemaType}, got ${generatedSchemaType ?? "(missing)"}` - ) - ] - : []), - optionalCheck( - "flow includes expected top-level step ids", - missingTopLevelIds.length === 0, - missingTopLevelIds.length > 0 - ? `missing ids: ${missingTopLevelIds.join(", ")}` - : undefined - ) - ]; -} - -export function validateAppArtifact(input: { - generatedApp: AppLikeFiles; - initialApp?: AppLikeFiles; -}): BenchmarkCheck[] { - const frontendEntries = Object.entries(input.generatedApp.frontend ?? {}); - const emptyFrontendFiles = frontendEntries - .filter(([, content]) => normalizeText(content).length === 0) - .map(([path]) => path); - const backendReferenceKeys = collectBackendReferences( - frontendEntries.map(([, content]) => content) - ); - const missingBackendReferences = backendReferenceKeys.filter( - (key) => input.generatedApp.backend[key] === undefined - ); - const invalidInlineRunnables = Object.entries(input.generatedApp.backend ?? {}) - .filter(([, runnable]) => runnable.type === "inline") - .filter(([, runnable]) => !hasSupportedEntrypoint(runnable.inlineScript?.content ?? "")) - .map(([key]) => key); - const hasChangedFromInitial = input.initialApp - ? !appFilesEqual(input.generatedApp, input.initialApp) - : true; - - return [ - requiredCheck("app has frontend files", frontendEntries.length > 0), - requiredCheck( - "app has frontend entrypoint", - Object.keys(input.generatedApp.frontend ?? {}).some( - (filePath) => filePath === "/index.tsx" || filePath === "/index.jsx" - ) - ), - requiredCheck( - "frontend files are non-empty", - emptyFrontendFiles.length === 0, - emptyFrontendFiles.length > 0 - ? `empty files: ${emptyFrontendFiles.join(", ")}` - : undefined - ), - requiredCheck( - "frontend backend references resolve", - missingBackendReferences.length === 0, - missingBackendReferences.length > 0 - ? `missing runnables: ${missingBackendReferences.join(", ")}` - : undefined - ), - requiredCheck( - "inline backend runnables export entrypoint", - invalidInlineRunnables.length === 0, - invalidInlineRunnables.length > 0 - ? `invalid inline runnables: ${invalidInlineRunnables.join(", ")}` - : undefined - ), - ...(input.initialApp - ? [requiredCheck("app differs from initial input", hasChangedFromInitial)] - : []) - ]; -} - -function hasSupportedEntrypoint(code: string): boolean { - return ( - /export\s+(async\s+)?function\s+main\s*\(/.test(code) || - /export\s+(async\s+)?function\s+preprocessor\s*\(/.test(code) - ); -} - -function getScriptLintErrors(code: string, lang: string): string[] { - if (!TS_LIKE_LANGUAGES.has(lang)) { - return hasSupportedEntrypoint(code) - ? [] - : ["Script must export a main or preprocessor function."]; - } - - const output = ts.transpileModule(code, { - compilerOptions: { - target: ts.ScriptTarget.ES2022, - module: ts.ModuleKind.ESNext, - moduleResolution: ts.ModuleResolutionKind.Bundler, - noEmit: true, - allowJs: true, - checkJs: false, - strict: false, - skipLibCheck: true - }, - fileName: "script.ts", - reportDiagnostics: true - }); - - const diagnostics = (output.diagnostics ?? []).map((diagnostic) => - ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n") - ); - - if (!hasSupportedEntrypoint(code)) { - diagnostics.push("Script must export a main or preprocessor function."); - } - - return diagnostics; -} - -function getFlowModules(flow: FlowLikeArtifact): Array> { - const rootModules = Array.isArray(flow.value?.modules) ? flow.value.modules : []; - const collected: Array> = []; - - for (const module of rootModules) { - visitFlowModule(module, collected); - } - - return collected; -} - -function visitFlowModule( - module: Record, - collected: Array> -): void { - collected.push(module); - - const value = asRecord(module.value); - const nestedModules = Array.isArray(value?.modules) ? value.modules : []; - for (const nested of nestedModules) { - if (isRecord(nested)) { - visitFlowModule(nested, collected); - } - } - - const branches = Array.isArray(value?.branches) ? value.branches : []; - for (const branch of branches) { - const branchRecord = asRecord(branch); - const branchModules = Array.isArray(branchRecord?.modules) - ? branchRecord.modules - : []; - for (const nested of branchModules) { - if (isRecord(nested)) { - visitFlowModule(nested, collected); - } - } - } - - const defaultModules = Array.isArray(value?.default) ? value.default : []; - for (const nested of defaultModules) { - if (isRecord(nested)) { - visitFlowModule(nested, collected); - } - } -} - -function collectFlowModuleTypes( - modules: Array> -): Set { - const types = new Set(); - for (const module of modules) { - const value = asRecord(module.value); - if (typeof value?.type === "string") { - types.add(value.type); - } - } - return types; -} - -function getTopLevelFlowModuleIds(flow: FlowLikeArtifact): string[] { - const rootModules = Array.isArray(flow.value?.modules) ? flow.value.modules : []; - return rootModules - .map((module) => (isRecord(module) && typeof module.id === "string" ? module.id : null)) - .filter((id): id is string => id !== null); -} - -function getSchemaRootType(schema: Record | undefined): string | null { - return typeof schema?.type === "string" ? schema.type : null; -} - -function collectBackendReferences(frontendContents: string[]): string[] { - const references = new Set(); - const backendCallPattern = /backend\.([A-Za-z0-9_]+)\s*\(/g; - - for (const content of frontendContents) { - for (const match of content.matchAll(backendCallPattern)) { - const key = match[1]; - if (key) { - references.add(key); - } - } - } - - return [...references].sort((left, right) => left.localeCompare(right)); -} - -function appFilesEqual(left: AppLikeFiles, right: AppLikeFiles): boolean { - return stableStringify(left) === stableStringify(right); -} - -function stableStringify(value: unknown): string { - return JSON.stringify(sortJsonValue(value)); -} - -function sortJsonValue(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map(sortJsonValue); - } - - if (!isRecord(value)) { - return value; - } - - return Object.fromEntries( - Object.entries(value) - .sort((left, right) => left[0].localeCompare(right[0])) - .map(([key, nestedValue]) => [key, sortJsonValue(nestedValue)]) - ); -} - -function normalizeText(value: string): string { - return value.replace(/\r\n/g, "\n").trim(); -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null; -} - -function asRecord(value: unknown): Record | null { - return isRecord(value) ? value : null; -} diff --git a/ai_evals/bun.lock b/ai_evals/bun.lock index aea6fca7f7..1f03e89706 100644 --- a/ai_evals/bun.lock +++ b/ai_evals/bun.lock @@ -6,6 +6,7 @@ "name": "windmill-ai-evals", "dependencies": { "@anthropic-ai/claude-agent-sdk": "^0.2.25", + "@anthropic-ai/sdk": "^0.39.0", "commander": "^14.0.3", }, "devDependencies": { @@ -17,7 +18,7 @@ "packages": { "@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.2.87", "", { "dependencies": { "@anthropic-ai/sdk": "^0.74.0", "@modelcontextprotocol/sdk": "^1.27.1" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "^0.34.2", "@img/sharp-darwin-x64": "^0.34.2", "@img/sharp-linux-arm": "^0.34.2", "@img/sharp-linux-arm64": "^0.34.2", "@img/sharp-linux-x64": "^0.34.2", "@img/sharp-linuxmusl-arm64": "^0.34.2", "@img/sharp-linuxmusl-x64": "^0.34.2", "@img/sharp-win32-arm64": "^0.34.2", "@img/sharp-win32-x64": "^0.34.2" }, "peerDependencies": { "zod": "^4.0.0" } }, "sha512-WWmgBPxPhBOvNT0ujI8vPTI2lK+w5YEkEZ/y1mH0EDkK/0kBnxVJNhCtG5vnueiAViwLoUOFn66pbkDiivijdA=="], - "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.74.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-srbJV7JKsc5cQ6eVuFzjZO7UR3xEPJqPamHFIe29bs38Ij2IripoAhC0S5NslNbaFUYqBKypmmpzMTpqfHEUDw=="], + "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.39.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" } }, "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg=="], "@babel/runtime": ["@babel/runtime@7.29.2", "", {}, "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g=="], @@ -59,14 +60,22 @@ "@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="], - "@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="], + "@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="], + + "@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="], + + "abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="], "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], + "agentkeepalive": ["agentkeepalive@4.6.0", "", { "dependencies": { "humanize-ms": "^1.2.1" } }, "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ=="], + "ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="], "ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="], + "asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="], + "body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="], "bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="], @@ -77,6 +86,8 @@ "call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="], + "combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="], + "commander": ["commander@14.0.3", "", {}, "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw=="], "content-disposition": ["content-disposition@1.0.1", "", {}, "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q=="], @@ -93,6 +104,8 @@ "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], + "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], @@ -107,10 +120,14 @@ "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="], + "es-set-tostringtag": ["es-set-tostringtag@2.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA=="], + "escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="], "etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="], + "event-target-shim": ["event-target-shim@5.0.1", "", {}, "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="], + "eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="], "eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="], @@ -125,6 +142,12 @@ "finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="], + "form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="], + + "form-data-encoder": ["form-data-encoder@1.7.2", "", {}, "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="], + + "formdata-node": ["formdata-node@4.4.1", "", { "dependencies": { "node-domexception": "1.0.0", "web-streams-polyfill": "4.0.0-beta.3" } }, "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ=="], + "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="], "fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="], @@ -139,12 +162,16 @@ "has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="], + "has-tostringtag": ["has-tostringtag@1.0.2", "", { "dependencies": { "has-symbols": "^1.0.3" } }, "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw=="], + "hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="], "hono": ["hono@4.12.9", "", {}, "sha512-wy3T8Zm2bsEvxKZM5w21VdHDDcwVS1yUFFY6i8UobSsKfFceT7TOwhbhfKsDyx7tYQlmRM5FLpIuYvNFyjctiA=="], "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], + "humanize-ms": ["humanize-ms@1.2.1", "", { "dependencies": { "ms": "^2.0.0" } }, "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ=="], + "iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], @@ -179,6 +206,10 @@ "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], + "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="], + + "node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="], + "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], @@ -231,18 +262,26 @@ "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], + "tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="], + "ts-algebra": ["ts-algebra@2.0.0", "", {}, "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw=="], "type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="], "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], - "undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], + "undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="], "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="], "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], + "web-streams-polyfill": ["web-streams-polyfill@4.0.0-beta.3", "", {}, "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug=="], + + "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], + + "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], + "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], @@ -250,5 +289,19 @@ "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], "zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="], + + "@anthropic-ai/claude-agent-sdk/@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.74.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-srbJV7JKsc5cQ6eVuFzjZO7UR3xEPJqPamHFIe29bs38Ij2IripoAhC0S5NslNbaFUYqBKypmmpzMTpqfHEUDw=="], + + "@types/node-fetch/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="], + + "bun-types/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="], + + "form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], + + "@types/node-fetch/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], + + "bun-types/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], + + "form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], } } diff --git a/ai_evals/cases/app.json b/ai_evals/cases/app.json new file mode 100644 index 0000000000..355a376df6 --- /dev/null +++ b/ai_evals/cases/app.json @@ -0,0 +1,44 @@ +[ + { + "id": "app-test1-counter-create", + "prompt": "Create a counter app with increment/decrement buttons" + }, + { + "id": "app-test2-counter-reset", + "prompt": "Add a reset button that sets the counter back to 0", + "initial": "ai_evals/fixtures/frontend/app/initial/test1_counter_app" + }, + { + "id": "app-test3-shopping-cart-quantity", + "prompt": "Add a quantity selector (+ and - buttons) to each cart item so users can adjust quantities without removing and re-adding items", + "initial": "ai_evals/fixtures/frontend/app/initial/shopping_cart" + }, + { + "id": "app-test4-shopping-cart-discount", + "prompt": "Add a discount code input field in the cart. When the code \"SAVE10\" is entered, apply a 10% discount to the total", + "initial": "ai_evals/fixtures/frontend/app/initial/shopping_cart" + }, + { + "id": "app-test5-file-manager-search", + "prompt": "Add a search bar in the toolbar that filters files and folders by name as the user types", + "initial": "ai_evals/fixtures/frontend/app/initial/file_manager" + }, + { + "id": "app-test6-file-manager-details", + "prompt": "Show file size (formatted as KB/MB) and modified date in the file list for each item", + "initial": "ai_evals/fixtures/frontend/app/initial/file_manager" + }, + { + "id": "app-test7-file-manager-select-all", + "prompt": "Add a \"Select All\" checkbox in the file list header and individual checkboxes for each file. Add a \"Delete Selected\" button that appears when items are selected", + "initial": "ai_evals/fixtures/frontend/app/initial/file_manager" + }, + { + "id": "app-test8-quiz-create", + "prompt": "Create a multiple choice quiz app with 5 questions about general knowledge. Show one question at a time with 4 answer options. Track the score and show results at the end with percentage correct." + }, + { + "id": "app-test9-recipe-book-create", + "prompt": "Create a recipe book app where users can add recipes with a name, ingredients list, and instructions. Include a search bar to filter recipes by name and the ability to delete recipes." + } +] diff --git a/ai_evals/cases/cli.json b/ai_evals/cases/cli.json new file mode 100644 index 0000000000..6d2eb9757c --- /dev/null +++ b/ai_evals/cases/cli.json @@ -0,0 +1,12 @@ +[ + { + "id": "bun-hello-script", + "prompt": "This is a benchmark harness. Create exactly one Windmill Bun/TypeScript script at {{workspace_root}}/f/evals/hello.ts. The script must export async function main(name: string) and return an object { greeting: `Hello, ${name}!` }. Keep it minimal. Do not create other scripts. Do not run any CLI commands. After writing the file, tell me exactly which wmill commands I should run next.", + "expected": "ai_evals/fixtures/cli/expected/bun-hello-script" + }, + { + "id": "bun-hello-flow", + "prompt": "This is a benchmark harness. Create exactly one Windmill flow folder at {{workspace_root}}/f/evals/hello__flow. The flow must contain flow.yaml and one inline Bun script file named hello.ts. The flow should accept a name string input and return an object { greeting: `Hello, ${name}!` }. Use a single rawscript step wired to that input. Keep it minimal. Do not create any other flows or scripts. Do not run any CLI commands. After writing the files, tell me exactly which wmill commands I should run next.", + "expected": "ai_evals/fixtures/cli/expected/bun-hello-flow" + } +] diff --git a/ai_evals/cases/cli/flow.json b/ai_evals/cases/cli/flow.json deleted file mode 100644 index e057c995fa..0000000000 --- a/ai_evals/cases/cli/flow.json +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "id": "bun-hello-flow", - "surface": "cli", - "title": "Create a minimal Bun flow in a fresh CLI workspace.", - "user_prompt": "This is a benchmark harness. Create exactly one Windmill flow folder at {{workspace_root}}/f/evals/hello__flow. The flow must contain flow.yaml and one inline Bun script file named hello.ts. The flow should accept a name string input and return an object { greeting: `Hello, ${name}!` }. Use a single rawscript step wired to that input. Keep it minimal. Do not create any other flows or scripts. Do not run any CLI commands. After writing the files, tell me exactly which wmill commands I should run next.", - "initial_state": {}, - "workspace_context": { - "max_turns": 8 - }, - "artifact_checks": { - "expected_skill": "write-flow", - "expected_output_substrings": [ - "wmill flow generate-locks", - "wmill sync push" - ], - "expected_files": [ - { - "path": "f/evals/hello__flow/flow.yaml", - "must_contain": [ - "value:", - "modules:", - "name:" - ] - }, - { - "path": "f/evals/hello__flow/hello.ts", - "must_contain": [ - "export async function main(name: string)", - "greeting: `Hello, ${name}!`" - ] - } - ] - }, - "judge_rubric": {}, - "tags": [ - "cli", - "flow", - "create" - ] - } -] diff --git a/ai_evals/cases/cli/script.json b/ai_evals/cases/cli/script.json deleted file mode 100644 index e4eac24819..0000000000 --- a/ai_evals/cases/cli/script.json +++ /dev/null @@ -1,34 +0,0 @@ -[ - { - "id": "bun-hello-script", - "surface": "cli", - "title": "Create a minimal Bun script in a fresh CLI workspace.", - "user_prompt": "This is a benchmark harness. Create exactly one Windmill Bun/TypeScript script at {{workspace_root}}/f/evals/hello.ts. The script must export async function main(name: string) and return an object { greeting: `Hello, ${name}!` }. Keep it minimal. Do not create other scripts. Do not run any CLI commands. After writing the file, tell me exactly which wmill commands I should run next.", - "initial_state": {}, - "workspace_context": { - "max_turns": 6 - }, - "artifact_checks": { - "expected_skill": "write-script-bun", - "expected_output_substrings": [ - "wmill script generate-metadata", - "wmill sync push" - ], - "expected_files": [ - { - "path": "f/evals/hello.ts", - "must_contain": [ - "export async function main(name: string)", - "return { greeting: `Hello, ${name}!` };" - ] - } - ] - }, - "judge_rubric": {}, - "tags": [ - "cli", - "script", - "create" - ] - } -] diff --git a/ai_evals/cases/flow.json b/ai_evals/cases/flow.json new file mode 100644 index 0000000000..5f5f1703f7 --- /dev/null +++ b/ai_evals/cases/flow.json @@ -0,0 +1,45 @@ +[ + { + "id": "flow-test0-sum-two-numbers", + "prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nCreate a flow with a single Bun rawscript step named \"sum_numbers\".\nThe flow input must be two numbers named a and b.\nThe rawscript must read a and b from flow input and return a + b.\nDo not add extra steps, branches, loops, AI agents, or test steps.", + "expected": "ai_evals/fixtures/frontend/flow/expected/test0_sum_two_numbers.json" + }, + { + "id": "flow-test1-user-role-actions", + "prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nSTEP 1: Fetch mock users from api\nSTEP 2: Filter only active users:\nSTEP 3: Loop on all users\nSTEP 4: Do branches based on user's role, do different action based on that. Roles are admin, user, moderator\nSTEP 5: Return action taken for each user", + "expected": "ai_evals/fixtures/frontend/flow/expected/test1.json" + }, + { + "id": "flow-test2-order-processing", + "prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nSTEP 1: Receive order data from input (order has items array with name/price/quantity, customer_email, shipping_address)\nSTEP 2: Validate order - check all items have valid price > 0 and quantity > 0, return validation result\nSTEP 3: Calculate order total with 8% tax rate\nSTEP 4: Check inventory for each item (loop through items, return mock availability)\nSTEP 5: Branch based on inventory - if all items available, create shipment record; otherwise create backorder record\nSTEP 6: Send confirmation (mock email to customer_email)\nSTEP 7: Return final order summary with status", + "expected": "ai_evals/fixtures/frontend/flow/expected/test2.json" + }, + { + "id": "flow-test3-data-pipeline", + "prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nSTEP 1: Fetch list of data sources from configuration (return mock array of 3 source objects with id and url)\nSTEP 2: For each data source in parallel:\n - Fetch raw data from the source (mock fetch returning sample records)\n - Transform/clean the data (filter out invalid entries)\n - Validate the transformed data (return validation score 0-100)\nSTEP 3: Aggregate all validated data into single dataset with combined records\nSTEP 4: Calculate overall data quality score (average of all validation scores)\nSTEP 5: Branch based on quality score:\n - If score >= 90: Store in primary database and return success\n - If score >= 70 and < 90: Store in secondary database with warning flag\n - If score < 70: Store in quarantine and send alert\nSTEP 6: Return processing report with statistics (total records, quality score, destination)", + "expected": "ai_evals/fixtures/frontend/flow/expected/test3.json" + }, + { + "id": "flow-test4-ai-agent-tools", + "prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nCreate a customer support flow with an AI agent:\n\nSTEP 1: Receive customer query from input (customer_id string, query_text string)\nSTEP 2: Fetch customer profile and order history (mock data based on customer_id)\nSTEP 3: Use an AI agent to handle the customer query. The agent should have access to these tools:\n - lookup_order: Takes order_id, returns order details (mock data)\n - check_refund_eligibility: Takes order_id, returns eligibility status and reason\n - create_support_ticket: Takes description and priority (low/medium/high), returns ticket_id\n - search_faq: Takes search_query, returns relevant FAQ answers\n The agent should use the customer profile context and respond helpfully.\nSTEP 4: Log the interaction to audit trail (customer_id, query, response summary)\nSTEP 5: Return the agent's response and any actions taken", + "expected": "ai_evals/fixtures/frontend/flow/expected/test4.json" + }, + { + "id": "flow-test5-simple-modification", + "prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nModify this existing flow to add error handling:\n- Add a new step after process_data called \"validate_data\" to validate the processed data\n- The validation step should check if the data array is not empty\n- If validation fails (empty array), it should return an error object with message \"No data to save\"\n- If validation passes, return the data for the next step\n- Update save_results to handle the validation result appropriately", + "initial": "ai_evals/fixtures/frontend/flow/initial/test5_initial.json", + "expected": "ai_evals/fixtures/frontend/flow/expected/test5_modify_simple.json" + }, + { + "id": "flow-test6-branching-in-loop", + "prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nModify the order processing loop to handle different order types:\n- Inside the loop_orders, replace the simple process_order step with branching based on order.type\n- For type \"express\": add a step called handle_express that marks as priority and calculates express shipping cost ($15.99)\n- For type \"standard\": add a step called handle_standard that calculates standard shipping cost ($5.99)\n- For type \"pickup\": add a step called handle_pickup that marks as no shipping required (cost $0)\n- Move the original process_order step to the default branch for unknown order types\n- Each branch step should return the orderId, shipping cost, and shipping type", + "initial": "ai_evals/fixtures/frontend/flow/initial/test6_initial.json", + "expected": "ai_evals/fixtures/frontend/flow/expected/test6_modify_medium.json" + }, + { + "id": "flow-test7-parallel-refactor", + "prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nRefactor this flow for better performance by parallelizing the enrichment steps:\n- The three enrichment steps (enrich_price, enrich_inventory, enrich_reviews) currently run sequentially\n- Wrap them in a parallel branch (branchall) called \"parallel_enrichment\" so they run concurrently\n- Each enrichment step should include basic error handling with try/catch that returns a fallback value if it fails\n- Update the combine_data step to receive results from the parallel branch (results.parallel_enrichment returns an array of branch results)\n- The combine_data step should check if any enrichment used a fallback value and set a hasFallbacks flag\n- Keep get_item as the first step and return_result as the last step unchanged", + "initial": "ai_evals/fixtures/frontend/flow/initial/test7_initial.json", + "expected": "ai_evals/fixtures/frontend/flow/expected/test7_modify_complex.json" + } +] diff --git a/ai_evals/cases/frontend/app.json b/ai_evals/cases/frontend/app.json deleted file mode 100644 index e6c90e8b21..0000000000 --- a/ai_evals/cases/frontend/app.json +++ /dev/null @@ -1,167 +0,0 @@ -[ - { - "id": "app-test1-counter-create", - "surface": "frontend-app", - "title": "test1: creates a simple counter app", - "user_prompt": "Create a counter app with increment/decrement buttons", - "initial_state": {}, - "workspace_context": {}, - "artifact_checks": {}, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "app", - "create" - ] - }, - { - "id": "app-test2-counter-reset", - "surface": "frontend-app", - "title": "test2: modifies existing counter app to add reset button", - "user_prompt": "Add a reset button that sets the counter back to 0", - "initial_state": { - "app_fixture_path": "ai_evals/fixtures/frontend/app/initial/test1_counter_app" - }, - "workspace_context": {}, - "artifact_checks": {}, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "app", - "modify" - ] - }, - { - "id": "app-test3-shopping-cart-quantity", - "surface": "frontend-app", - "title": "test3: shopping cart - add quantity selector", - "user_prompt": "Add a quantity selector (+ and - buttons) to each cart item so users can adjust quantities without removing and re-adding items", - "initial_state": { - "app_fixture_path": "ai_evals/fixtures/frontend/app/initial/shopping_cart" - }, - "workspace_context": {}, - "artifact_checks": {}, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "app", - "modify" - ] - }, - { - "id": "app-test4-shopping-cart-discount", - "surface": "frontend-app", - "title": "test4: shopping cart - add discount code", - "user_prompt": "Add a discount code input field in the cart. When the code \"SAVE10\" is entered, apply a 10% discount to the total", - "initial_state": { - "app_fixture_path": "ai_evals/fixtures/frontend/app/initial/shopping_cart" - }, - "workspace_context": {}, - "artifact_checks": {}, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "app", - "modify" - ] - }, - { - "id": "app-test5-file-manager-search", - "surface": "frontend-app", - "title": "test5: file manager - add search bar", - "user_prompt": "Add a search bar in the toolbar that filters files and folders by name as the user types", - "initial_state": { - "app_fixture_path": "ai_evals/fixtures/frontend/app/initial/file_manager" - }, - "workspace_context": {}, - "artifact_checks": {}, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "app", - "modify" - ] - }, - { - "id": "app-test6-file-manager-details", - "surface": "frontend-app", - "title": "test6: file manager - show file details", - "user_prompt": "Show file size (formatted as KB/MB) and modified date in the file list for each item", - "initial_state": { - "app_fixture_path": "ai_evals/fixtures/frontend/app/initial/file_manager" - }, - "workspace_context": {}, - "artifact_checks": {}, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "app", - "modify" - ] - }, - { - "id": "app-test7-file-manager-select-all", - "surface": "frontend-app", - "title": "test7: file manager - add select all checkbox", - "user_prompt": "Add a \"Select All\" checkbox in the file list header and individual checkboxes for each file. Add a \"Delete Selected\" button that appears when items are selected", - "initial_state": { - "app_fixture_path": "ai_evals/fixtures/frontend/app/initial/file_manager" - }, - "workspace_context": {}, - "artifact_checks": {}, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "app", - "modify" - ] - }, - { - "id": "app-test8-quiz-create", - "surface": "frontend-app", - "title": "test8: create quiz app from scratch", - "user_prompt": "Create a multiple choice quiz app with 5 questions about general knowledge. Show one question at a time with 4 answer options. Track the score and show results at the end with percentage correct.", - "initial_state": {}, - "workspace_context": {}, - "artifact_checks": {}, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "app", - "create" - ] - }, - { - "id": "app-test9-recipe-book-create", - "surface": "frontend-app", - "title": "test9: create recipe book from scratch", - "user_prompt": "Create a recipe book app where users can add recipes with a name, ingredients list, and instructions. Include a search bar to filter recipes by name and the ability to delete recipes.", - "initial_state": {}, - "workspace_context": {}, - "artifact_checks": {}, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "app", - "create" - ] - } -] diff --git a/ai_evals/cases/frontend/flow.json b/ai_evals/cases/frontend/flow.json deleted file mode 100644 index 4123fc878a..0000000000 --- a/ai_evals/cases/frontend/flow.json +++ /dev/null @@ -1,141 +0,0 @@ -[ - { - "id": "flow-test1-user-role-actions", - "surface": "frontend-flow", - "title": "test1: user role-based actions with loop and branches", - "user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nSTEP 1: Fetch mock users from api\nSTEP 2: Filter only active users:\nSTEP 3: Loop on all users\nSTEP 4: Do branches based on user's role, do different action based on that. Roles are admin, user, moderator\nSTEP 5: Return action taken for each user", - "initial_state": {}, - "workspace_context": {}, - "artifact_checks": { - "expected_flow_path": "ai_evals/fixtures/frontend/flow/expected/test1.json" - }, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "flow", - "create" - ] - }, - { - "id": "flow-test2-order-processing", - "surface": "frontend-flow", - "title": "test2: e-commerce order processing with inventory check and branching", - "user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nSTEP 1: Receive order data from input (order has items array with name/price/quantity, customer_email, shipping_address)\nSTEP 2: Validate order - check all items have valid price > 0 and quantity > 0, return validation result\nSTEP 3: Calculate order total with 8% tax rate\nSTEP 4: Check inventory for each item (loop through items, return mock availability)\nSTEP 5: Branch based on inventory - if all items available, create shipment record; otherwise create backorder record\nSTEP 6: Send confirmation (mock email to customer_email)\nSTEP 7: Return final order summary with status", - "initial_state": {}, - "workspace_context": {}, - "artifact_checks": { - "expected_flow_path": "ai_evals/fixtures/frontend/flow/expected/test2.json" - }, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "flow", - "create" - ] - }, - { - "id": "flow-test3-data-pipeline", - "surface": "frontend-flow", - "title": "test3: data pipeline with parallel processing and quality-based routing", - "user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nSTEP 1: Fetch list of data sources from configuration (return mock array of 3 source objects with id and url)\nSTEP 2: For each data source in parallel:\n - Fetch raw data from the source (mock fetch returning sample records)\n - Transform/clean the data (filter out invalid entries)\n - Validate the transformed data (return validation score 0-100)\nSTEP 3: Aggregate all validated data into single dataset with combined records\nSTEP 4: Calculate overall data quality score (average of all validation scores)\nSTEP 5: Branch based on quality score:\n - If score >= 90: Store in primary database and return success\n - If score >= 70 and < 90: Store in secondary database with warning flag\n - If score < 70: Store in quarantine and send alert\nSTEP 6: Return processing report with statistics (total records, quality score, destination)", - "initial_state": {}, - "workspace_context": {}, - "artifact_checks": { - "expected_flow_path": "ai_evals/fixtures/frontend/flow/expected/test3.json" - }, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "flow", - "create" - ] - }, - { - "id": "flow-test4-ai-agent-tools", - "surface": "frontend-flow", - "title": "test4: AI agent with tools for customer support", - "user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nCreate a customer support flow with an AI agent:\n\nSTEP 1: Receive customer query from input (customer_id string, query_text string)\nSTEP 2: Fetch customer profile and order history (mock data based on customer_id)\nSTEP 3: Use an AI agent to handle the customer query. The agent should have access to these tools:\n - lookup_order: Takes order_id, returns order details (mock data)\n - check_refund_eligibility: Takes order_id, returns eligibility status and reason\n - create_support_ticket: Takes description and priority (low/medium/high), returns ticket_id\n - search_faq: Takes search_query, returns relevant FAQ answers\n The agent should use the customer profile context and respond helpfully.\nSTEP 4: Log the interaction to audit trail (customer_id, query, response summary)\nSTEP 5: Return the agent's response and any actions taken", - "initial_state": {}, - "workspace_context": {}, - "artifact_checks": { - "expected_flow_path": "ai_evals/fixtures/frontend/flow/expected/test4.json" - }, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "flow", - "create" - ] - }, - { - "id": "flow-test5-simple-modification", - "surface": "frontend-flow", - "title": "test5: simple modification - add validation step to existing flow", - "user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nModify this existing flow to add error handling:\n- Add a new step after process_data called \"validate_data\" to validate the processed data\n- The validation step should check if the data array is not empty\n- If validation fails (empty array), it should return an error object with message \"No data to save\"\n- If validation passes, return the data for the next step\n- Update save_results to handle the validation result appropriately", - "initial_state": { - "flow_path": "ai_evals/fixtures/frontend/flow/initial/test5_initial.json" - }, - "workspace_context": {}, - "artifact_checks": { - "expected_flow_path": "ai_evals/fixtures/frontend/flow/expected/test5_modify_simple.json" - }, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "flow", - "modify" - ] - }, - { - "id": "flow-test6-branching-in-loop", - "surface": "frontend-flow", - "title": "test6: medium modification - add branching inside existing loop", - "user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nModify the order processing loop to handle different order types:\n- Inside the loop_orders, replace the simple process_order step with branching based on order.type\n- For type \"express\": add a step called handle_express that marks as priority and calculates express shipping cost ($15.99)\n- For type \"standard\": add a step called handle_standard that calculates standard shipping cost ($5.99)\n- For type \"pickup\": add a step called handle_pickup that marks as no shipping required (cost $0)\n- Move the original process_order step to the default branch for unknown order types\n- Each branch step should return the orderId, shipping cost, and shipping type", - "initial_state": { - "flow_path": "ai_evals/fixtures/frontend/flow/initial/test6_initial.json" - }, - "workspace_context": {}, - "artifact_checks": { - "expected_flow_path": "ai_evals/fixtures/frontend/flow/expected/test6_modify_medium.json" - }, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "flow", - "modify" - ] - }, - { - "id": "flow-test7-parallel-refactor", - "surface": "frontend-flow", - "title": "test7: complex modification - refactor sequential to parallel execution", - "user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nRefactor this flow for better performance by parallelizing the enrichment steps:\n- The three enrichment steps (enrich_price, enrich_inventory, enrich_reviews) currently run sequentially\n- Wrap them in a parallel branch (branchall) called \"parallel_enrichment\" so they run concurrently\n- Each enrichment step should include basic error handling with try/catch that returns a fallback value if it fails\n- Update the combine_data step to receive results from the parallel branch (results.parallel_enrichment returns an array of branch results)\n- The combine_data step should check if any enrichment used a fallback value and set a hasFallbacks flag\n- Keep get_item as the first step and return_result as the last step unchanged", - "initial_state": { - "flow_path": "ai_evals/fixtures/frontend/flow/initial/test7_initial.json" - }, - "workspace_context": {}, - "artifact_checks": { - "expected_flow_path": "ai_evals/fixtures/frontend/flow/expected/test7_modify_complex.json" - }, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "flow", - "modify" - ] - } -] diff --git a/ai_evals/cases/frontend/script.json b/ai_evals/cases/frontend/script.json deleted file mode 100644 index f71e27a284..0000000000 --- a/ai_evals/cases/frontend/script.json +++ /dev/null @@ -1,23 +0,0 @@ -[ - { - "id": "script-test1-greet-user", - "surface": "frontend-script", - "title": "test1: create a greeting script", - "user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE.\n\nUpdate the current bun script so it exports `main(name: string)` and returns the plain string `Hello, ${name}!`.\nDo not return an object or array.\nDo not add external dependencies.", - "initial_state": { - "script_path": "ai_evals/fixtures/frontend/script/initial/test1_empty_bun.json" - }, - "workspace_context": {}, - "artifact_checks": { - "expected_script_path": "ai_evals/fixtures/frontend/script/expected/test1_greet_user.json" - }, - "judge_rubric": { - "min_score": 80 - }, - "tags": [ - "frontend", - "script", - "modify" - ] - } -] diff --git a/ai_evals/cases/script.json b/ai_evals/cases/script.json new file mode 100644 index 0000000000..5d1c8b2ace --- /dev/null +++ b/ai_evals/cases/script.json @@ -0,0 +1,8 @@ +[ + { + "id": "script-test1-greet-user", + "prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE.\n\nUpdate the current bun script so it exports `main(name: string)` and returns the plain string `Hello, ${name}!`.\nDo not return an object or array.\nDo not add external dependencies.", + "initial": "ai_evals/fixtures/frontend/script/initial/test1_empty_bun.json", + "expected": "ai_evals/fixtures/frontend/script/expected/test1_greet_user.json" + } +] diff --git a/ai_evals/cli/README.md b/ai_evals/cli/README.md deleted file mode 100644 index 37c4e56724..0000000000 --- a/ai_evals/cli/README.md +++ /dev/null @@ -1,116 +0,0 @@ -# Benchmark CLI - -The benchmark CLI is built around saved local results. - -The normal loop is: - -1. run the suite on the current checkout -2. make your change -3. run again -4. diff the two saved results - -## Commands - -List cases: - -```bash -cd ai_evals -bun run cli -- list-cases -bun run cli -- list-cases --surface flow -``` - -Run the current checkout and save a result under `ai_evals/results/`: - -```bash -cd ai_evals -bun run cli -- run --surface flow --runs 3 -bun run cli -- run --surface cli --runs 3 -``` - -Diff two saved results: - -```bash -cd ai_evals -bun run cli -- diff-results ai_evals/results/before.json ai_evals/results/after.json -``` - -Show recent official history: - -```bash -cd ai_evals -bun run cli -- history --limit 10 -``` - -Promote one local result into official history: - -```bash -cd ai_evals -bun run cli -- promote-result ai_evals/results/latest.json --label main -``` - -## Frontend Workflow - -If you are improving frontend flow/app/script chat, use the surface directly: - -```bash -cd ai_evals -bun run cli -- run --surface flow --runs 3 -``` - -Optional frontend overrides: - -- `--provider anthropic|openai` -- `--model ` -- `--system-prompt-file ` to fully replace the system prompt -- `--append-system-prompt-file ` to append extra instructions to the - default system prompt - -Example: - -```bash -cd ai_evals -bun run cli -- run --surface flow --append-system-prompt-file ./prompt-experiment.md --runs 3 -``` - -The benchmark user prompt still comes from the case manifest. These flags are -for current-checkout experiments, not for a checked-in variant system. - -## CLI Guidance Workflow - -If you are improving CLI skills or project guidance, run the `cli` surface: - -```bash -cd ai_evals -bun run cli -- run --surface cli --runs 3 -``` - -Optional CLI overrides: - -- `--skills-source ` -- `--agents-source ` -- `--claude-source ` - -Example: - -```bash -cd ai_evals -bun run cli -- run --surface cli --skills-source ./system_prompts/auto-generated/skills --runs 3 -``` - -For debugging one CLI case locally, keep the final temp workspace: - -```bash -cd ai_evals -bun run cli -- run --surface cli --case bun-hello-script --runs 1 --keep-workspace -``` - -## Result Files - -Each `run` command writes one JSON result file containing: - -- run metadata -- aggregate metrics -- per-case summaries -- per-attempt details - -Those files are meant for local comparison and are ignored by git. diff --git a/ai_evals/cli/index.ts b/ai_evals/cli/index.ts index 4bf9f735ba..51a7b12c87 100644 --- a/ai_evals/cli/index.ts +++ b/ai_evals/cli/index.ts @@ -1,57 +1,18 @@ #!/usr/bin/env bun -import { execFileSync } from "node:child_process"; -import { readFile } from "node:fs/promises"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { Command, InvalidArgumentError, Option } from "commander"; -import { - cleanupWorkspace, - loadCliArtifactEvalCases, - runCliArtifactEvalCase, - type CliArtifactEvalCase, - type CliGuidanceConfig, -} from "../adapters/cli/artifact-eval"; -import { - CLI_BENCHMARK_MODEL, - CLI_BENCHMARK_PROVIDER, -} from "../adapters/cli/runtime"; -import { - runFrontendBenchmarkAdapter, - type FrontendAdapterPayload, - type FrontendBenchmarkConfig as FrontendAdapterConfig, -} from "../adapters/frontend/runtime"; -import { - loadEvalCaseSummaries, - type EvalSurfaceName, -} from "../adapters/shared/evalCases"; -import { - appendOfficialRun, - loadSummaryHistory, -} from "../history/writer.mjs"; -import { - buildBenchmarkRunDiff, - buildOfficialRunFromResult, - formatSurfaceLabel, - readBenchmarkRunResult, - writeBenchmarkRunResult, - type AggregateMetrics, - type AttemptSummary, - type BenchmarkCaseResult, - type BenchmarkRunDiff, - type BenchmarkRunResult, - type CanonicalSurfaceName, - type SurfaceName, -} from "./results"; - -const USER_SURFACES = ["cli", "flow", "app", "script"] as const; -const PROVIDERS = ["anthropic", "openai"] as const; +import { Command, InvalidArgumentError } from "commander"; +import { loadCases, loadSelectedCases } from "../core/cases"; +import { buildRunResult, formatRunSummary, writeRunResult } from "../core/results"; +import { runSuite } from "../core/runSuite"; +import { EVAL_MODES, type EvalMode } from "../core/types"; +import { DEFAULT_JUDGE_MODEL } from "../core/judge"; +import { createCliModeRunner, getCliRunModelLabel } from "../modes/cli"; +import { runFrontendBenchmarkAdapter } from "../adapters/frontend/runtime"; async function main() { const program = new Command() .name("bun run cli --") - .description("Run AI evals on the current checkout and diff saved results") - .usage(" [options]") + .description("Run AI eval cases against the current production prompts and guidance") .showHelpAfterError() .showSuggestionAfterError() .addHelpText( @@ -59,881 +20,128 @@ async function main() { [ "", "Examples:", - " bun run cli -- list-cases", - " bun run cli -- run --surface flow --runs 3", - " bun run cli -- run --surface cli --skills-source ./system_prompts/auto-generated/skills", - " bun run cli -- diff-results ai_evals/results/before.json ai_evals/results/after.json", - " bun run cli -- promote-result ai_evals/results/latest.json --label main", + " bun run cli -- cases", + " bun run cli -- cases flow", + " bun run cli -- run flow", + " bun run cli -- run flow flow-test5-simple-modification --runs 3", + " bun run cli -- run cli bun-hello-script", ].join("\n") ); program - .command("list-cases") - .description("List benchmark cases") - .addOption(createOptionalSurfaceOption()) - .option("--json", "print machine-readable output") - .action(async (options: { surface?: SurfaceName; json?: boolean }) => { - await handleListCases(options.surface, options.json ?? false); + .command("cases") + .description("List available cases") + .argument("[mode]", "cli, flow, script, or app", parseOptionalMode) + .action(async (mode?: EvalMode) => { + await handleCases(mode); }); program .command("run") - .description("Run the current checkout on one surface and save a local result") - .addOption(createRequiredSurfaceOption()) - .option("--case ", "case id (repeatable)", collectOptionValues) - .option("--runs ", "number of runs per case", parsePositiveInteger, 1) - .option("--output ", "write the local result to this path") - .option("--label