diff --git a/ai_evals/adapters/frontend/benchmarkRunner.ts b/ai_evals/adapters/frontend/benchmarkRunner.ts index 5a8fae5215..a2f8556173 100644 --- a/ai_evals/adapters/frontend/benchmarkRunner.ts +++ b/ai_evals/adapters/frontend/benchmarkRunner.ts @@ -16,6 +16,7 @@ export async function runFrontendBenchmarkFromEnv(): Promise const caseIds = parseOptionalJsonStringArray(process.env.WMILL_FRONTEND_AI_EVAL_CASE_IDS); const runs = parsePositiveInteger(process.env.WMILL_FRONTEND_AI_EVAL_RUNS, "WMILL_FRONTEND_AI_EVAL_RUNS"); const emitProgress = process.env.WMILL_FRONTEND_AI_EVAL_PROGRESS === "1"; + const verbose = process.env.WMILL_FRONTEND_AI_EVAL_VERBOSE === "1"; const selectedCases = await loadSelectedCases(mode, caseIds); const modeRunner = getModeRunner(mode); @@ -25,6 +26,8 @@ export async function runFrontendBenchmarkFromEnv(): Promise runs, runModel: getFrontendRunModelLabel(), judgeModel: DEFAULT_JUDGE_MODEL, + concurrency: verbose ? 1 : undefined, + verbose, onProgress: emitProgress ? (event) => emitFrontendBenchmarkProgress(event) : undefined, }); diff --git a/ai_evals/adapters/frontend/core/app/appEvalRunner.ts b/ai_evals/adapters/frontend/core/app/appEvalRunner.ts index ea086bf979..a01410d2a6 100644 --- a/ai_evals/adapters/frontend/core/app/appEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/app/appEvalRunner.ts @@ -15,6 +15,7 @@ import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/com import { createAppFileHelpers } from './fileHelpers' import { runEval } from '../shared' import type { AIProvider } from '$lib/gen/types.gen' +import type { ModeRunContext } from '../../../../core/types' export interface AppEvalResult { success: boolean @@ -32,6 +33,7 @@ export interface AppEvalOptions { maxIterations?: number provider?: AIProvider workspaceRoot?: string + runContext?: ModeRunContext } export async function runAppEval( @@ -62,6 +64,9 @@ export async function runAppEval( helpers, apiKey, getOutput: getFiles, + onAssistantMessageStart: options?.runContext?.onAssistantMessageStart, + onAssistantToken: options?.runContext?.onAssistantChunk, + onAssistantMessageEnd: options?.runContext?.onAssistantMessageEnd, options: { maxIterations: options?.maxIterations, model, diff --git a/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts b/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts index 272308b0b3..d4b3d247f5 100644 --- a/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts @@ -13,6 +13,7 @@ import { import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared' import { createFlowFileHelpers } from './fileHelpers' import { runEval } from '../shared' +import type { ModeRunContext } from '../../../../core/types' export interface FlowFixture { value?: { @@ -36,6 +37,7 @@ export interface FlowEvalOptions { maxIterations?: number provider?: AIProvider workspaceRoot?: string + runContext?: ModeRunContext } export async function runFlowEval( @@ -66,6 +68,9 @@ export async function runFlowEval( helpers, apiKey, getOutput: getFlow, + onAssistantMessageStart: options?.runContext?.onAssistantMessageStart, + onAssistantToken: options?.runContext?.onAssistantChunk, + onAssistantMessageEnd: options?.runContext?.onAssistantMessageEnd, options: { maxIterations: options?.maxIterations, model, diff --git a/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts b/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts index ae1cccbb97..ec90bf35b2 100644 --- a/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts @@ -12,6 +12,7 @@ import { import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared' import { createScriptFileHelpers, type ScriptEvalState } from './fileHelpers' import { runEval } from '../shared' +import type { ModeRunContext } from '../../../../core/types' export interface ScriptEvalResult { success: boolean @@ -28,6 +29,7 @@ export interface ScriptEvalOptions { maxIterations?: number provider?: AIProvider workspaceRoot?: string + runContext?: ModeRunContext } function resolveModelProvider( @@ -79,6 +81,9 @@ export async function runScriptEval( helpers, apiKey, getOutput: getScript, + onAssistantMessageStart: options.runContext?.onAssistantMessageStart, + onAssistantToken: options.runContext?.onAssistantChunk, + onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd, options: { maxIterations: options.maxIterations, model, diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 55c3941483..3313bbabfb 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -34,6 +34,9 @@ export interface RunEvalParams { getOutput: () => TOutput /** Optional configuration */ options?: EvalRunnerOptions + onAssistantMessageStart?: () => void + onAssistantToken?: (token: string) => void + onAssistantMessageEnd?: () => void } /** @@ -79,8 +82,12 @@ export async function runEval( helpers, apiKey, getOutput, - options + options, + onAssistantMessageStart, + onAssistantToken, + onAssistantMessageEnd } = params + let shouldEmitMessageStart = true const model = options?.model ?? 'gpt-4o' const maxIterations = options?.maxIterations ?? 20 @@ -124,8 +131,17 @@ export async function runEval( } = { setToolStatus: () => {}, removeToolStatus: () => {}, - onNewToken: () => {}, - onMessageEnd: () => {} + onNewToken: (token: string) => { + if (shouldEmitMessageStart) { + onAssistantMessageStart?.() + shouldEmitMessageStart = false + } + onAssistantToken?.(token) + }, + onMessageEnd: () => { + onAssistantMessageEnd?.() + shouldEmitMessageStart = true + } } const abortController = new AbortController() diff --git a/ai_evals/adapters/frontend/progress.ts b/ai_evals/adapters/frontend/progress.ts index ddf3c3a827..5ec414c9fd 100644 --- a/ai_evals/adapters/frontend/progress.ts +++ b/ai_evals/adapters/frontend/progress.ts @@ -30,6 +30,34 @@ export type FrontendBenchmarkProgressEvent = judgeScore: number | null error: string | null } + | { + type: 'assistant-message-start' + surface: FrontendBenchmarkProgressSurface + caseId: string + caseNumber: number + totalCases: number + attempt: number + runs: number + } + | { + type: 'assistant-chunk' + surface: FrontendBenchmarkProgressSurface + caseId: string + caseNumber: number + totalCases: number + attempt: number + runs: number + chunk: string + } + | { + type: 'assistant-message-end' + surface: FrontendBenchmarkProgressSurface + caseId: string + caseNumber: number + totalCases: number + attempt: number + runs: number + } export const FRONTEND_BENCHMARK_PROGRESS_PREFIX = 'WMILL_FRONTEND_AI_EVAL_PROGRESS ' @@ -77,6 +105,10 @@ export function formatFrontendBenchmarkProgressEvent( } return parts.join(' | ') } + case 'assistant-message-start': + case 'assistant-chunk': + case 'assistant-message-end': + return '' } } diff --git a/ai_evals/adapters/frontend/runtime.ts b/ai_evals/adapters/frontend/runtime.ts index 386278afde..d0282b36ae 100644 --- a/ai_evals/adapters/frontend/runtime.ts +++ b/ai_evals/adapters/frontend/runtime.ts @@ -19,6 +19,7 @@ export async function runFrontendBenchmarkAdapter(input: { mode: FrontendMode caseIds: string[] runs: number + verbose?: boolean }): Promise { const tempDir = await mkdtemp(path.join(tmpdir(), 'wmill-frontend-benchmark-')) const outputPath = path.join(tempDir, 'result.json') @@ -43,7 +44,8 @@ export async function runFrontendBenchmarkAdapter(input: { WMILL_FRONTEND_AI_EVAL_MODE: input.mode, WMILL_FRONTEND_AI_EVAL_CASE_IDS: JSON.stringify(input.caseIds), WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs), - WMILL_FRONTEND_AI_EVAL_PROGRESS: '1' + WMILL_FRONTEND_AI_EVAL_PROGRESS: '1', + WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? '1' : '0' } } ) @@ -74,6 +76,7 @@ async function runVitestBenchmark( let stdout = '' let stderr = '' let stderrLineBuffer = '' + let assistantStreamOpen = false child.stdout?.setEncoding('utf8') child.stdout?.on('data', (chunk: string) => { @@ -83,21 +86,33 @@ async function runVitestBenchmark( child.stderr?.setEncoding('utf8') child.stderr?.on('data', (chunk: string) => { stderrLineBuffer += chunk - const { remainder, passthrough } = drainProgressLines(stderrLineBuffer) + const { remainder, passthrough, nextAssistantStreamOpen } = drainProgressLines( + stderrLineBuffer, + assistantStreamOpen + ) stderrLineBuffer = remainder stderr += passthrough + assistantStreamOpen = nextAssistantStreamOpen }) await new Promise((resolve, reject) => { child.once('error', reject) child.once('close', (code) => { if (stderrLineBuffer.length > 0) { - const { remainder, passthrough } = drainProgressLines(`${stderrLineBuffer}\n`) + const { + remainder, + passthrough, + nextAssistantStreamOpen + } = drainProgressLines(`${stderrLineBuffer}\n`, assistantStreamOpen) stderrLineBuffer = remainder stderr += passthrough + assistantStreamOpen = nextAssistantStreamOpen } if (code === 0) { + if (assistantStreamOpen) { + process.stderr.write('\n') + } resolve() return } @@ -111,14 +126,24 @@ async function runVitestBenchmark( function drainProgressLines(buffer: string): { remainder: string passthrough: string + nextAssistantStreamOpen: boolean +} +function drainProgressLines( + buffer: string, + initialAssistantStreamOpen: boolean +): { + remainder: string + passthrough: string + nextAssistantStreamOpen: boolean } { let remainder = buffer let passthrough = '' + let assistantStreamOpen = initialAssistantStreamOpen while (true) { const newlineIndex = remainder.indexOf('\n') if (newlineIndex === -1) { - return { remainder, passthrough } + return { remainder, passthrough, nextAssistantStreamOpen: assistantStreamOpen } } const line = remainder.slice(0, newlineIndex).replace(/\r$/, '') @@ -126,6 +151,34 @@ function drainProgressLines(buffer: string): { const progressEvent = parseFrontendBenchmarkProgressLine(line) if (progressEvent) { + if (progressEvent.type === 'assistant-message-start') { + if (assistantStreamOpen) { + process.stderr.write('\n') + } + process.stderr.write( + `${formatCasePrefix(progressEvent.caseNumber, progressEvent.totalCases)} ${progressEvent.caseId} attempt ${progressEvent.attempt}/${progressEvent.runs} assistant:\n` + ) + assistantStreamOpen = true + continue + } + + if (progressEvent.type === 'assistant-chunk') { + process.stderr.write(progressEvent.chunk) + continue + } + + if (progressEvent.type === 'assistant-message-end') { + if (assistantStreamOpen) { + process.stderr.write('\n') + } + assistantStreamOpen = false + continue + } + + if (assistantStreamOpen) { + process.stderr.write('\n') + assistantStreamOpen = false + } process.stderr.write(`${formatFrontendBenchmarkProgressEvent(progressEvent)}\n`) continue } @@ -139,6 +192,10 @@ function drainProgressLines(buffer: string): { } } +function formatCasePrefix(caseNumber: number, totalCases: number): string { + return `[${caseNumber}/${totalCases}]` +} + function shouldSuppressFrontendStderrLine(line: string): boolean { return ( line.startsWith('[baseline-browser-mapping] ') || diff --git a/ai_evals/cli/index.ts b/ai_evals/cli/index.ts index 51a7b12c87..cf55612490 100644 --- a/ai_evals/cli/index.ts +++ b/ai_evals/cli/index.ts @@ -23,6 +23,7 @@ async function main() { " bun run cli -- cases", " bun run cli -- cases flow", " bun run cli -- run flow", + " bun run cli -- run flow flow-test0-sum-two-numbers --verbose", " bun run cli -- run flow flow-test5-simple-modification --runs 3", " bun run cli -- run cli bun-hello-script", ].join("\n") @@ -43,6 +44,7 @@ async function main() { .argument("[caseIds...]", "specific case ids to run") .option("--runs ", "number of attempts per case", parsePositiveInteger, 1) .option("--output ", "write the result JSON to this path") + .option("--verbose", "stream assistant output during frontend runs") .action( async ( mode: EvalMode, @@ -50,6 +52,7 @@ async function main() { options: { runs: number; output?: string; + verbose?: boolean; } ) => { await handleRun({ @@ -57,6 +60,7 @@ async function main() { caseIds, runs: options.runs, outputPath: options.output, + verbose: options.verbose ?? false, }); } ); @@ -82,6 +86,7 @@ async function handleRun(input: { caseIds: string[]; runs: number; outputPath?: string; + verbose: boolean; }) { const selectedCases = await loadSelectedCases(input.mode, input.caseIds); process.stderr.write(`Starting ${input.mode} benchmark...\n`); @@ -93,6 +98,7 @@ async function handleRun(input: { mode: input.mode, caseIds: input.caseIds, runs: input.runs, + verbose: input.verbose, }); const resultPath = await writeRunResult(result, input.outputPath); diff --git a/ai_evals/core/runSuite.ts b/ai_evals/core/runSuite.ts index 4e83969bbb..28462c1ac8 100644 --- a/ai_evals/core/runSuite.ts +++ b/ai_evals/core/runSuite.ts @@ -14,10 +14,12 @@ export async function runSuite(input: { runs: number; runModel: string | null; judgeModel?: string | null; + concurrency?: number; + verbose?: boolean; onProgress?: (event: FrontendBenchmarkProgressEvent) => void; }): Promise { const judgeModel = input.judgeModel ?? DEFAULT_JUDGE_MODEL; - const concurrency = Math.max(1, input.modeRunner.concurrency); + const concurrency = Math.max(1, input.concurrency ?? input.modeRunner.concurrency); const results = new Array(input.cases.length); let cursor = 0; @@ -51,6 +53,7 @@ export async function runSuite(input: { judgeThreshold: input.modeRunner.judgeThreshold ?? 80, modeRunner: input.modeRunner, totalCases: input.cases.length, + verbose: input.verbose ?? false, onProgress: input.onProgress, }), }; @@ -72,6 +75,7 @@ async function runCaseAttempts(input: { judgeThreshold: number; modeRunner: ModeRunner; totalCases: number; + verbose: boolean; onProgress?: (event: FrontendBenchmarkProgressEvent) => void; }): Promise { const attempts: BenchmarkAttemptResult[] = []; @@ -92,7 +96,51 @@ async function runCaseAttempts(input: { const expected = await input.modeRunner.loadExpected(input.evalCase.expectedPath); try { - const run = await input.modeRunner.run(input.evalCase.prompt, initial); + const run = await input.modeRunner.run(input.evalCase.prompt, initial, { + caseId: input.evalCase.id, + caseNumber: input.caseIndex + 1, + totalCases: input.totalCases, + attempt, + runs: input.runs, + verbose: input.verbose, + onAssistantMessageStart: input.verbose + ? () => + input.onProgress?.({ + type: "assistant-message-start", + surface: input.modeRunner.mode as Exclude, + caseId: input.evalCase.id, + caseNumber: input.caseIndex + 1, + totalCases: input.totalCases, + attempt, + runs: input.runs, + }) + : undefined, + onAssistantChunk: input.verbose + ? (chunk: string) => + input.onProgress?.({ + type: "assistant-chunk", + surface: input.modeRunner.mode as Exclude, + caseId: input.evalCase.id, + caseNumber: input.caseIndex + 1, + totalCases: input.totalCases, + attempt, + runs: input.runs, + chunk, + }) + : undefined, + onAssistantMessageEnd: input.verbose + ? () => + input.onProgress?.({ + type: "assistant-message-end", + surface: input.modeRunner.mode as Exclude, + caseId: input.evalCase.id, + caseNumber: input.caseIndex + 1, + totalCases: input.totalCases, + attempt, + runs: input.runs, + }) + : undefined, + }); const checks: BenchmarkCheck[] = [ buildCheck("run succeeded", run.success, run.error), ...input.modeRunner.validate({ diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 9800c57da7..5c7f23975d 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -32,13 +32,29 @@ export interface ModeRunOutput { skillsInvoked: string[]; } +export interface ModeRunContext { + caseId: string; + caseNumber: number; + totalCases: number; + attempt: number; + runs: number; + verbose: boolean; + onAssistantMessageStart?: () => void; + onAssistantChunk?: (chunk: string) => void; + onAssistantMessageEnd?: () => void; +} + export interface ModeRunner { mode: EvalMode; concurrency: number; judgeThreshold?: number; loadInitial(path?: string): Promise; loadExpected(path?: string): Promise; - run(prompt: string, initial: TInitial | undefined): Promise>; + run( + prompt: string, + initial: TInitial | undefined, + context: ModeRunContext + ): Promise>; validate(input: { prompt: string; initial: TInitial | undefined; @@ -115,4 +131,32 @@ export type FrontendBenchmarkProgressEvent = durationMs: number; judgeScore: number | null; error: string | null; + } + | { + type: "assistant-message-start"; + surface: Exclude; + caseId: string; + caseNumber: number; + totalCases: number; + attempt: number; + runs: number; + } + | { + type: "assistant-chunk"; + surface: Exclude; + caseId: string; + caseNumber: number; + totalCases: number; + attempt: number; + runs: number; + chunk: string; + } + | { + type: "assistant-message-end"; + surface: Exclude; + caseId: string; + caseNumber: number; + totalCases: number; + attempt: number; + runs: number; }; diff --git a/ai_evals/modes/app.ts b/ai_evals/modes/app.ts index 418fd1c84a..bfcd74afa7 100644 --- a/ai_evals/modes/app.ts +++ b/ai_evals/modes/app.ts @@ -16,12 +16,13 @@ export function createAppModeRunner(): ModeRunner(path) : undefined; }, - async run(prompt, initial) { + async run(prompt, initial, context) { const result = await runFlowEval(prompt, getFrontendApiKey(), { initialFlow: initial, provider: FRONTEND_PROVIDER, model: FRONTEND_MODEL, + runContext: context, }); return { diff --git a/ai_evals/modes/script.ts b/ai_evals/modes/script.ts index 200674b6b3..7792a3d279 100644 --- a/ai_evals/modes/script.ts +++ b/ai_evals/modes/script.ts @@ -16,7 +16,7 @@ export function createScriptModeRunner(): ModeRunner(path) : undefined; }, - async run(prompt, initial) { + async run(prompt, initial, context) { if (!initial) { throw new Error("Script evals require an initial script fixture"); } @@ -25,6 +25,7 @@ export function createScriptModeRunner(): ModeRunner