starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-23 00:03:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: stream verbose frontend benchmark output

Browse Source 
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
centdix
2026-04-08 14:27:19 +02:00
co-authored by Claude Opus 4.5
parent 550bcbcff3
commit d0fd8c40ae
14 changed files with 238 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ai_evals/adapters/frontend/benchmarkRunner.ts
+3
View File
@@ -16,6 +16,7 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
const caseIds = parseOptionalJsonStringArray(process.env.WMILL_FRONTEND_AI_EVAL_CASE_IDS);
const runs = parsePositiveInteger(process.env.WMILL_FRONTEND_AI_EVAL_RUNS, "WMILL_FRONTEND_AI_EVAL_RUNS");
const emitProgress = process.env.WMILL_FRONTEND_AI_EVAL_PROGRESS === "1";
const verbose = process.env.WMILL_FRONTEND_AI_EVAL_VERBOSE === "1";
const selectedCases = await loadSelectedCases(mode, caseIds);
const modeRunner = getModeRunner(mode);
@@ -25,6 +26,8 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
runs,
runModel: getFrontendRunModelLabel(),
judgeModel: DEFAULT_JUDGE_MODEL,
concurrency: verbose ? 1 : undefined,
verbose,
onProgress: emitProgress ? (event) => emitFrontendBenchmarkProgress(event) : undefined,
});
ai_evals/adapters/frontend/core/app/appEvalRunner.ts
+5
View File
@@ -15,6 +15,7 @@ import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/com
import { createAppFileHelpers } from './fileHelpers'
import { runEval } from '../shared'
import type { AIProvider } from '$lib/gen/types.gen'
import type { ModeRunContext } from '../../../../core/types'
export interface AppEvalResult {
success: boolean
@@ -32,6 +33,7 @@ export interface AppEvalOptions {
maxIterations?: number
provider?: AIProvider
workspaceRoot?: string
runContext?: ModeRunContext
}
export async function runAppEval(
@@ -62,6 +64,9 @@ export async function runAppEval(
helpers,
apiKey,
getOutput: getFiles,
onAssistantMessageStart: options?.runContext?.onAssistantMessageStart,
onAssistantToken: options?.runContext?.onAssistantChunk,
onAssistantMessageEnd: options?.runContext?.onAssistantMessageEnd,
options: {
maxIterations: options?.maxIterations,
model,
ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts
+5
View File
@@ -13,6 +13,7 @@ import {
import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared'
import { createFlowFileHelpers } from './fileHelpers'
import { runEval } from '../shared'
import type { ModeRunContext } from '../../../../core/types'
export interface FlowFixture {
value?: {
@@ -36,6 +37,7 @@ export interface FlowEvalOptions {
maxIterations?: number
provider?: AIProvider
workspaceRoot?: string
runContext?: ModeRunContext
}
export async function runFlowEval(
@@ -66,6 +68,9 @@ export async function runFlowEval(
helpers,
apiKey,
getOutput: getFlow,
onAssistantMessageStart: options?.runContext?.onAssistantMessageStart,
onAssistantToken: options?.runContext?.onAssistantChunk,
onAssistantMessageEnd: options?.runContext?.onAssistantMessageEnd,
options: {
maxIterations: options?.maxIterations,
model,
ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts
+5
View File
@@ -12,6 +12,7 @@ import {
import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared'
import { createScriptFileHelpers, type ScriptEvalState } from './fileHelpers'
import { runEval } from '../shared'
import type { ModeRunContext } from '../../../../core/types'
export interface ScriptEvalResult {
success: boolean
@@ -28,6 +29,7 @@ export interface ScriptEvalOptions {
maxIterations?: number
provider?: AIProvider
workspaceRoot?: string
runContext?: ModeRunContext
}
function resolveModelProvider(
@@ -79,6 +81,9 @@ export async function runScriptEval(
helpers,
apiKey,
getOutput: getScript,
onAssistantMessageStart: options.runContext?.onAssistantMessageStart,
onAssistantToken: options.runContext?.onAssistantChunk,
onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd,
options: {
maxIterations: options.maxIterations,
model,
ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts
+19 -3
View File
@@ -34,6 +34,9 @@ export interface RunEvalParams<THelpers, TOutput> {
getOutput: () => TOutput
/** Optional configuration */
options?: EvalRunnerOptions
onAssistantMessageStart?: () => void
onAssistantToken?: (token: string) => void
onAssistantMessageEnd?: () => void
}
/**
@@ -79,8 +82,12 @@ export async function runEval<THelpers, TOutput>(
helpers,
apiKey,
getOutput,
options
options,
onAssistantMessageStart,
onAssistantToken,
onAssistantMessageEnd
} = params
let shouldEmitMessageStart = true
const model = options?.model ?? 'gpt-4o'
const maxIterations = options?.maxIterations ?? 20
@@ -124,8 +131,17 @@ export async function runEval<THelpers, TOutput>(
} = {
setToolStatus: () => {},
removeToolStatus: () => {},
onNewToken: () => {},
onMessageEnd: () => {}
onNewToken: (token: string) => {
if (shouldEmitMessageStart) {
onAssistantMessageStart?.()
shouldEmitMessageStart = false
}
onAssistantToken?.(token)
},
onMessageEnd: () => {
onAssistantMessageEnd?.()
shouldEmitMessageStart = true
}
}
const abortController = new AbortController()
ai_evals/adapters/frontend/progress.ts
+32
View File
@@ -30,6 +30,34 @@ export type FrontendBenchmarkProgressEvent =
judgeScore: number | null
error: string | null
}
| {
type: 'assistant-message-start'
surface: FrontendBenchmarkProgressSurface
caseId: string
caseNumber: number
totalCases: number
attempt: number
runs: number
}
| {
type: 'assistant-chunk'
surface: FrontendBenchmarkProgressSurface
caseId: string
caseNumber: number
totalCases: number
attempt: number
runs: number
chunk: string
}
| {
type: 'assistant-message-end'
surface: FrontendBenchmarkProgressSurface
caseId: string
caseNumber: number
totalCases: number
attempt: number
runs: number
}
export const FRONTEND_BENCHMARK_PROGRESS_PREFIX = 'WMILL_FRONTEND_AI_EVAL_PROGRESS '
@@ -77,6 +105,10 @@ export function formatFrontendBenchmarkProgressEvent(
}
return parts.join(' | ')
}
case 'assistant-message-start':
case 'assistant-chunk':
case 'assistant-message-end':
return ''
}
}
ai_evals/adapters/frontend/runtime.ts
+61 -4
View File
@@ -19,6 +19,7 @@ export async function runFrontendBenchmarkAdapter(input: {
mode: FrontendMode
caseIds: string[]
runs: number
verbose?: boolean
}): Promise<BenchmarkRunResult> {
const tempDir = await mkdtemp(path.join(tmpdir(), 'wmill-frontend-benchmark-'))
const outputPath = path.join(tempDir, 'result.json')
@@ -43,7 +44,8 @@ export async function runFrontendBenchmarkAdapter(input: {
WMILL_FRONTEND_AI_EVAL_MODE: input.mode,
WMILL_FRONTEND_AI_EVAL_CASE_IDS: JSON.stringify(input.caseIds),
WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs),
WMILL_FRONTEND_AI_EVAL_PROGRESS: '1'
WMILL_FRONTEND_AI_EVAL_PROGRESS: '1',
WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? '1' : '0'
}
}
)
@@ -74,6 +76,7 @@ async function runVitestBenchmark(
let stdout = ''
let stderr = ''
let stderrLineBuffer = ''
let assistantStreamOpen = false
child.stdout?.setEncoding('utf8')
child.stdout?.on('data', (chunk: string) => {
@@ -83,21 +86,33 @@ async function runVitestBenchmark(
child.stderr?.setEncoding('utf8')
child.stderr?.on('data', (chunk: string) => {
stderrLineBuffer += chunk
const { remainder, passthrough } = drainProgressLines(stderrLineBuffer)
const { remainder, passthrough, nextAssistantStreamOpen } = drainProgressLines(
stderrLineBuffer,
assistantStreamOpen
)
stderrLineBuffer = remainder
stderr += passthrough
assistantStreamOpen = nextAssistantStreamOpen
})
await new Promise<void>((resolve, reject) => {
child.once('error', reject)
child.once('close', (code) => {
if (stderrLineBuffer.length > 0) {
const { remainder, passthrough } = drainProgressLines(`${stderrLineBuffer}\n`)
const {
remainder,
passthrough,
nextAssistantStreamOpen
} = drainProgressLines(`${stderrLineBuffer}\n`, assistantStreamOpen)
stderrLineBuffer = remainder
stderr += passthrough
assistantStreamOpen = nextAssistantStreamOpen
}
if (code === 0) {
if (assistantStreamOpen) {
process.stderr.write('\n')
}
resolve()
return
}
@@ -111,14 +126,24 @@ async function runVitestBenchmark(
function drainProgressLines(buffer: string): {
remainder: string
passthrough: string
nextAssistantStreamOpen: boolean
}
function drainProgressLines(
buffer: string,
initialAssistantStreamOpen: boolean
): {
remainder: string
passthrough: string
nextAssistantStreamOpen: boolean
} {
let remainder = buffer
let passthrough = ''
let assistantStreamOpen = initialAssistantStreamOpen
while (true) {
const newlineIndex = remainder.indexOf('\n')
if (newlineIndex === -1) {
return { remainder, passthrough }
return { remainder, passthrough, nextAssistantStreamOpen: assistantStreamOpen }
}
const line = remainder.slice(0, newlineIndex).replace(/\r$/, '')
@@ -126,6 +151,34 @@ function drainProgressLines(buffer: string): {
const progressEvent = parseFrontendBenchmarkProgressLine(line)
if (progressEvent) {
if (progressEvent.type === 'assistant-message-start') {
if (assistantStreamOpen) {
process.stderr.write('\n')
}
process.stderr.write(
`${formatCasePrefix(progressEvent.caseNumber, progressEvent.totalCases)} ${progressEvent.caseId} attempt ${progressEvent.attempt}/${progressEvent.runs} assistant:\n`
)
assistantStreamOpen = true
continue
}
if (progressEvent.type === 'assistant-chunk') {
process.stderr.write(progressEvent.chunk)
continue
}
if (progressEvent.type === 'assistant-message-end') {
if (assistantStreamOpen) {
process.stderr.write('\n')
}
assistantStreamOpen = false
continue
}
if (assistantStreamOpen) {
process.stderr.write('\n')
assistantStreamOpen = false
}
process.stderr.write(`${formatFrontendBenchmarkProgressEvent(progressEvent)}\n`)
continue
}
@@ -139,6 +192,10 @@ function drainProgressLines(buffer: string): {
}
}
function formatCasePrefix(caseNumber: number, totalCases: number): string {
return `[${caseNumber}/${totalCases}]`
}
function shouldSuppressFrontendStderrLine(line: string): boolean {
return (
line.startsWith('[baseline-browser-mapping] ') ||
ai_evals/cli/index.ts
+6
View File
@@ -23,6 +23,7 @@ async function main() {
" bun run cli -- cases",
" bun run cli -- cases flow",
" bun run cli -- run flow",
" bun run cli -- run flow flow-test0-sum-two-numbers --verbose",
" bun run cli -- run flow flow-test5-simple-modification --runs 3",
" bun run cli -- run cli bun-hello-script",
].join("\n")
@@ -43,6 +44,7 @@ async function main() {
.argument("[caseIds...]", "specific case ids to run")
.option("--runs <n>", "number of attempts per case", parsePositiveInteger, 1)
.option("--output <path>", "write the result JSON to this path")
.option("--verbose", "stream assistant output during frontend runs")
.action(
async (
mode: EvalMode,
@@ -50,6 +52,7 @@ async function main() {
options: {
runs: number;
output?: string;
verbose?: boolean;
}
) => {
await handleRun({
@@ -57,6 +60,7 @@ async function main() {
caseIds,
runs: options.runs,
outputPath: options.output,
verbose: options.verbose ?? false,
});
}
);
@@ -82,6 +86,7 @@ async function handleRun(input: {
caseIds: string[];
runs: number;
outputPath?: string;
verbose: boolean;
}) {
const selectedCases = await loadSelectedCases(input.mode, input.caseIds);
process.stderr.write(`Starting ${input.mode} benchmark...\n`);
@@ -93,6 +98,7 @@ async function handleRun(input: {
mode: input.mode,
caseIds: input.caseIds,
runs: input.runs,
verbose: input.verbose,
});
const resultPath = await writeRunResult(result, input.outputPath);
ai_evals/core/runSuite.ts
+50 -2
View File
@@ -14,10 +14,12 @@ export async function runSuite<TInitial, TExpected, TActual>(input: {
runs: number;
runModel: string | null;
judgeModel?: string | null;
concurrency?: number;
verbose?: boolean;
onProgress?: (event: FrontendBenchmarkProgressEvent) => void;
}): Promise<BenchmarkCaseResult[]> {
const judgeModel = input.judgeModel ?? DEFAULT_JUDGE_MODEL;
const concurrency = Math.max(1, input.modeRunner.concurrency);
const concurrency = Math.max(1, input.concurrency ?? input.modeRunner.concurrency);
const results = new Array<BenchmarkCaseResult>(input.cases.length);
let cursor = 0;
@@ -51,6 +53,7 @@ export async function runSuite<TInitial, TExpected, TActual>(input: {
judgeThreshold: input.modeRunner.judgeThreshold ?? 80,
modeRunner: input.modeRunner,
totalCases: input.cases.length,
verbose: input.verbose ?? false,
onProgress: input.onProgress,
}),
};
@@ -72,6 +75,7 @@ async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
judgeThreshold: number;
modeRunner: ModeRunner<TInitial, TExpected, TActual>;
totalCases: number;
verbose: boolean;
onProgress?: (event: FrontendBenchmarkProgressEvent) => void;
}): Promise<BenchmarkAttemptResult[]> {
const attempts: BenchmarkAttemptResult[] = [];
@@ -92,7 +96,51 @@ async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
const expected = await input.modeRunner.loadExpected(input.evalCase.expectedPath);
try {
const run = await input.modeRunner.run(input.evalCase.prompt, initial);
const run = await input.modeRunner.run(input.evalCase.prompt, initial, {
caseId: input.evalCase.id,
caseNumber: input.caseIndex + 1,
totalCases: input.totalCases,
attempt,
runs: input.runs,
verbose: input.verbose,
onAssistantMessageStart: input.verbose
? () =>
input.onProgress?.({
type: "assistant-message-start",
surface: input.modeRunner.mode as Exclude<typeof input.modeRunner.mode, "cli">,
caseId: input.evalCase.id,
caseNumber: input.caseIndex + 1,
totalCases: input.totalCases,
attempt,
runs: input.runs,
})
: undefined,
onAssistantChunk: input.verbose
? (chunk: string) =>
input.onProgress?.({
type: "assistant-chunk",
surface: input.modeRunner.mode as Exclude<typeof input.modeRunner.mode, "cli">,
caseId: input.evalCase.id,
caseNumber: input.caseIndex + 1,
totalCases: input.totalCases,
attempt,
runs: input.runs,
chunk,
})
: undefined,
onAssistantMessageEnd: input.verbose
? () =>
input.onProgress?.({
type: "assistant-message-end",
surface: input.modeRunner.mode as Exclude<typeof input.modeRunner.mode, "cli">,
caseId: input.evalCase.id,
caseNumber: input.caseIndex + 1,
totalCases: input.totalCases,
attempt,
runs: input.runs,
})
: undefined,
});
const checks: BenchmarkCheck[] = [
buildCheck("run succeeded", run.success, run.error),
...input.modeRunner.validate({
ai_evals/core/types.ts
+45 -1
View File
@@ -32,13 +32,29 @@ export interface ModeRunOutput<TActual> {
skillsInvoked: string[];
}
export interface ModeRunContext {
caseId: string;
caseNumber: number;
totalCases: number;
attempt: number;
runs: number;
verbose: boolean;
onAssistantMessageStart?: () => void;
onAssistantChunk?: (chunk: string) => void;
onAssistantMessageEnd?: () => void;
}
export interface ModeRunner<TInitial, TExpected, TActual> {
mode: EvalMode;
concurrency: number;
judgeThreshold?: number;
loadInitial(path?: string): Promise<TInitial | undefined>;
loadExpected(path?: string): Promise<TExpected | undefined>;
run(prompt: string, initial: TInitial | undefined): Promise<ModeRunOutput<TActual>>;
run(
prompt: string,
initial: TInitial | undefined,
context: ModeRunContext
): Promise<ModeRunOutput<TActual>>;
validate(input: {
prompt: string;
initial: TInitial | undefined;
@@ -115,4 +131,32 @@ export type FrontendBenchmarkProgressEvent =
durationMs: number;
judgeScore: number | null;
error: string | null;
}
| {
type: "assistant-message-start";
surface: Exclude<EvalMode, "cli">;
caseId: string;
caseNumber: number;
totalCases: number;
attempt: number;
runs: number;
}
| {
type: "assistant-chunk";
surface: Exclude<EvalMode, "cli">;
caseId: string;
caseNumber: number;
totalCases: number;
attempt: number;
runs: number;
chunk: string;
}
| {
type: "assistant-message-end";
surface: Exclude<EvalMode, "cli">;
caseId: string;
caseNumber: number;
totalCases: number;
attempt: number;
runs: number;
};
ai_evals/modes/app.ts
+2 -1
View File
@@ -16,12 +16,13 @@ export function createAppModeRunner(): ModeRunner<AppFilesState, AppFilesState,
async loadExpected(path) {
return path ? (await loadAppFixture(path)) : undefined;
},
async run(prompt, initial) {
async run(prompt, initial, context) {
const result = await runAppEval(prompt, getFrontendApiKey(), {
initialFrontend: initial?.frontend,
initialBackend: initial?.backend as AppFiles["backend"] | undefined,
provider: FRONTEND_PROVIDER,
model: FRONTEND_MODEL,
runContext: context,
});
return {
ai_evals/modes/cli.ts
+1 -1
View File
@@ -49,7 +49,7 @@ export function createCliModeRunner(): ModeRunner<CliWorkspaceFixture, CliWorksp
}
: undefined;
},
async run(prompt, initial) {
async run(prompt, initial, _context) {
const workspaceDir = await mkdtemp(join(tmpdir(), "wmill-cli-benchmark-"));
try {
ai_evals/modes/flow.ts
+2 -1
View File
@@ -15,11 +15,12 @@ export function createFlowModeRunner(): ModeRunner<FlowState, FlowState, FlowSta
async loadExpected(path) {
return path ? await readJsonFile<FlowState>(path) : undefined;
},
async run(prompt, initial) {
async run(prompt, initial, context) {
const result = await runFlowEval(prompt, getFrontendApiKey(), {
initialFlow: initial,
provider: FRONTEND_PROVIDER,
model: FRONTEND_MODEL,
runContext: context,
});
return {
ai_evals/modes/script.ts
+2 -1
View File
@@ -16,7 +16,7 @@ export function createScriptModeRunner(): ModeRunner<ScriptEvalState, ScriptEval
async loadExpected(path) {
return path ? await readJsonFile<ScriptEvalState>(path) : undefined;
},
async run(prompt, initial) {
async run(prompt, initial, context) {
if (!initial) {
throw new Error("Script evals require an initial script fixture");
}
@@ -25,6 +25,7 @@ export function createScriptModeRunner(): ModeRunner<ScriptEvalState, ScriptEval
initialScript: initial,
provider: FRONTEND_PROVIDER,
model: FRONTEND_MODEL,
runContext: context,
});
return {