From fddd8e288fc0fd7af3b1df1ddd4476fb57694ed3 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:03:44 +0200 Subject: [PATCH] fix: add proxy eval coverage for gemini schemas (#8897) * feat: add proxy transport for ai evals Co-Authored-By: Claude Opus 4.5 * fix: strip propertyNames for gemini schemas Co-Authored-By: Claude Opus 4.5 * fix: require explicit eval transport Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: Claude Opus 4.5 --- ai_evals/README.md | 13 +- ai_evals/adapters/frontend/benchmarkRunner.ts | 58 ++- .../frontend/core/app/appEvalRunner.ts | 171 ++++---- .../frontend/core/app/appFixtureLoader.ts | 241 ++++++----- .../frontend/core/flow/flowEvalRunner.ts | 193 +++++---- .../frontend/core/script/scriptEvalRunner.ts | 193 +++++---- .../frontend/core/shared/baseEvalRunner.ts | 401 ++++++++++-------- .../core/shared/providerConfig.test.ts | 25 +- .../frontend/core/shared/providerConfig.ts | 88 +++- .../adapters/frontend/core/shared/types.ts | 46 +- ai_evals/adapters/frontend/runtime.ts | 359 ++++++++-------- ai_evals/adapters/frontend/windmillBackend.ts | 184 ++++++++ ai_evals/cli/index.ts | 100 ++++- ai_evals/core/appDiagnostics.ts | 207 ++++++--- ai_evals/core/backendValidation.ts | 82 ++-- ai_evals/core/frontendTransport.test.ts | 64 +++ ai_evals/core/frontendTransport.ts | 49 +++ ai_evals/core/results.ts | 132 ++++-- ai_evals/core/types.ts | 4 +- ai_evals/core/windmillBackendSettings.ts | 67 +++ ai_evals/modes/app.ts | 37 +- ai_evals/modes/flow.ts | 122 +++--- ai_evals/modes/script.ts | 121 +++--- backend/windmill-common/src/ai_google.rs | 24 +- 24 files changed, 1926 insertions(+), 1055 deletions(-) create mode 100644 ai_evals/adapters/frontend/windmillBackend.ts create mode 100644 ai_evals/core/frontendTransport.test.ts create mode 100644 ai_evals/core/frontendTransport.ts create mode 100644 ai_evals/core/windmillBackendSettings.ts diff --git a/ai_evals/README.md b/ai_evals/README.md index 66db9dd42f..ccf58b3974 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -55,6 +55,7 @@ bun run cli -- run flow flow-test4-order-processing-loop --model opus bun run cli -- run flow flow-test0-sum-two-numbers --models haiku,opus,4o bun run cli -- run flow flow-test0-sum-two-numbers --runs 3 --verbose bun run cli -- run flow --record +GEMINI_API_KEY=... bun run cli -- run app app-test1-counter-create --model gemini-pro --transport proxy WMILL_AI_EVAL_BACKEND_URL=http://127.0.0.1:8000 bun run cli -- run flow --backend-validation preview bun run cli -- run cli bun-hello-script ``` @@ -71,6 +72,7 @@ Public CLI surface: - `--output `: custom result JSON path - `--model `: choose the model under test - `--models `: run the same cases sequentially against several model aliases +- `--transport `: frontend request transport (`direct` by default, `proxy` to exercise `/api/w/{workspace}/ai/proxy`) - `--verbose`: stream assistant output for frontend runs - `--record`: append a compact tracked summary line to `ai_evals/history/.jsonl` for full-suite runs only - `--backend-validation `: optional backend smoke validation (`off` or `preview`) for `script` and `flow` evals @@ -155,6 +157,15 @@ Supported backend validation env vars: - `WMILL_AI_EVAL_KEEP_WORKSPACES=1` - `WMILL_AI_EVAL_WORKSPACE_PREFIX=ai-evals` +Frontend proxy transport uses the same backend auth/workspace env vars. + +When `--transport proxy` is set: + +- `ai_evals` creates or reuses a backend workspace +- it upserts a provider resource under `f/evals/ai/` +- frontend requests go through `/api/w/{workspace}/ai/proxy` +- result JSON and history records include `transport` so direct vs proxy runs stay distinguishable + ## Results And Artifacts Every run writes: @@ -171,7 +182,7 @@ If `--record` is used, the CLI also appends one compact JSON line to: Each recorded line contains: -- run metadata (`createdAt`, `gitSha`, `mode`, `runModel`, `judgeModel`) +- run metadata (`createdAt`, `gitSha`, `mode`, `runModel`, `transport`, `judgeModel`) - suite totals (`caseCount`, `attemptCount`, `passedAttempts`, `passRate`, `averageDurationMs`, `averageJudgeScore`) - average token usage (`averageTokenUsagePerAttempt`) - per-case metrics under `cases[]` (`averageDurationMs`, `averageJudgeScore`, `averageTokenUsagePerAttempt`, pass rate) diff --git a/ai_evals/adapters/frontend/benchmarkRunner.ts b/ai_evals/adapters/frontend/benchmarkRunner.ts index 6330211f98..474d434803 100644 --- a/ai_evals/adapters/frontend/benchmarkRunner.ts +++ b/ai_evals/adapters/frontend/benchmarkRunner.ts @@ -1,5 +1,6 @@ import { loadSelectedCases } from "../../core/cases"; import { resolveBackendValidationSettings } from "../../core/backendValidation"; +import { resolveFrontendEvalTransportSettings } from "../../core/frontendTransport"; import { formatRunModelLabel, getFrontendEvalModel, @@ -18,18 +19,35 @@ export type FrontendBenchmarkMode = "flow" | "app" | "script"; export async function runFrontendBenchmarkFromEnv(): Promise { const mode = parseMode(process.env.WMILL_FRONTEND_AI_EVAL_MODE); - const caseIds = parseOptionalJsonStringArray(process.env.WMILL_FRONTEND_AI_EVAL_CASE_IDS); - const runs = parsePositiveInteger(process.env.WMILL_FRONTEND_AI_EVAL_RUNS, "WMILL_FRONTEND_AI_EVAL_RUNS"); + const caseIds = parseOptionalJsonStringArray( + process.env.WMILL_FRONTEND_AI_EVAL_CASE_IDS, + ); + const runs = parsePositiveInteger( + process.env.WMILL_FRONTEND_AI_EVAL_RUNS, + "WMILL_FRONTEND_AI_EVAL_RUNS", + ); const emitProgress = process.env.WMILL_FRONTEND_AI_EVAL_PROGRESS === "1"; const verbose = process.env.WMILL_FRONTEND_AI_EVAL_VERBOSE === "1"; - const model = resolveEvalModel(mode, process.env.WMILL_FRONTEND_AI_EVAL_MODEL); + const model = resolveEvalModel( + mode, + process.env.WMILL_FRONTEND_AI_EVAL_MODEL, + ); const backendValidation = resolveBackendValidationSettings({ evalMode: mode, requestedMode: process.env.WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION, }); + const transportSettings = resolveFrontendEvalTransportSettings({ + evalMode: mode, + requestedTransport: process.env.WMILL_FRONTEND_AI_EVAL_TRANSPORT, + }); const selectedCases = await loadSelectedCases(mode, caseIds); - const modeRunner = getModeRunner(mode, getFrontendEvalModel(model), backendValidation); + const modeRunner = getModeRunner( + mode, + getFrontendEvalModel(model), + backendValidation, + transportSettings, + ); const runModel = formatRunModelLabel(mode, model); const caseResults = await runSuite({ modeRunner, @@ -39,13 +57,16 @@ export async function runFrontendBenchmarkFromEnv(): Promise judgeModel: DEFAULT_JUDGE_MODEL, concurrency: verbose ? 1 : undefined, verbose, - onProgress: emitProgress ? (event) => emitFrontendBenchmarkProgress(event) : undefined, + onProgress: emitProgress + ? (event) => emitFrontendBenchmarkProgress(event) + : undefined, }); return buildRunResult({ mode, runs, runModel, + transport: transportSettings.transport, judgeModel: DEFAULT_JUDGE_MODEL, caseResults, }); @@ -54,15 +75,20 @@ export async function runFrontendBenchmarkFromEnv(): Promise function getModeRunner( mode: FrontendBenchmarkMode, model: ReturnType, - backendValidation: ReturnType + backendValidation: ReturnType, + transportSettings: ReturnType, ): ModeRunner { switch (mode) { case "flow": - return createFlowModeRunner(model, backendValidation); + return createFlowModeRunner(model, backendValidation, transportSettings); case "app": - return createAppModeRunner(model); + return createAppModeRunner(model, transportSettings); case "script": - return createScriptModeRunner(model, backendValidation); + return createScriptModeRunner( + model, + backendValidation, + transportSettings, + ); } } @@ -78,13 +104,21 @@ function parseOptionalJsonStringArray(value: string | undefined): string[] { return []; } const parsed = JSON.parse(value) as unknown; - if (!Array.isArray(parsed) || parsed.some((entry) => typeof entry !== "string")) { - throw new Error("WMILL_FRONTEND_AI_EVAL_CASE_IDS must be a JSON string array"); + if ( + !Array.isArray(parsed) || + parsed.some((entry) => typeof entry !== "string") + ) { + throw new Error( + "WMILL_FRONTEND_AI_EVAL_CASE_IDS must be a JSON string array", + ); } return parsed; } -function parsePositiveInteger(value: string | undefined, envName: string): number { +function parsePositiveInteger( + value: string | undefined, + envName: string, +): number { const parsed = Number(value); if (!Number.isInteger(parsed) || parsed <= 0) { throw new Error(`${envName} must be a positive integer`); diff --git a/ai_evals/adapters/frontend/core/app/appEvalRunner.ts b/ai_evals/adapters/frontend/core/app/appEvalRunner.ts index 9d5d081401..0366e419ad 100644 --- a/ai_evals/adapters/frontend/core/app/appEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/app/appEvalRunner.ts @@ -1,95 +1,106 @@ -import { mkdtemp } from 'fs/promises' -import { tmpdir } from 'os' -import { join } from 'path' +import { mkdtemp } from "fs/promises"; +import { tmpdir } from "os"; +import { join } from "path"; import type { - BackendRunnable, - AppAIChatHelpers -} from '../../../../../frontend/src/lib/components/copilot/chat/app/core' + BackendRunnable, + AppAIChatHelpers, +} from "../../../../../frontend/src/lib/components/copilot/chat/app/core"; import { - getAppTools, - prepareAppSystemMessage, - prepareAppUserMessage -} from '../../../../../frontend/src/lib/components/copilot/chat/app/core' -import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared' -import { createAppFileHelpers } from './fileHelpers' -import { runEval } from '../shared' -import type { AIProvider } from '$lib/gen/types.gen' -import type { ModeRunContext } from '../../../../core/types' -import type { TokenUsage } from '../shared/types' -import type { AppFilesState } from '../../../../core/validators' + getAppTools, + prepareAppSystemMessage, + prepareAppUserMessage, +} from "../../../../../frontend/src/lib/components/copilot/chat/app/core"; +import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; +import { createAppFileHelpers } from "./fileHelpers"; +import { runEval } from "../shared"; +import type { AIProvider } from "$lib/gen/types.gen"; +import type { ModeRunContext } from "../../../../core/types"; +import type { TokenUsage } from "../shared/types"; +import type { AppFilesState } from "../../../../core/validators"; +import type { FrontendEvalTransport } from "../../../../core/frontendTransport"; +import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; export interface AppEvalResult { - success: boolean - files: AppFilesState - error?: string - assistantMessageCount: number - toolCallCount: number - toolsUsed: string[] - tokenUsage: TokenUsage + success: boolean; + files: AppFilesState; + error?: string; + assistantMessageCount: number; + toolCallCount: number; + toolsUsed: string[]; + tokenUsage: TokenUsage; } export interface AppEvalOptions { - initialFrontend?: Record - initialBackend?: Record - initialDatatables?: AppFilesState['datatables'] - model?: string - maxIterations?: number - provider?: AIProvider - workspaceRoot?: string - runContext?: ModeRunContext + initialFrontend?: Record; + initialBackend?: AppFilesState["backend"]; + initialDatatables?: AppFilesState["datatables"]; + model?: string; + maxIterations?: number; + provider?: AIProvider; + transport?: FrontendEvalTransport; + backend?: WindmillBackendSettings; + workspaceRoot?: string; + runContext?: ModeRunContext; } export async function runAppEval( - userPrompt: string, - apiKey: string, - options?: AppEvalOptions + userPrompt: string, + apiKey: string, + options?: AppEvalOptions, ): Promise { - const workspaceRoot = - options?.workspaceRoot ?? - (await mkdtemp(join(tmpdir(), 'wmill-frontend-app-benchmark-'))) - const { helpers, getEvalState, cleanup } = await createAppFileHelpers( - options?.initialFrontend ?? {}, - options?.initialBackend ?? {}, - options?.initialDatatables ?? [], - workspaceRoot - ) + const workspaceRoot = + options?.workspaceRoot ?? + (await mkdtemp(join(tmpdir(), "wmill-frontend-app-benchmark-"))); + const { helpers, getEvalState, cleanup } = await createAppFileHelpers( + options?.initialFrontend ?? {}, + (options?.initialBackend ?? {}) as Record, + options?.initialDatatables ?? [], + workspaceRoot, + ); - try { - const systemMessage = prepareAppSystemMessage() - const tools = getAppTools() as ProductionTool[] - const model = options?.model ?? 'claude-haiku-4-5-20251001' - const userMessage = prepareAppUserMessage(userPrompt, helpers.getSelectedContext()) + try { + const systemMessage = prepareAppSystemMessage(); + const tools = getAppTools() as ProductionTool[]; + const model = options?.model ?? "claude-haiku-4-5-20251001"; + const userMessage = prepareAppUserMessage( + userPrompt, + helpers.getSelectedContext(), + ); - const rawResult = await runEval({ - userPrompt, - systemMessage, - userMessage, - tools, - helpers, - apiKey, - getOutput: getEvalState, - onAssistantMessageStart: options?.runContext?.onAssistantMessageStart, - onAssistantToken: options?.runContext?.onAssistantChunk, - onAssistantMessageEnd: options?.runContext?.onAssistantMessageEnd, - onToolCall: options?.runContext?.onToolCall, - options: { - maxIterations: options?.maxIterations, - model, - workspace: workspaceRoot, - provider: options?.provider - } - }) + const rawResult = await runEval({ + userPrompt, + systemMessage, + userMessage, + tools, + helpers, + apiKey, + getOutput: getEvalState, + onAssistantMessageStart: options?.runContext?.onAssistantMessageStart, + onAssistantToken: options?.runContext?.onAssistantChunk, + onAssistantMessageEnd: options?.runContext?.onAssistantMessageEnd, + onToolCall: options?.runContext?.onToolCall, + options: { + maxIterations: options?.maxIterations, + model, + workspace: workspaceRoot, + provider: options?.provider, + transport: options?.transport, + backend: options?.backend, + proxyCaseId: options?.runContext?.caseId, + proxyAttempt: options?.runContext?.attempt, + }, + }); - return { - files: rawResult.output, - success: rawResult.success, - error: rawResult.error, - assistantMessageCount: rawResult.iterations, - toolCallCount: rawResult.toolCallsCount, - toolsUsed: rawResult.toolsCalled, - tokenUsage: rawResult.tokenUsage - } - } finally { - await cleanup() - } + return { + files: rawResult.output, + success: rawResult.success, + error: rawResult.error, + assistantMessageCount: rawResult.iterations, + toolCallCount: rawResult.toolCallsCount, + toolsUsed: rawResult.toolsCalled, + tokenUsage: rawResult.tokenUsage, + }; + } finally { + await cleanup(); + } } diff --git a/ai_evals/adapters/frontend/core/app/appFixtureLoader.ts b/ai_evals/adapters/frontend/core/app/appFixtureLoader.ts index b0864cfba0..bd821eafb4 100644 --- a/ai_evals/adapters/frontend/core/app/appFixtureLoader.ts +++ b/ai_evals/adapters/frontend/core/app/appFixtureLoader.ts @@ -1,16 +1,16 @@ import type { - BackendRunnable, - DataTableSchema, - InlineScript -} from '../../../../../frontend/src/lib/components/copilot/chat/app/core' -import type { AppFilesState } from '../../../../core/validators' + BackendRunnable, + DataTableSchema, + InlineScript, +} from "../../../../../frontend/src/lib/components/copilot/chat/app/core"; +import type { AppFilesState } from "../../../../core/validators"; /** * Backend runnable metadata stored in meta.json files. */ interface BackendMeta { - name: string - language: 'bun' | 'python3' + name: string; + language: "bun" | "python3"; } /** @@ -18,49 +18,53 @@ interface BackendMeta { * File paths are relative to the base directory with a leading '/'. */ async function readFilesRecursively( - dir: string, - basePath: string = '' + dir: string, + basePath: string = "", ): Promise> { - // @ts-ignore - Node.js fs/promises - const { readdir, readFile } = await import('fs/promises') - // @ts-ignore - Node.js path - const { join } = await import('path') + // @ts-ignore - Node.js fs/promises + const { readdir, readFile } = await import("fs/promises"); + // @ts-ignore - Node.js path + const { join } = await import("path"); - const result: Record = {} - const entries = await readdir(dir, { withFileTypes: true }) + const result: Record = {}; + const entries = await readdir(dir, { withFileTypes: true }); - for (const entry of entries) { - const fullPath = join(dir, entry.name) - const relativePath = basePath ? `${basePath}/${entry.name}` : `/${entry.name}` + for (const entry of entries) { + const fullPath = join(dir, entry.name); + const relativePath = basePath + ? `${basePath}/${entry.name}` + : `/${entry.name}`; - if (entry.isDirectory()) { - const subFiles = await readFilesRecursively(fullPath, relativePath) - Object.assign(result, subFiles) - } else { - const content = await readFile(fullPath, 'utf-8') - result[relativePath] = content - } - } + if (entry.isDirectory()) { + const subFiles = await readFilesRecursively(fullPath, relativePath); + Object.assign(result, subFiles); + } else { + const content = await readFile(fullPath, "utf-8"); + result[relativePath] = content; + } + } - return result + return result; } /** * Loads frontend files from a directory. * All files are read recursively and paths become keys with leading '/'. */ -async function loadFrontend(frontendPath: string): Promise> { - // @ts-ignore - Node.js fs/promises - const { access } = await import('fs/promises') +async function loadFrontend( + frontendPath: string, +): Promise> { + // @ts-ignore - Node.js fs/promises + const { access } = await import("fs/promises"); - try { - await access(frontendPath) - } catch { - // Directory doesn't exist, return empty - return {} - } + try { + await access(frontendPath); + } catch { + // Directory doesn't exist, return empty + return {}; + } - return readFilesRecursively(frontendPath) + return readFilesRecursively(frontendPath); } /** @@ -69,78 +73,89 @@ async function loadFrontend(frontendPath: string): Promise> { - // @ts-ignore - Node.js fs/promises - const { readdir, readFile, access } = await import('fs/promises') - // @ts-ignore - Node.js path - const { join } = await import('path') +async function loadBackend( + backendPath: string, +): Promise> { + // @ts-ignore - Node.js fs/promises + const { readdir, readFile, access } = await import("fs/promises"); + // @ts-ignore - Node.js path + const { join } = await import("path"); - try { - await access(backendPath) - } catch { - // Directory doesn't exist, return empty - return {} - } + try { + await access(backendPath); + } catch { + // Directory doesn't exist, return empty + return {}; + } - const result: Record = {} - const entries = await readdir(backendPath, { withFileTypes: true }) + const result: Record = {}; + const entries = await readdir(backendPath, { withFileTypes: true }); - for (const entry of entries) { - if (!entry.isDirectory()) continue + for (const entry of entries) { + if (!entry.isDirectory()) continue; - const runnableKey = entry.name - const runnablePath = join(backendPath, entry.name) + const runnableKey = entry.name; + const runnablePath = join(backendPath, entry.name); - // Read meta.json - const metaPath = join(runnablePath, 'meta.json') - let meta: BackendMeta - try { - const metaContent = await readFile(metaPath, 'utf-8') - meta = JSON.parse(metaContent) - } catch { - console.warn(`Missing or invalid meta.json for runnable '${runnableKey}', skipping`) - continue - } + // Read meta.json + const metaPath = join(runnablePath, "meta.json"); + let meta: BackendMeta; + try { + const metaContent = await readFile(metaPath, "utf-8"); + meta = JSON.parse(metaContent); + } catch { + console.warn( + `Missing or invalid meta.json for runnable '${runnableKey}', skipping`, + ); + continue; + } - // Find and read the main file (main.ts or main.py) - const runnableFiles = await readdir(runnablePath) - const mainFile = runnableFiles.find((f) => f === 'main.ts' || f === 'main.py') + // Find and read the main file (main.ts or main.py) + const runnableFiles = await readdir(runnablePath); + const mainFile = runnableFiles.find( + (f) => f === "main.ts" || f === "main.py", + ); - if (!mainFile) { - console.warn(`No main.ts or main.py found for runnable '${runnableKey}', skipping`) - continue - } + if (!mainFile) { + console.warn( + `No main.ts or main.py found for runnable '${runnableKey}', skipping`, + ); + continue; + } - const content = await readFile(join(runnablePath, mainFile), 'utf-8') + const content = await readFile(join(runnablePath, mainFile), "utf-8"); - const inlineScript: InlineScript = { - language: meta.language, - content - } + const inlineScript: InlineScript = { + language: meta.language, + content, + }; - result[runnableKey] = { - name: meta.name, - type: 'inline', - inlineScript - } - } + result[runnableKey] = { + name: meta.name, + type: "inline", + inlineScript, + }; + } - return result + return result; } async function loadDatatables(fixturePath: string): Promise { - // @ts-ignore - Node.js fs/promises - const { readFile } = await import('fs/promises') - // @ts-ignore - Node.js path - const { join } = await import('path') + // @ts-ignore - Node.js fs/promises + const { readFile } = await import("fs/promises"); + // @ts-ignore - Node.js path + const { join } = await import("path"); - try { - const content = await readFile(join(fixturePath, 'datatables.json'), 'utf-8') - const parsed = JSON.parse(content) - return Array.isArray(parsed) ? (parsed as DataTableSchema[]) : [] - } catch { - return [] - } + try { + const content = await readFile( + join(fixturePath, "datatables.json"), + "utf-8", + ); + const parsed = JSON.parse(content); + return Array.isArray(parsed) ? (parsed as DataTableSchema[]) : []; + } catch { + return []; + } } /** @@ -162,32 +177,32 @@ async function loadDatatables(fixturePath: string): Promise { * @param fixturePath - Path to the fixture directory * @returns AppFiles object with frontend and backend */ -export async function loadAppFixture(fixturePath: string): Promise { - // @ts-ignore - Node.js path - const { join } = await import('path') +export async function loadAppFixture( + fixturePath: string, +): Promise { + // @ts-ignore - Node.js path + const { join } = await import("path"); - const frontend = await loadFrontend(join(fixturePath, 'frontend')) - const backend = await loadBackend(join(fixturePath, 'backend')) - const datatables = await loadDatatables(fixturePath) + const frontend = await loadFrontend(join(fixturePath, "frontend")); + const backend = await loadBackend(join(fixturePath, "backend")); + const datatables = await loadDatatables(fixturePath); - return { frontend, backend, datatables } + return { frontend, backend, datatables }; } /** * Loads an app fixture and returns the separate frontend and backend objects. * Convenience function for use with runAppEval options. */ -export async function loadAppFixtureForEval( - fixturePath: string -): Promise<{ - initialFrontend: Record - initialBackend: Record - initialDatatables: DataTableSchema[] +export async function loadAppFixtureForEval(fixturePath: string): Promise<{ + initialFrontend: Record; + initialBackend: AppFilesState["backend"]; + initialDatatables: DataTableSchema[]; }> { - const { frontend, backend, datatables } = await loadAppFixture(fixturePath) - return { - initialFrontend: frontend, - initialBackend: backend, - initialDatatables: datatables - } + const { frontend, backend, datatables } = await loadAppFixture(fixturePath); + return { + initialFrontend: frontend, + initialBackend: backend, + initialDatatables: datatables, + }; } diff --git a/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts b/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts index 7575457a0f..1fad846277 100644 --- a/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts @@ -1,108 +1,119 @@ -import { mkdtemp } from 'fs/promises' -import { tmpdir } from 'os' -import { join } from 'path' -import type { FlowModule } from '$lib/gen' -import type { AIProvider } from '$lib/gen/types.gen' -import type { ExtendedOpenFlow } from '$lib/components/flows/types' +import { mkdtemp } from "fs/promises"; +import { tmpdir } from "os"; +import { join } from "path"; +import type { FlowModule } from "$lib/gen"; +import type { AIProvider } from "$lib/gen/types.gen"; +import type { ExtendedOpenFlow } from "$lib/components/flows/types"; import { - flowTools, - prepareFlowSystemMessage, - prepareFlowUserMessage, - type FlowAIChatHelpers -} from '../../../../../frontend/src/lib/components/copilot/chat/flow/core' -import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared' -import { createFlowFileHelpers, type FlowWorkspaceFixtures } from './fileHelpers' -import { runEval } from '../shared' -import type { ModeRunContext } from '../../../../core/types' -import type { TokenUsage } from '../shared/types' + flowTools, + prepareFlowSystemMessage, + prepareFlowUserMessage, + type FlowAIChatHelpers, +} from "../../../../../frontend/src/lib/components/copilot/chat/flow/core"; +import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; +import { + createFlowFileHelpers, + type FlowWorkspaceFixtures, +} from "./fileHelpers"; +import { runEval } from "../shared"; +import type { ModeRunContext } from "../../../../core/types"; +import type { TokenUsage } from "../shared/types"; +import type { FrontendEvalTransport } from "../../../../core/frontendTransport"; +import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; export interface FlowFixture { - value?: { - modules?: FlowModule[] - preprocessor_module?: FlowModule - failure_module?: FlowModule - } - schema?: Record + value?: { + modules?: FlowModule[]; + preprocessor_module?: FlowModule; + failure_module?: FlowModule; + }; + schema?: Record; } export interface FlowEvalResult { - success: boolean - flow: ExtendedOpenFlow - error?: string - assistantMessageCount: number - toolCallCount: number - toolsUsed: string[] - tokenUsage: TokenUsage + success: boolean; + flow: ExtendedOpenFlow; + error?: string; + assistantMessageCount: number; + toolCallCount: number; + toolsUsed: string[]; + tokenUsage: TokenUsage; } export interface FlowEvalOptions { - initialFlow?: FlowFixture - workspaceFixtures?: FlowWorkspaceFixtures - model?: string - maxIterations?: number - provider?: AIProvider - workspaceRoot?: string - runContext?: ModeRunContext + initialFlow?: FlowFixture; + workspaceFixtures?: FlowWorkspaceFixtures; + model?: string; + maxIterations?: number; + provider?: AIProvider; + transport?: FrontendEvalTransport; + backend?: WindmillBackendSettings; + workspaceRoot?: string; + runContext?: ModeRunContext; } export async function runFlowEval( - userPrompt: string, - apiKey: string, - options?: FlowEvalOptions + userPrompt: string, + apiKey: string, + options?: FlowEvalOptions, ): Promise { - const workspaceRoot = - options?.workspaceRoot ?? - (await mkdtemp(join(tmpdir(), 'wmill-frontend-flow-benchmark-'))) - const { helpers, getFlow, cleanup } = await createFlowFileHelpers( - options?.initialFlow?.value?.modules ?? [], - options?.initialFlow?.schema, - options?.initialFlow?.value?.preprocessor_module, - options?.initialFlow?.value?.failure_module, - workspaceRoot, - options?.workspaceFixtures - ) + const workspaceRoot = + options?.workspaceRoot ?? + (await mkdtemp(join(tmpdir(), "wmill-frontend-flow-benchmark-"))); + const { helpers, getFlow, cleanup } = await createFlowFileHelpers( + options?.initialFlow?.value?.modules ?? [], + options?.initialFlow?.schema, + options?.initialFlow?.value?.preprocessor_module, + options?.initialFlow?.value?.failure_module, + workspaceRoot, + options?.workspaceFixtures, + ); - try { - const systemMessage = prepareFlowSystemMessage() - const tools = flowTools as ProductionTool[] - const model = options?.model ?? 'claude-haiku-4-5-20251001' - const userMessage = prepareFlowUserMessage( - userPrompt, - helpers.getFlowAndSelectedId(), - [], - helpers.inlineScriptSession - ) + try { + const systemMessage = prepareFlowSystemMessage(); + const tools = flowTools as ProductionTool[]; + const model = options?.model ?? "claude-haiku-4-5-20251001"; + const userMessage = prepareFlowUserMessage( + userPrompt, + helpers.getFlowAndSelectedId(), + [], + helpers.inlineScriptSession, + ); - const rawResult = await runEval({ - userPrompt, - systemMessage, - userMessage, - tools, - helpers, - apiKey, - getOutput: getFlow, - onAssistantMessageStart: options?.runContext?.onAssistantMessageStart, - onAssistantToken: options?.runContext?.onAssistantChunk, - onAssistantMessageEnd: options?.runContext?.onAssistantMessageEnd, - onToolCall: options?.runContext?.onToolCall, - options: { - maxIterations: options?.maxIterations, - model, - workspace: workspaceRoot, - provider: options?.provider - } - }) + const rawResult = await runEval({ + userPrompt, + systemMessage, + userMessage, + tools, + helpers, + apiKey, + getOutput: getFlow, + onAssistantMessageStart: options?.runContext?.onAssistantMessageStart, + onAssistantToken: options?.runContext?.onAssistantChunk, + onAssistantMessageEnd: options?.runContext?.onAssistantMessageEnd, + onToolCall: options?.runContext?.onToolCall, + options: { + maxIterations: options?.maxIterations, + model, + workspace: workspaceRoot, + provider: options?.provider, + transport: options?.transport, + backend: options?.backend, + proxyCaseId: options?.runContext?.caseId, + proxyAttempt: options?.runContext?.attempt, + }, + }); - return { - flow: rawResult.output, - success: rawResult.success, - error: rawResult.error, - assistantMessageCount: rawResult.iterations, - toolCallCount: rawResult.toolCallsCount, - toolsUsed: rawResult.toolsCalled, - tokenUsage: rawResult.tokenUsage - } - } finally { - await cleanup() - } + return { + flow: rawResult.output, + success: rawResult.success, + error: rawResult.error, + assistantMessageCount: rawResult.iterations, + toolCallCount: rawResult.toolCallsCount, + toolsUsed: rawResult.toolsCalled, + tokenUsage: rawResult.tokenUsage, + }; + } finally { + await cleanup(); + } } diff --git a/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts b/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts index dd4053ae9c..3fe1d1b653 100644 --- a/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts @@ -1,110 +1,119 @@ -import { mkdtemp } from 'fs/promises' -import { tmpdir } from 'os' -import { join } from 'path' -import type { AIProvider, AIProviderModel } from '$lib/gen/types.gen' -import type { ContextElement } from '../../../../../frontend/src/lib/components/copilot/chat/context' +import { mkdtemp } from "fs/promises"; +import { tmpdir } from "os"; +import { join } from "path"; +import type { AIProvider, AIProviderModel } from "$lib/gen/types.gen"; +import type { ContextElement } from "../../../../../frontend/src/lib/components/copilot/chat/context"; import { - prepareScriptSystemMessage, - prepareScriptTools, - prepareScriptUserMessage, - type ScriptChatHelpers -} from '../../../../../frontend/src/lib/components/copilot/chat/script/core' -import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared' -import { createScriptFileHelpers, type ScriptEvalState } from './fileHelpers' -import { runEval } from '../shared' -import type { ModeRunContext } from '../../../../core/types' -import type { TokenUsage } from '../shared/types' + prepareScriptSystemMessage, + prepareScriptTools, + prepareScriptUserMessage, + type ScriptChatHelpers, +} from "../../../../../frontend/src/lib/components/copilot/chat/script/core"; +import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; +import { createScriptFileHelpers, type ScriptEvalState } from "./fileHelpers"; +import { runEval } from "../shared"; +import type { ModeRunContext } from "../../../../core/types"; +import type { TokenUsage } from "../shared/types"; +import type { FrontendEvalTransport } from "../../../../core/frontendTransport"; +import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; export interface ScriptEvalResult { - success: boolean - script: ScriptEvalState - error?: string - assistantMessageCount: number - toolCallCount: number - toolsUsed: string[] - tokenUsage: TokenUsage + success: boolean; + script: ScriptEvalState; + error?: string; + assistantMessageCount: number; + toolCallCount: number; + toolsUsed: string[]; + tokenUsage: TokenUsage; } export interface ScriptEvalOptions { - initialScript: ScriptEvalState - model?: string - maxIterations?: number - provider?: AIProvider - workspaceRoot?: string - runContext?: ModeRunContext + initialScript: ScriptEvalState; + model?: string; + maxIterations?: number; + provider?: AIProvider; + transport?: FrontendEvalTransport; + backend?: WindmillBackendSettings; + workspaceRoot?: string; + runContext?: ModeRunContext; } function resolveModelProvider( - model: string, - provider?: AIProvider + model: string, + provider?: AIProvider, ): AIProviderModel { - if (provider) { - return { provider, model } - } - if (model.startsWith('claude')) { - return { provider: 'anthropic', model } - } - return { provider: 'openai', model } + if (provider) { + return { provider, model }; + } + if (model.startsWith("claude")) { + return { provider: "anthropic", model }; + } + return { provider: "openai", model }; } export async function runScriptEval( - userPrompt: string, - apiKey: string, - options: ScriptEvalOptions + userPrompt: string, + apiKey: string, + options: ScriptEvalOptions, ): Promise { - const workspaceRoot = - options.workspaceRoot ?? (await mkdtemp(join(tmpdir(), 'wmill-frontend-script-benchmark-'))) - const { helpers, getScript, cleanup } = await createScriptFileHelpers( - options.initialScript, - workspaceRoot - ) + const workspaceRoot = + options.workspaceRoot ?? + (await mkdtemp(join(tmpdir(), "wmill-frontend-script-benchmark-"))); + const { helpers, getScript, cleanup } = await createScriptFileHelpers( + options.initialScript, + workspaceRoot, + ); - try { - const model = options.model ?? 'claude-haiku-4-5-20251001' - const modelProvider = resolveModelProvider(model, options.provider) - const selectedContext: ContextElement[] = [] - const systemMessage = prepareScriptSystemMessage( - modelProvider, - options.initialScript.lang, - {} - ) - const tools = prepareScriptTools( - modelProvider, - options.initialScript.lang, - selectedContext - ) as ProductionTool[] - const userMessage = prepareScriptUserMessage(userPrompt, selectedContext) + try { + const model = options.model ?? "claude-haiku-4-5-20251001"; + const modelProvider = resolveModelProvider(model, options.provider); + const selectedContext: ContextElement[] = []; + const systemMessage = prepareScriptSystemMessage( + modelProvider, + options.initialScript.lang, + {}, + ); + const tools = prepareScriptTools( + modelProvider, + options.initialScript.lang, + selectedContext, + ) as ProductionTool[]; + const userMessage = prepareScriptUserMessage(userPrompt, selectedContext); - const rawResult = await runEval({ - userPrompt, - systemMessage, - userMessage, - tools, - helpers, - apiKey, - getOutput: getScript, - onAssistantMessageStart: options.runContext?.onAssistantMessageStart, - onAssistantToken: options.runContext?.onAssistantChunk, - onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd, - onToolCall: options.runContext?.onToolCall, - options: { - maxIterations: options.maxIterations, - model, - workspace: workspaceRoot, - provider: modelProvider.provider - } - }) + const rawResult = await runEval({ + userPrompt, + systemMessage, + userMessage, + tools, + helpers, + apiKey, + getOutput: getScript, + onAssistantMessageStart: options.runContext?.onAssistantMessageStart, + onAssistantToken: options.runContext?.onAssistantChunk, + onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd, + onToolCall: options.runContext?.onToolCall, + options: { + maxIterations: options.maxIterations, + model, + workspace: workspaceRoot, + provider: modelProvider.provider, + transport: options.transport, + backend: options.backend, + proxyCaseId: options.runContext?.caseId, + proxyAttempt: options.runContext?.attempt, + }, + }); - return { - script: rawResult.output, - success: rawResult.success, - error: rawResult.error, - assistantMessageCount: rawResult.iterations, - toolCallCount: rawResult.toolCallsCount, - toolsUsed: rawResult.toolsCalled, - tokenUsage: rawResult.tokenUsage - } - } finally { - await cleanup() - } + return { + script: rawResult.output, + success: rawResult.success, + error: rawResult.error, + assistantMessageCount: rawResult.iterations, + toolCallCount: rawResult.toolCallsCount, + toolsUsed: rawResult.toolsCalled, + tokenUsage: rawResult.tokenUsage, + }; + } finally { + await cleanup(); + } } diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 8d8b07fea7..5b1f2e948e 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -1,46 +1,51 @@ import type { - ChatCompletionMessageParam, - ChatCompletionSystemMessageParam -} from 'openai/resources/chat/completions.mjs' -import type { AIProvider } from '$lib/gen/types.gen' -import type { ToolCallDetail, EvalRunnerOptions, RawEvalResult } from './types' -import { runChatLoop, type ChatClients } from '../../../../../frontend/src/lib/components/copilot/chat/chatLoop' -import type { - Tool as ProductionTool, - ToolCallbacks -} from '../../../../../frontend/src/lib/components/copilot/chat/shared' + ChatCompletionMessageParam, + ChatCompletionSystemMessageParam, +} from "openai/resources/chat/completions.mjs"; +import type { AIProvider } from "$lib/gen/types.gen"; +import type { ToolCallDetail, EvalRunnerOptions, RawEvalResult } from "./types"; import { - createEvalClients, - type FrontendEvalProvider, - resolveEvalModelProvider -} from './providerConfig' + runChatLoop, + type ChatClients, +} from "../../../../../frontend/src/lib/components/copilot/chat/chatLoop"; +import type { + Tool as ProductionTool, + ToolCallbacks, +} from "../../../../../frontend/src/lib/components/copilot/chat/shared"; +import { + buildProxyResourcePath, + createEvalClients, + type FrontendEvalProvider, + resolveEvalModelProvider, +} from "./providerConfig"; +import { WindmillBackendClient } from "../../windmillBackend"; /** * Parameters for running a base evaluation. */ export interface RunEvalParams { - /** The user's prompt/instruction */ - userPrompt: string - /** System message for the LLM */ - systemMessage: ChatCompletionSystemMessageParam - /** User message for the LLM */ - userMessage: ChatCompletionMessageParam - /** Tool definitions for the LLM API (unused — derived from tools) */ - toolDefs?: unknown - /** Full tool implementations for execution */ - tools: ProductionTool[] - /** Domain-specific helpers for tool execution */ - helpers: THelpers - /** API key for the provider */ - apiKey: string - /** Function to get the current output state */ - getOutput: () => TOutput - /** Optional configuration */ - options?: EvalRunnerOptions - onAssistantMessageStart?: () => void - onAssistantToken?: (token: string) => void - onAssistantMessageEnd?: () => void - onToolCall?: (input: { toolName: string; argumentsText: string }) => void + /** The user's prompt/instruction */ + userPrompt: string; + /** System message for the LLM */ + systemMessage: ChatCompletionSystemMessageParam; + /** User message for the LLM */ + userMessage: ChatCompletionMessageParam; + /** Tool definitions for the LLM API (unused — derived from tools) */ + toolDefs?: unknown; + /** Full tool implementations for execution */ + tools: ProductionTool[]; + /** Domain-specific helpers for tool execution */ + helpers: THelpers; + /** API key for the provider */ + apiKey: string; + /** Function to get the current output state */ + getOutput: () => TOutput; + /** Optional configuration */ + options?: EvalRunnerOptions; + onAssistantMessageStart?: () => void; + onAssistantToken?: (token: string) => void; + onAssistantMessageEnd?: () => void; + onToolCall?: (input: { toolName: string; argumentsText: string }) => void; } /** @@ -48,152 +53,206 @@ export interface RunEvalParams { * Uses streaming via real provider SDKs instead of OpenRouter non-streaming. */ export async function runEval( - params: RunEvalParams + params: RunEvalParams, ): Promise> { - const { - systemMessage, - userMessage, - tools, - helpers, - apiKey, - getOutput, - options, - onAssistantMessageStart, - onAssistantToken, - onAssistantMessageEnd, - onToolCall - } = params - let shouldEmitMessageStart = true + const { + systemMessage, + userMessage, + tools, + helpers, + apiKey, + getOutput, + options, + onAssistantMessageStart, + onAssistantToken, + onAssistantMessageEnd, + onToolCall, + } = params; + let shouldEmitMessageStart = true; - const model = options?.model ?? 'gpt-4o' - const maxIterations = options?.maxIterations ?? 20 - const workspace = options?.workspace ?? 'test-workspace' - const provider = toFrontendEvalProvider(options?.provider) + const model = options?.model ?? "gpt-4o"; + const maxIterations = options?.maxIterations ?? 20; + const workspace = options?.workspace ?? "test-workspace"; + const provider = toFrontendEvalProvider(options?.provider); - const modelProvider = resolveEvalModelProvider(model, provider) - const clients = createEvalClients(modelProvider.provider, apiKey) as unknown as ChatClients + const modelProvider = resolveEvalModelProvider(model, provider); - const messages: ChatCompletionMessageParam[] = [userMessage] - let toolCallsCount = 0 - const toolsCalled: string[] = [] - const toolCallDetails: ToolCallDetail[] = [] + const messages: ChatCompletionMessageParam[] = [userMessage]; + let toolCallsCount = 0; + const toolsCalled: string[] = []; + const toolCallDetails: ToolCallDetail[] = []; - // Wrap tools to intercept fn calls for tracking. - // Cast to ProductionTool since the eval Tool has a narrower toolCallbacks type - // but the actual callbacks passed at runtime will satisfy both interfaces. - const wrappedTools = tools.map((tool) => ({ - ...tool, - fn: async (p: any) => { - toolCallsCount++ - toolsCalled.push(tool.def.function.name) - let argumentsText = '' - try { - const args = typeof p.args === 'string' ? JSON.parse(p.args) : p.args - toolCallDetails.push({ name: tool.def.function.name, arguments: args }) - argumentsText = JSON.stringify(args) - } catch { - toolCallDetails.push({ - name: tool.def.function.name, - arguments: p.args - }) - argumentsText = typeof p.args === 'string' ? p.args : JSON.stringify(p.args) - } - onToolCall?.({ - toolName: tool.def.function.name, - argumentsText - }) - return tool.fn(p) - } - })) + // Wrap tools to intercept fn calls for tracking. + // Cast to ProductionTool since the eval Tool has a narrower toolCallbacks type + // but the actual callbacks passed at runtime will satisfy both interfaces. + const wrappedTools = tools.map((tool) => ({ + ...tool, + fn: async (p: any) => { + toolCallsCount++; + toolsCalled.push(tool.def.function.name); + let argumentsText = ""; + try { + const args = typeof p.args === "string" ? JSON.parse(p.args) : p.args; + toolCallDetails.push({ name: tool.def.function.name, arguments: args }); + argumentsText = JSON.stringify(args); + } catch { + toolCallDetails.push({ + name: tool.def.function.name, + arguments: p.args, + }); + argumentsText = + typeof p.args === "string" ? p.args : JSON.stringify(p.args); + } + onToolCall?.({ + toolName: tool.def.function.name, + argumentsText, + }); + return tool.fn(p); + }, + })); - // No-op callbacks for eval - const callbacks: ToolCallbacks & { - onNewToken: (token: string) => void - onMessageEnd: () => void - } = { - setToolStatus: () => {}, - removeToolStatus: () => {}, - onNewToken: (token: string) => { - if (shouldEmitMessageStart) { - onAssistantMessageStart?.() - shouldEmitMessageStart = false - } - onAssistantToken?.(token) - }, - onMessageEnd: () => { - if (!shouldEmitMessageStart) { - onAssistantMessageEnd?.() - } - shouldEmitMessageStart = true - } - } + // No-op callbacks for eval + const callbacks: ToolCallbacks & { + onNewToken: (token: string) => void; + onMessageEnd: () => void; + } = { + setToolStatus: () => {}, + removeToolStatus: () => {}, + onNewToken: (token: string) => { + if (shouldEmitMessageStart) { + onAssistantMessageStart?.(); + shouldEmitMessageStart = false; + } + onAssistantToken?.(token); + }, + onMessageEnd: () => { + if (!shouldEmitMessageStart) { + onAssistantMessageEnd?.(); + } + shouldEmitMessageStart = true; + }, + }; - const abortController = new AbortController() + const abortController = new AbortController(); - try { - const result = await runChatLoop({ - messages, - systemMessage, - tools: wrappedTools, - helpers, - abortController, - callbacks, - modelProvider, - clients, - workspace, - maxIterations, - skipResponsesApi: modelProvider.provider !== 'openai' - }) + const executeChatLoop = async (clients: ChatClients) => { + try { + const result = await runChatLoop({ + messages, + systemMessage, + tools: wrappedTools, + helpers, + abortController, + callbacks, + modelProvider, + clients, + workspace, + maxIterations, + skipResponsesApi: modelProvider.provider !== "openai", + }); - if (result.hitMaxIterations) { - return { - success: false, - output: getOutput(), - error: `Reached max turns (${maxIterations})`, - tokenUsage: result.tokenUsage, - toolCallsCount, - toolsCalled, - toolCallDetails, - iterations: Math.max(1, result.addedMessages.filter((m) => m.role === 'assistant').length), - messages - } - } + if (result.hitMaxIterations) { + return { + success: false, + output: getOutput(), + error: `Reached max turns (${maxIterations})`, + tokenUsage: result.tokenUsage, + toolCallsCount, + toolsCalled, + toolCallDetails, + iterations: Math.max( + 1, + result.addedMessages.filter((m) => m.role === "assistant").length, + ), + messages, + }; + } - return { - success: true, - output: getOutput(), - tokenUsage: result.tokenUsage, - toolCallsCount, - toolsCalled, - toolCallDetails, - iterations: Math.max(1, result.addedMessages.filter((m) => m.role === 'assistant').length), - messages - } - } catch (err) { - let errorMessage: string - if (err instanceof Error) { - errorMessage = err.stack ?? err.message - } else { - errorMessage = String(err) - } + return { + success: true, + output: getOutput(), + tokenUsage: result.tokenUsage, + toolCallsCount, + toolsCalled, + toolCallDetails, + iterations: Math.max( + 1, + result.addedMessages.filter((m) => m.role === "assistant").length, + ), + messages, + }; + } catch (err) { + let errorMessage: string; + if (err instanceof Error) { + errorMessage = err.stack ?? err.message; + } else { + errorMessage = String(err); + } - return { - success: false, - output: getOutput(), - error: errorMessage, - tokenUsage: { prompt: 0, completion: 0, total: 0 }, - toolCallsCount, - toolsCalled, - toolCallDetails, - iterations: 0, - messages - } - } + return { + success: false, + output: getOutput(), + error: errorMessage, + tokenUsage: { prompt: 0, completion: 0, total: 0 }, + toolCallsCount, + toolsCalled, + toolCallDetails, + iterations: 0, + messages, + }; + } + }; + + if (options?.transport === "proxy") { + const backendSettings = options.backend; + if (!backendSettings) { + throw new Error("Missing backend settings for proxy transport"); + } + + const backendClient = new WindmillBackendClient(backendSettings); + return await backendClient.withWorkspace( + options.proxyCaseId ?? "eval", + options.proxyAttempt ?? 1, + async (proxyWorkspaceId) => { + const resourcePath = buildProxyResourcePath(modelProvider.provider); + await backendClient.upsertResource({ + workspaceId: proxyWorkspaceId, + path: resourcePath, + resourceType: modelProvider.provider, + value: { api_key: apiKey }, + }); + const token = await backendClient.getToken(); + const clients = createEvalClients({ + provider: modelProvider.provider, + apiKey, + transport: "proxy", + proxy: { + baseURL: `${backendSettings.baseUrl}/api/w/${encodeURIComponent(proxyWorkspaceId)}/ai/proxy`, + bearerToken: token, + resourcePath, + }, + }) as unknown as ChatClients; + return await executeChatLoop(clients); + }, + ); + } + + const clients = createEvalClients({ + provider: modelProvider.provider, + apiKey, + }) as unknown as ChatClients; + return await executeChatLoop(clients); } -function toFrontendEvalProvider(provider?: AIProvider): FrontendEvalProvider | undefined { - if (provider === 'anthropic' || provider === 'openai' || provider === 'googleai') { - return provider - } - return undefined +function toFrontendEvalProvider( + provider?: AIProvider, +): FrontendEvalProvider | undefined { + if ( + provider === "anthropic" || + provider === "openai" || + provider === "googleai" + ) { + return provider; + } + return undefined; } diff --git a/ai_evals/adapters/frontend/core/shared/providerConfig.test.ts b/ai_evals/adapters/frontend/core/shared/providerConfig.test.ts index ad0ef7652a..e504ced376 100644 --- a/ai_evals/adapters/frontend/core/shared/providerConfig.test.ts +++ b/ai_evals/adapters/frontend/core/shared/providerConfig.test.ts @@ -1,12 +1,17 @@ import { describe, expect, it } from "bun:test"; import { + buildProxyHeaders, + buildProxyResourcePath, buildOpenAICompatibleClientOptions, resolveEvalModelProvider, } from "./providerConfig"; describe("buildOpenAICompatibleClientOptions", () => { it("adds Gemini's OpenAI-compatible base URL and client header", () => { - const options = buildOpenAICompatibleClientOptions("googleai", "gemini-test-key"); + const options = buildOpenAICompatibleClientOptions( + "googleai", + "gemini-test-key", + ); expect(options).toMatchObject({ apiKey: "gemini-test-key", @@ -18,12 +23,28 @@ describe("buildOpenAICompatibleClientOptions", () => { }); it("keeps the default OpenAI-compatible config for OpenAI", () => { - expect(buildOpenAICompatibleClientOptions("openai", "openai-test-key")).toEqual({ + expect( + buildOpenAICompatibleClientOptions("openai", "openai-test-key"), + ).toEqual({ apiKey: "openai-test-key", }); }); }); +describe("proxy helpers", () => { + it("builds provider-scoped proxy resource paths", () => { + expect(buildProxyResourcePath("googleai")).toBe("f/evals/ai/googleai"); + expect(buildProxyResourcePath("anthropic")).toBe("f/evals/ai/anthropic"); + }); + + it("adds auth and resource headers for workspace proxy requests", () => { + expect(buildProxyHeaders("token-123", "f/evals/ai/googleai")).toEqual({ + Authorization: "Bearer token-123", + "X-Resource-Path": "f/evals/ai/googleai", + }); + }); +}); + describe("resolveEvalModelProvider", () => { it("infers googleai from Gemini model ids", () => { expect(resolveEvalModelProvider("gemini-2.5-flash")).toEqual({ diff --git a/ai_evals/adapters/frontend/core/shared/providerConfig.ts b/ai_evals/adapters/frontend/core/shared/providerConfig.ts index 44a698b2af..62ba221a40 100644 --- a/ai_evals/adapters/frontend/core/shared/providerConfig.ts +++ b/ai_evals/adapters/frontend/core/shared/providerConfig.ts @@ -1,6 +1,7 @@ import Anthropic from "@anthropic-ai/sdk"; import OpenAI from "openai"; import type { FrontendEvalModelConfig } from "../../../../core/models"; +import type { FrontendEvalTransport } from "../../../../core/frontendTransport"; export type FrontendEvalProvider = FrontendEvalModelConfig["provider"]; @@ -14,12 +15,34 @@ export interface ResolvedEvalModelProvider { model: string; } -const GEMINI_OPENAI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai/"; +export interface EvalProxyClientConfig { + baseURL: string; + bearerToken: string; + resourcePath: string; +} + +const GEMINI_OPENAI_BASE_URL = + "https://generativelanguage.googleapis.com/v1beta/openai/"; const GEMINI_GOOG_API_CLIENT = "windmill-ai-evals/1.0"; +const EVAL_PROXY_RESOURCE_PREFIX = "f/evals/ai"; + +export function buildProxyHeaders( + bearerToken: string, + resourcePath: string, +): Record { + return { + Authorization: `Bearer ${bearerToken}`, + "X-Resource-Path": resourcePath, + }; +} + +export function buildProxyResourcePath(provider: FrontendEvalProvider): string { + return `${EVAL_PROXY_RESOURCE_PREFIX}/${provider}`; +} export function buildOpenAICompatibleClientOptions( provider: Exclude, - apiKey: string + apiKey: string, ): ConstructorParameters[0] { if (provider === "googleai") { return { @@ -34,26 +57,71 @@ export function buildOpenAICompatibleClientOptions( return { apiKey }; } -export function createEvalClients( - provider: FrontendEvalProvider, - apiKey: string -): EvalClients { - if (provider === "anthropic") { +function buildProxyOpenAIClientOptions( + proxy: EvalProxyClientConfig, +): ConstructorParameters[0] { + return { + apiKey: "unused", + baseURL: proxy.baseURL, + defaultHeaders: buildProxyHeaders(proxy.bearerToken, proxy.resourcePath), + }; +} + +export function createEvalClients(input: { + provider: FrontendEvalProvider; + apiKey: string; + transport?: FrontendEvalTransport; + proxy?: EvalProxyClientConfig; +}): EvalClients { + const transport = input.transport ?? "direct"; + + if (input.provider === "anthropic") { + if (transport === "proxy") { + if (!input.proxy) { + throw new Error( + "Missing proxy client configuration for proxy transport", + ); + } + return { + openai: new OpenAI({ apiKey: "unused" }), + anthropic: new Anthropic({ + apiKey: "unused", + baseURL: input.proxy.baseURL, + defaultHeaders: buildProxyHeaders( + input.proxy.bearerToken, + input.proxy.resourcePath, + ), + }), + }; + } + return { openai: new OpenAI({ apiKey: "unused" }), - anthropic: new Anthropic({ apiKey }), + anthropic: new Anthropic({ apiKey: input.apiKey }), + }; + } + + if (transport === "proxy") { + if (!input.proxy) { + throw new Error("Missing proxy client configuration for proxy transport"); + } + return { + openai: new OpenAI(buildProxyOpenAIClientOptions(input.proxy)), + anthropic: new Anthropic({ apiKey: "unused" }), }; } return { - openai: new OpenAI(buildOpenAICompatibleClientOptions(provider, apiKey)), + openai: new OpenAI( + buildOpenAICompatibleClientOptions(input.provider, input.apiKey), + ), anthropic: new Anthropic({ apiKey: "unused" }), }; } export function resolveEvalModelProvider( model: string, - provider?: FrontendEvalProvider + provider?: FrontendEvalProvider, ): ResolvedEvalModelProvider { if (provider) { return { provider, model }; diff --git a/ai_evals/adapters/frontend/core/shared/types.ts b/ai_evals/adapters/frontend/core/shared/types.ts index 4bc3a49b3c..f2a3f04794 100644 --- a/ai_evals/adapters/frontend/core/shared/types.ts +++ b/ai_evals/adapters/frontend/core/shared/types.ts @@ -1,32 +1,38 @@ -import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs' -import type { AIProvider } from '$lib/gen/types.gen' +import type { ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs"; +import type { AIProvider } from "$lib/gen/types.gen"; +import type { FrontendEvalTransport } from "../../../../core/frontendTransport"; +import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; export interface TokenUsage { - prompt: number - completion: number - total: number + prompt: number; + completion: number; + total: number; } export interface ToolCallDetail { - name: string - arguments: Record + name: string; + arguments: Record; } export interface EvalRunnerOptions { - maxIterations?: number - model?: string - workspace?: string - provider?: AIProvider + maxIterations?: number; + model?: string; + workspace?: string; + provider?: AIProvider; + transport?: FrontendEvalTransport; + backend?: WindmillBackendSettings; + proxyCaseId?: string; + proxyAttempt?: number; } export interface RawEvalResult { - success: boolean - output: TOutput - error?: string - tokenUsage: TokenUsage - toolCallsCount: number - toolsCalled: string[] - toolCallDetails: ToolCallDetail[] - iterations: number - messages: ChatCompletionMessageParam[] + success: boolean; + output: TOutput; + error?: string; + tokenUsage: TokenUsage; + toolCallsCount: number; + toolsCalled: string[]; + toolCallDetails: ToolCallDetail[]; + iterations: number; + messages: ChatCompletionMessageParam[]; } diff --git a/ai_evals/adapters/frontend/runtime.ts b/ai_evals/adapters/frontend/runtime.ts index fe40c0f032..6a76fceeb0 100644 --- a/ai_evals/adapters/frontend/runtime.ts +++ b/ai_evals/adapters/frontend/runtime.ts @@ -1,213 +1,228 @@ -import { spawn } from 'node:child_process' -import { mkdtemp, readFile, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import path from 'node:path' -import { fileURLToPath } from 'node:url' +import { spawn } from "node:child_process"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; import { - formatFrontendBenchmarkProgressEvent, - parseFrontendBenchmarkProgressLine -} from './progress' -import type { BenchmarkRunResult } from '../../core/types' + formatFrontendBenchmarkProgressEvent, + parseFrontendBenchmarkProgressLine, +} from "./progress"; +import type { BenchmarkRunResult } from "../../core/types"; -const REPO_ROOT = fileURLToPath(new URL('../../../', import.meta.url)) -const FRONTEND_DIR = path.join(REPO_ROOT, 'frontend') -const FRONTEND_BENCHMARK_TEST = '../ai_evals/adapters/frontend/vitestAdapter.test.ts' -const FRONTEND_BENCHMARK_CONFIG = '../ai_evals/adapters/frontend/vitest.config.ts' +const REPO_ROOT = fileURLToPath(new URL("../../../", import.meta.url)); +const FRONTEND_DIR = path.join(REPO_ROOT, "frontend"); +const FRONTEND_BENCHMARK_TEST = + "../ai_evals/adapters/frontend/vitestAdapter.test.ts"; +const FRONTEND_BENCHMARK_CONFIG = + "../ai_evals/adapters/frontend/vitest.config.ts"; -export type FrontendMode = 'flow' | 'app' | 'script' +export type FrontendMode = "flow" | "app" | "script"; export async function runFrontendBenchmarkAdapter(input: { - mode: FrontendMode - caseIds: string[] - runs: number - model?: string - verbose?: boolean - backendValidation?: string + mode: FrontendMode; + caseIds: string[]; + runs: number; + model?: string; + transport?: string; + verbose?: boolean; + backendValidation?: string; }): Promise { - const tempDir = await mkdtemp(path.join(tmpdir(), 'wmill-frontend-benchmark-')) - const outputPath = path.join(tempDir, 'result.json') + const tempDir = await mkdtemp( + path.join(tmpdir(), "wmill-frontend-benchmark-"), + ); + const outputPath = path.join(tempDir, "result.json"); + const env: NodeJS.ProcessEnv = { + ...process.env, + BROWSERSLIST_IGNORE_OLD_DATA: "1", + WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH: outputPath, + WMILL_FRONTEND_AI_EVAL_MODE: input.mode, + WMILL_FRONTEND_AI_EVAL_CASE_IDS: JSON.stringify(input.caseIds), + WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs), + WMILL_FRONTEND_AI_EVAL_MODEL: input.model ?? "", + WMILL_FRONTEND_AI_EVAL_PROGRESS: "1", + WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? "1" : "0", + WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION: input.backendValidation ?? "", + }; - try { - await runVitestBenchmark( - path.join(FRONTEND_DIR, 'node_modules', '.bin', 'vitest'), - [ - 'run', - FRONTEND_BENCHMARK_TEST, - '--project', - 'server', - '--config', - FRONTEND_BENCHMARK_CONFIG - ], - { - cwd: FRONTEND_DIR, - env: { - ...process.env, - BROWSERSLIST_IGNORE_OLD_DATA: '1', - WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH: outputPath, - WMILL_FRONTEND_AI_EVAL_MODE: input.mode, - WMILL_FRONTEND_AI_EVAL_CASE_IDS: JSON.stringify(input.caseIds), - WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs), - WMILL_FRONTEND_AI_EVAL_MODEL: input.model ?? "", - WMILL_FRONTEND_AI_EVAL_PROGRESS: '1', - WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? '1' : '0', - WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION: input.backendValidation ?? '' - } - } - ) + if (input.transport) { + env.WMILL_FRONTEND_AI_EVAL_TRANSPORT = input.transport; + } - const raw = await readFile(outputPath, 'utf8') - return JSON.parse(raw) as BenchmarkRunResult - } catch (error) { - throw new Error(`Frontend benchmark adapter failed:\n${toErrorMessage(error)}`) - } finally { - await rm(tempDir, { recursive: true, force: true }) - } + try { + await runVitestBenchmark( + path.join(FRONTEND_DIR, "node_modules", ".bin", "vitest"), + [ + "run", + FRONTEND_BENCHMARK_TEST, + "--project", + "server", + "--config", + FRONTEND_BENCHMARK_CONFIG, + ], + { + cwd: FRONTEND_DIR, + env, + }, + ); + + const raw = await readFile(outputPath, "utf8"); + return JSON.parse(raw) as BenchmarkRunResult; + } catch (error) { + throw new Error( + `Frontend benchmark adapter failed:\n${toErrorMessage(error)}`, + ); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } } async function runVitestBenchmark( - command: string, - args: string[], - options: { - cwd: string - env: NodeJS.ProcessEnv - } + command: string, + args: string[], + options: { + cwd: string; + env: NodeJS.ProcessEnv; + }, ): Promise { - const child = spawn(command, args, { - cwd: options.cwd, - env: options.env, - stdio: ['ignore', 'pipe', 'pipe'] - }) + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env, + stdio: ["ignore", "pipe", "pipe"], + }); - let stdout = '' - let stderr = '' - let stderrLineBuffer = '' - let assistantStreamOpen = false + let stdout = ""; + let stderr = ""; + let stderrLineBuffer = ""; + let assistantStreamOpen = false; - child.stdout?.setEncoding('utf8') - child.stdout?.on('data', (chunk: string) => { - stdout += chunk - }) + child.stdout?.setEncoding("utf8"); + child.stdout?.on("data", (chunk: string) => { + stdout += chunk; + }); - child.stderr?.setEncoding('utf8') - child.stderr?.on('data', (chunk: string) => { - stderrLineBuffer += chunk - const { remainder, passthrough, nextAssistantStreamOpen } = drainProgressLines( - stderrLineBuffer, - assistantStreamOpen - ) - stderrLineBuffer = remainder - stderr += passthrough - assistantStreamOpen = nextAssistantStreamOpen - }) + child.stderr?.setEncoding("utf8"); + child.stderr?.on("data", (chunk: string) => { + stderrLineBuffer += chunk; + const { remainder, passthrough, nextAssistantStreamOpen } = + drainProgressLines(stderrLineBuffer, assistantStreamOpen); + stderrLineBuffer = remainder; + stderr += passthrough; + assistantStreamOpen = nextAssistantStreamOpen; + }); - await new Promise((resolve, reject) => { - child.on('error', reject) - child.on('close', (code) => { - if (stderrLineBuffer.length > 0) { - const { - remainder, - passthrough, - nextAssistantStreamOpen - } = drainProgressLines(`${stderrLineBuffer}\n`, assistantStreamOpen) - stderrLineBuffer = remainder - stderr += passthrough - assistantStreamOpen = nextAssistantStreamOpen - } + await new Promise((resolve, reject) => { + child.on("error", reject); + child.on("close", (code) => { + if (stderrLineBuffer.length > 0) { + const { remainder, passthrough, nextAssistantStreamOpen } = + drainProgressLines(`${stderrLineBuffer}\n`, assistantStreamOpen); + stderrLineBuffer = remainder; + stderr += passthrough; + assistantStreamOpen = nextAssistantStreamOpen; + } - if (code === 0) { - if (assistantStreamOpen) { - process.stderr.write('\n') - } - resolve() - return - } + if (code === 0) { + if (assistantStreamOpen) { + process.stderr.write("\n"); + } + resolve(); + return; + } - const details = [`vitest exited with code ${code}`, stdout, stderr].filter(Boolean).join('\n') - reject(new Error(details)) - }) - }) + const details = [`vitest exited with code ${code}`, stdout, stderr] + .filter(Boolean) + .join("\n"); + reject(new Error(details)); + }); + }); } function drainProgressLines( - buffer: string, - initialAssistantStreamOpen: boolean + buffer: string, + initialAssistantStreamOpen: boolean, ): { - remainder: string - passthrough: string - nextAssistantStreamOpen: boolean + remainder: string; + passthrough: string; + nextAssistantStreamOpen: boolean; } { - let remainder = buffer - let passthrough = '' - let assistantStreamOpen = initialAssistantStreamOpen + let remainder = buffer; + let passthrough = ""; + let assistantStreamOpen = initialAssistantStreamOpen; - while (true) { - const newlineIndex = remainder.indexOf('\n') - if (newlineIndex === -1) { - return { remainder, passthrough, nextAssistantStreamOpen: assistantStreamOpen } - } + while (true) { + const newlineIndex = remainder.indexOf("\n"); + if (newlineIndex === -1) { + return { + remainder, + passthrough, + nextAssistantStreamOpen: assistantStreamOpen, + }; + } - const line = remainder.slice(0, newlineIndex).replace(/\r$/, '') - remainder = remainder.slice(newlineIndex + 1) + const line = remainder.slice(0, newlineIndex).replace(/\r$/, ""); + remainder = remainder.slice(newlineIndex + 1); - const progressEvent = parseFrontendBenchmarkProgressLine(line) - if (progressEvent) { - if (progressEvent.type === 'assistant-message-start') { - if (assistantStreamOpen) { - process.stderr.write('\n') - } - process.stderr.write( - `${formatCasePrefix(progressEvent.caseNumber, progressEvent.totalCases)} ${progressEvent.caseId} attempt ${progressEvent.attempt}/${progressEvent.runs} assistant:\n` - ) - assistantStreamOpen = true - continue - } + const progressEvent = parseFrontendBenchmarkProgressLine(line); + if (progressEvent) { + if (progressEvent.type === "assistant-message-start") { + if (assistantStreamOpen) { + process.stderr.write("\n"); + } + process.stderr.write( + `${formatCasePrefix(progressEvent.caseNumber, progressEvent.totalCases)} ${progressEvent.caseId} attempt ${progressEvent.attempt}/${progressEvent.runs} assistant:\n`, + ); + assistantStreamOpen = true; + continue; + } - if (progressEvent.type === 'assistant-chunk') { - process.stderr.write(progressEvent.chunk) - continue - } + if (progressEvent.type === "assistant-chunk") { + process.stderr.write(progressEvent.chunk); + continue; + } - if (progressEvent.type === 'assistant-message-end') { - if (assistantStreamOpen) { - process.stderr.write('\n') - } - assistantStreamOpen = false - continue - } + if (progressEvent.type === "assistant-message-end") { + if (assistantStreamOpen) { + process.stderr.write("\n"); + } + assistantStreamOpen = false; + continue; + } - if (assistantStreamOpen) { - process.stderr.write('\n') - assistantStreamOpen = false - } - process.stderr.write(`${formatFrontendBenchmarkProgressEvent(progressEvent)}\n`) - continue - } + if (assistantStreamOpen) { + process.stderr.write("\n"); + assistantStreamOpen = false; + } + process.stderr.write( + `${formatFrontendBenchmarkProgressEvent(progressEvent)}\n`, + ); + continue; + } - if (shouldSuppressFrontendStderrLine(line)) { - continue - } + if (shouldSuppressFrontendStderrLine(line)) { + continue; + } - passthrough += `${line}\n` - process.stderr.write(`${line}\n`) - } + passthrough += `${line}\n`; + process.stderr.write(`${line}\n`); + } } function formatCasePrefix(caseNumber: number, totalCases: number): string { - return `[${caseNumber}/${totalCases}]` + return `[${caseNumber}/${totalCases}]`; } function shouldSuppressFrontendStderrLine(line: string): boolean { - return ( - line.startsWith('[baseline-browser-mapping] ') || - line.startsWith('Browserslist: browsers data (caniuse-lite) is ') || - line.includes('update-browserslist-db@latest') || - line.includes('update-db#readme') - ) + return ( + line.startsWith("[baseline-browser-mapping] ") || + line.startsWith("Browserslist: browsers data (caniuse-lite) is ") || + line.includes("update-browserslist-db@latest") || + line.includes("update-db#readme") + ); } function toErrorMessage(error: unknown): string { - if (error instanceof Error) { - return error.message - } - return String(error) + if (error instanceof Error) { + return error.message; + } + return String(error); } diff --git a/ai_evals/adapters/frontend/windmillBackend.ts b/ai_evals/adapters/frontend/windmillBackend.ts new file mode 100644 index 0000000000..c8d9537779 --- /dev/null +++ b/ai_evals/adapters/frontend/windmillBackend.ts @@ -0,0 +1,184 @@ +import { randomUUID } from "node:crypto"; +import type { WindmillBackendSettings } from "../../core/windmillBackendSettings"; + +const tokenCache = new Map>(); +const sharedWorkspaceQueue = new Map>(); + +export class WindmillBackendClient { + constructor(private readonly settings: WindmillBackendSettings) {} + + async withWorkspace( + caseId: string, + attempt: number, + body: (workspaceId: string) => Promise, + ): Promise { + const workspaceId = + this.settings.workspaceOverride ?? + buildWorkspaceId(this.settings.workspacePrefix, caseId, attempt); + + const run = async () => { + await this.ensureWorkspace(workspaceId); + + try { + return await body(workspaceId); + } finally { + if (!this.settings.keepWorkspaces && !this.settings.workspaceOverride) { + await this.deleteWorkspace(workspaceId).catch(() => undefined); + } + } + }; + + if (this.settings.workspaceOverride) { + return await withSharedWorkspaceLock(workspaceId, run); + } + + return await run(); + } + + async request(path: string, init?: RequestInit): Promise { + const token = await this.getToken(); + return await fetch(`${this.settings.baseUrl}/api${path}`, { + ...init, + headers: { + Authorization: `Bearer ${token}`, + ...(init?.headers ?? {}), + }, + }); + } + + async getToken(): Promise { + const cacheKey = `${this.settings.baseUrl}|${this.settings.email}`; + let tokenPromise = tokenCache.get(cacheKey); + if (!tokenPromise) { + tokenPromise = this.login().catch((error) => { + if (tokenCache.get(cacheKey) === tokenPromise) { + tokenCache.delete(cacheKey); + } + throw error; + }); + tokenCache.set(cacheKey, tokenPromise); + } + return await tokenPromise; + } + + async upsertResource(input: { + workspaceId: string; + path: string; + resourceType: string; + value: Record; + }): Promise { + const response = await this.request( + `/w/${encodeURIComponent(input.workspaceId)}/resources/create?update_if_exists=true`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: input.path, + resource_type: input.resourceType, + value: input.value, + }), + }, + ); + await expectOk(response, `upsert resource ${input.path}`); + } + + private async ensureWorkspace(workspaceId: string): Promise { + const existsResponse = await this.request("/workspaces/exists", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ id: workspaceId }), + }); + await expectOk(existsResponse, `check workspace ${workspaceId}`); + + if ((await existsResponse.text()).trim() === "true") { + return; + } + + const createResponse = await this.request("/workspaces/create", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ id: workspaceId, name: workspaceId }), + }); + try { + await expectOk(createResponse, `create workspace ${workspaceId}`); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (message.includes("maximum number of workspaces")) { + throw new Error( + `${message}. Reuse an existing workspace with WMILL_AI_EVAL_BACKEND_WORKSPACE=.`, + ); + } + throw error; + } + } + + private async deleteWorkspace(workspaceId: string): Promise { + const response = await this.request( + `/workspaces/delete/${encodeURIComponent(workspaceId)}`, + { + method: "DELETE", + }, + ); + await expectOk(response, `delete workspace ${workspaceId}`); + } + + private async login(): Promise { + const response = await fetch(`${this.settings.baseUrl}/api/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + email: this.settings.email, + password: this.settings.password, + }), + }); + await expectOk(response, "login to Windmill backend"); + return (await response.text()).trim(); + } +} + +async function withSharedWorkspaceLock( + workspaceId: string, + body: () => Promise, +): Promise { + const previous = sharedWorkspaceQueue.get(workspaceId) ?? Promise.resolve(); + let releaseCurrent: (() => void) | undefined; + const current = new Promise((resolve) => { + releaseCurrent = resolve; + }); + const tail = previous.catch(() => undefined).then(() => current); + sharedWorkspaceQueue.set(workspaceId, tail); + + await previous.catch(() => undefined); + + try { + return await body(); + } finally { + releaseCurrent?.(); + if (sharedWorkspaceQueue.get(workspaceId) === tail) { + sharedWorkspaceQueue.delete(workspaceId); + } + } +} + +function buildWorkspaceId( + prefix: string, + caseId: string, + attempt: number, +): string { + const caseSlug = caseId + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 30); + const suffix = randomUUID().slice(0, 8); + return `${prefix}-${caseSlug || "case"}-a${attempt}-${suffix}`; +} + +async function expectOk(response: Response, context: string): Promise { + if (response.ok) { + return; + } + throw new Error( + `${context} failed: ${response.status} ${response.statusText} - ${await response.text()}`, + ); +} diff --git a/ai_evals/cli/index.ts b/ai_evals/cli/index.ts index 8cd5129360..202d826078 100644 --- a/ai_evals/cli/index.ts +++ b/ai_evals/cli/index.ts @@ -27,11 +27,18 @@ import { EVAL_MODES, type EvalMode } from "../core/types"; import { DEFAULT_JUDGE_MODEL } from "../core/judge"; import { createCliModeRunner } from "../modes/cli"; import { runFrontendBenchmarkAdapter } from "../adapters/frontend/runtime"; +import { + FRONTEND_EVAL_TRANSPORTS, + type FrontendEvalTransport, + parseFrontendEvalTransport, +} from "../core/frontendTransport"; async function main() { const program = new Command() .name("bun run cli --") - .description("Run AI eval cases against the current production prompts and guidance") + .description( + "Run AI eval cases against the current production prompts and guidance", + ) .showHelpAfterError() .showSuggestionAfterError() .addHelpText( @@ -53,7 +60,7 @@ async function main() { "", "Models:", getEvalModelHelpText(), - ].join("\n") + ].join("\n"), ); program @@ -76,15 +83,33 @@ async function main() { .description("Run one benchmark mode") .argument("", "cli, flow, script, or app", parseMode) .argument("[caseIds...]", "specific case ids to run") - .option("--runs ", "number of attempts per case", parsePositiveInteger, 1) + .option( + "--runs ", + "number of attempts per case", + parsePositiveInteger, + 1, + ) .option("--output ", "write the result JSON to this path") - .option("--model ", `model alias (${EVAL_MODELS.map((entry) => entry.id).join(", ")})`) - .option("--models ", "comma-separated model aliases to run sequentially") + .option( + "--model ", + `model alias (${EVAL_MODELS.map((entry) => entry.id).join(", ")})`, + ) + .option( + "--models ", + "comma-separated model aliases to run sequentially", + ) + .option( + "--transport ", + `frontend transport (${FRONTEND_EVAL_TRANSPORTS.join(", ")})`, + ) .option("--verbose", "stream assistant output during frontend runs") - .option("--record", "append a compact summary line to ai_evals/history/.jsonl") + .option( + "--record", + "append a compact summary line to ai_evals/history/.jsonl", + ) .option( "--backend-validation ", - `backend smoke validation (${BACKEND_VALIDATION_MODES.join(", ")})` + `backend smoke validation (${BACKEND_VALIDATION_MODES.join(", ")})`, ) .action( async ( @@ -95,10 +120,11 @@ async function main() { output?: string; model?: string; models?: string; + transport?: string; verbose?: boolean; record?: boolean; backendValidation?: string; - } + }, ) => { await handleRun({ mode, @@ -107,11 +133,14 @@ async function main() { outputPath: options.output, model: options.model, models: options.models, + transport: options.transport + ? parseFrontendEvalTransport(options.transport) + : undefined, verbose: options.verbose ?? false, record: options.record ?? false, backendValidation: options.backendValidation, }); - } + }, ); await program.parseAsync(process.argv); @@ -137,7 +166,10 @@ function handleModels() { ...(model.frontend ? ["flow", "script", "app"] : []), ...(model.cli ? ["cli"] : []), ]; - const aliases = [model.id, ...model.aliases.filter((alias) => alias !== model.id)]; + const aliases = [ + model.id, + ...model.aliases.filter((alias) => alias !== model.id), + ]; process.stdout.write(`- ${model.id}: ${model.label}\n`); process.stdout.write(` aliases: ${aliases.join(", ")}\n`); process.stdout.write(` modes: ${supports.join(", ")}\n`); @@ -152,48 +184,72 @@ async function handleRun(input: { outputPath?: string; model?: string; models?: string; + transport?: FrontendEvalTransport; verbose: boolean; record: boolean; backendValidation?: string; }) { if (input.record && input.caseIds.length > 0) { - throw new Error("--record only supports full-suite runs; omit case ids to record history"); + throw new Error( + "--record only supports full-suite runs; omit case ids to record history", + ); } if (input.model && input.models) { throw new Error("Use either --model or --models, not both"); } + if (input.mode === "cli" && input.transport === "proxy") { + throw new Error( + "--transport proxy is only supported for flow, script, and app modes", + ); + } const selectedCases = await loadSelectedCases(input.mode, input.caseIds); const models = resolveRequestedModels(input.mode, input.model, input.models); const backendValidation = parseBackendValidationMode( - input.backendValidation ?? process.env.WMILL_AI_EVAL_BACKEND_VALIDATION + input.backendValidation ?? process.env.WMILL_AI_EVAL_BACKEND_VALIDATION, ); if (input.outputPath && models.length > 1) { throw new Error("--output only supports a single model run"); } - if (backendValidation !== "off" && input.mode !== "flow" && input.mode !== "script") { - throw new Error("--backend-validation currently supports only flow and script modes"); + if ( + backendValidation !== "off" && + input.mode !== "flow" && + input.mode !== "script" + ) { + throw new Error( + "--backend-validation currently supports only flow and script modes", + ); } - const summaries: Array<{ label: string; passRate: number; averageDurationMs: number }> = []; + const summaries: Array<{ + label: string; + passRate: number; + averageDurationMs: number; + }> = []; for (const [index, model] of models.entries()) { const runModel = formatRunModelLabel(input.mode, model); if (models.length > 1) { process.stdout.write( - `${index > 0 ? "\n" : ""}=== ${input.mode} ${model.id} (${runModel}) ===\n` + `${index > 0 ? "\n" : ""}=== ${input.mode} ${model.id} (${runModel}) ===\n`, ); } process.stderr.write(`Starting ${input.mode} benchmark...\n`); const result = input.mode === "cli" - ? await runCliBenchmark(selectedCases, input.runs, getCliEvalModel(model), runModel) + ? await runCliBenchmark( + selectedCases, + input.runs, + getCliEvalModel(model), + runModel, + ) : await runFrontendBenchmarkAdapter({ mode: input.mode, caseIds: input.caseIds, runs: input.runs, model: model.id, + transport: input.transport, verbose: input.verbose, backendValidation, }); @@ -225,7 +281,7 @@ async function handleRun(input: { process.stdout.write("\nModel summary\n"); for (const summary of summaries) { process.stdout.write( - `- ${summary.label}: ${formatPercent(summary.passRate)} | ${Math.round(summary.averageDurationMs)}ms\n` + `- ${summary.label}: ${formatPercent(summary.passRate)} | ${Math.round(summary.averageDurationMs)}ms\n`, ); } } @@ -235,7 +291,7 @@ async function runCliBenchmark( cases: Awaited>, runs: number, model: ReturnType, - runModel: string + runModel: string, ) { const caseResults = await runSuite({ modeRunner: createCliModeRunner(model), @@ -258,7 +314,9 @@ function parseMode(value: string): EvalMode { if (EVAL_MODES.includes(value as EvalMode)) { return value as EvalMode; } - throw new InvalidArgumentError(`mode must be one of: ${EVAL_MODES.join(", ")}`); + throw new InvalidArgumentError( + `mode must be one of: ${EVAL_MODES.join(", ")}`, + ); } function parseOptionalMode(value: string | undefined): EvalMode | undefined { @@ -276,7 +334,7 @@ function parsePositiveInteger(value: string): number { function resolveRequestedModels( mode: EvalMode, singleModel?: string, - multipleModels?: string + multipleModels?: string, ): EvalModelSpec[] { if (!multipleModels) { return [resolveEvalModel(mode, singleModel)]; diff --git a/ai_evals/core/appDiagnostics.ts b/ai_evals/core/appDiagnostics.ts index e5b1dd7ef6..249473ad74 100644 --- a/ai_evals/core/appDiagnostics.ts +++ b/ai_evals/core/appDiagnostics.ts @@ -7,7 +7,14 @@ const BACKEND_ROOT = "/__ai_evals__/backend"; const FRONTEND_REACT_SHIM_PATH = `${FRONTEND_ROOT}/__react_shim__.d.ts`; const FRONTEND_WMILL_TYPES_PATH = `${FRONTEND_ROOT}/wmill.d.ts`; const BACKEND_WINDMILL_CLIENT_SHIM_PATH = `${BACKEND_ROOT}/__windmill_client__.d.ts`; -const TS_LIKE_LANGUAGES = new Set(["bun", "deno", "nativets", "bunnative", "ts", "typescript"]); +const TS_LIKE_LANGUAGES = new Set([ + "bun", + "deno", + "nativets", + "bunnative", + "ts", + "typescript", +]); const JS_LIKE_LANGUAGES = new Set(["javascript", "js", "nodejs"]); const SAFE_TYPE_REFERENCE_NAMES = new Set([ "Array", @@ -121,20 +128,26 @@ export interface AppDiagnosticsResult { } export function buildAppWmillTypes( - backend: Record = {} + backend: Record = {}, ): string { return `// THIS FILE IS READ-ONLY // AND GENERATED AUTOMATICALLY FROM YOUR RUNNABLES export declare const backend: { ${Object.entries(backend) - .map(([key, runnable]) => ` ${JSON.stringify(key)}: ${getRunnableSignature(runnable, false)};`) + .map( + ([key, runnable]) => + ` ${JSON.stringify(key)}: ${getRunnableSignature(runnable, false)};`, + ) .join("\n")} }; export declare const backendAsync: { ${Object.entries(backend) - .map(([key, runnable]) => ` ${JSON.stringify(key)}: ${getRunnableSignature(runnable, true)};`) + .map( + ([key, runnable]) => + ` ${JSON.stringify(key)}: ${getRunnableSignature(runnable, true)};`, + ) .join("\n")} }; @@ -165,16 +178,26 @@ export function collectAppDiagnostics(input: { frontend: Record; backend: Record; }): AppDiagnosticsResult { - const frontendDiagnostics = collectFrontendDiagnostics(input.frontend, input.backend); + const frontendDiagnostics = collectFrontendDiagnostics( + input.frontend, + input.backend, + ); const backendDiagnostics = collectBackendDiagnostics(input.backend); - const diagnostics = dedupeDiagnostics([...frontendDiagnostics, ...backendDiagnostics]).sort(compareDiagnostics); + const diagnostics = dedupeDiagnostics([ + ...frontendDiagnostics, + ...backendDiagnostics, + ]).sort(compareDiagnostics); return { diagnostics, lintResult: { errors: { - frontend: groupMessages(diagnostics.filter((diagnostic) => diagnostic.source === "frontend")), - backend: groupMessages(diagnostics.filter((diagnostic) => diagnostic.source === "backend")), + frontend: groupMessages( + diagnostics.filter((diagnostic) => diagnostic.source === "frontend"), + ), + backend: groupMessages( + diagnostics.filter((diagnostic) => diagnostic.source === "backend"), + ), }, warnings: { frontend: {}, @@ -188,20 +211,26 @@ export function collectAppDiagnostics(input: { function collectFrontendDiagnostics( frontend: Record, - backend: Record + backend: Record, ): AppStaticDiagnostic[] { const frontendFiles = Object.entries(frontend) .filter(([filePath]) => isFrontendCodeFile(filePath)) - .map(([filePath, content]) => [toFrontendVirtualPath(filePath), content] as const); + .map( + ([filePath, content]) => + [toFrontendVirtualPath(filePath), content] as const, + ); const virtualFiles = new Map([ [FRONTEND_REACT_SHIM_PATH, FRONTEND_REACT_SHIM], - [FRONTEND_WMILL_TYPES_PATH, wrapModuleDeclaration("wmill", buildAppWmillTypes(backend))], + [ + FRONTEND_WMILL_TYPES_PATH, + wrapModuleDeclaration("wmill", buildAppWmillTypes(backend)), + ], ...frontendFiles, ]); const host = createVirtualCompilerHost( virtualFiles, - getFrontendCompilerOptions() + getFrontendCompilerOptions(), ); const rootNames = [...virtualFiles.keys()]; const program = ts.createProgram({ @@ -210,9 +239,8 @@ function collectFrontendDiagnostics( host, }); - return ts - .getPreEmitDiagnostics(program) - .flatMap((diagnostic) => mapTypeScriptDiagnostic({ + return ts.getPreEmitDiagnostics(program).flatMap((diagnostic) => + mapTypeScriptDiagnostic({ diagnostic, source: "frontend", toTarget(fileName) { @@ -228,18 +256,22 @@ function collectFrontendDiagnostics( } return normalized.slice(FRONTEND_ROOT.length); }, - })); + }), + ); } function collectBackendDiagnostics( - backend: Record + backend: Record, ): AppStaticDiagnostic[] { const backendFiles = Object.entries(backend) .filter(([, runnable]) => isTypeCheckableBackendRunnable(runnable)) - .map(([key, runnable]) => [ - `${BACKEND_ROOT}/${key}/main.${getBackendFileExtension(runnable.inlineScript?.language)}`, - runnable.inlineScript?.content ?? "", - ] as const); + .map( + ([key, runnable]) => + [ + `${BACKEND_ROOT}/${key}/main.${getBackendFileExtension(runnable.inlineScript?.language)}`, + runnable.inlineScript?.content ?? "", + ] as const, + ); if (backendFiles.length === 0) { return []; @@ -251,7 +283,7 @@ function collectBackendDiagnostics( ]); const host = createVirtualCompilerHost( virtualFiles, - getBackendCompilerOptions() + getBackendCompilerOptions(), ); const rootNames = [...virtualFiles.keys()]; const program = ts.createProgram({ @@ -260,9 +292,8 @@ function collectBackendDiagnostics( host, }); - return ts - .getPreEmitDiagnostics(program) - .flatMap((diagnostic) => mapTypeScriptDiagnostic({ + return ts.getPreEmitDiagnostics(program).flatMap((diagnostic) => + mapTypeScriptDiagnostic({ diagnostic, source: "backend", toTarget(fileName) { @@ -277,7 +308,8 @@ function collectBackendDiagnostics( const runnableKey = relativePath.split("/")[0]; return runnableKey || null; }, - })); + }), + ); } function getFrontendCompilerOptions(): ts.CompilerOptions { @@ -317,25 +349,37 @@ function getBackendCompilerOptions(): ts.CompilerOptions { function createVirtualCompilerHost( files: Map, - options: ts.CompilerOptions + options: ts.CompilerOptions, ): ts.CompilerHost { const originalHost = ts.createCompilerHost(options, true); const originalGetSourceFile = originalHost.getSourceFile.bind(originalHost); const originalReadFile = originalHost.readFile.bind(originalHost); const originalFileExists = originalHost.fileExists.bind(originalHost); - const originalDirectoryExists = originalHost.directoryExists?.bind(originalHost); - const originalGetDirectories = originalHost.getDirectories?.bind(originalHost); + const originalDirectoryExists = + originalHost.directoryExists?.bind(originalHost); + const originalGetDirectories = + originalHost.getDirectories?.bind(originalHost); return { ...originalHost, getCurrentDirectory: () => "/", - getSourceFile(fileName, languageVersion, onError, shouldCreateNewSourceFile) { + getSourceFile( + fileName, + languageVersion, + onError, + shouldCreateNewSourceFile, + ) { const normalized = normalizeFileName(fileName); const content = files.get(normalized); if (content !== undefined) { return ts.createSourceFile(fileName, content, languageVersion, true); } - return originalGetSourceFile(fileName, languageVersion, onError, shouldCreateNewSourceFile); + return originalGetSourceFile( + fileName, + languageVersion, + onError, + shouldCreateNewSourceFile, + ); }, readFile(fileName) { const normalized = normalizeFileName(fileName); @@ -347,7 +391,11 @@ function createVirtualCompilerHost( }, directoryExists(dirName) { const normalized = normalizeFileName(dirName); - return hasVirtualDirectory(files, normalized) || originalDirectoryExists?.(dirName) || false; + return ( + hasVirtualDirectory(files, normalized) || + originalDirectoryExists?.(dirName) || + false + ); }, getDirectories(dirName) { const normalized = normalizeFileName(dirName); @@ -386,7 +434,9 @@ function mapTypeScriptDiagnostic(input: { { source, target, - message: ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n").trim(), + message: ts + .flattenDiagnosticMessageText(diagnostic.messageText, "\n") + .trim(), line: position ? position.line + 1 : undefined, column: position ? position.character + 1 : undefined, code: diagnostic.code, @@ -395,7 +445,7 @@ function mapTypeScriptDiagnostic(input: { } function groupMessages( - diagnostics: AppStaticDiagnostic[] + diagnostics: AppStaticDiagnostic[], ): Record { const grouped: Record = {}; @@ -414,7 +464,9 @@ function formatLintMessage(diagnostic: AppStaticDiagnostic): string { return diagnostic.message; } -function dedupeDiagnostics(diagnostics: AppStaticDiagnostic[]): AppStaticDiagnostic[] { +function dedupeDiagnostics( + diagnostics: AppStaticDiagnostic[], +): AppStaticDiagnostic[] { const uniqueDiagnostics = new Map(); for (const diagnostic of diagnostics) { @@ -434,7 +486,10 @@ function dedupeDiagnostics(diagnostics: AppStaticDiagnostic[]): AppStaticDiagnos return [...uniqueDiagnostics.values()]; } -function compareDiagnostics(a: AppStaticDiagnostic, b: AppStaticDiagnostic): number { +function compareDiagnostics( + a: AppStaticDiagnostic, + b: AppStaticDiagnostic, +): number { if (a.source !== b.source) { return a.source.localeCompare(b.source); } @@ -464,7 +519,10 @@ function normalizeFileName(fileName: string): string { return path.posix.normalize(fileName.replace(/\\/g, "/")); } -function hasVirtualDirectory(files: Map, dirName: string): boolean { +function hasVirtualDirectory( + files: Map, + dirName: string, +): boolean { const normalizedDirectory = dirName.endsWith("/") ? dirName : `${dirName}/`; for (const fileName of files.keys()) { if (fileName === dirName || fileName.startsWith(normalizedDirectory)) { @@ -474,7 +532,10 @@ function hasVirtualDirectory(files: Map, dirName: string): boole return false; } -function listVirtualDirectories(files: Map, dirName: string): string[] { +function listVirtualDirectories( + files: Map, + dirName: string, +): string[] { const normalizedDirectory = dirName.endsWith("/") ? dirName : `${dirName}/`; const directories = new Set(); @@ -504,14 +565,16 @@ function wrapModuleDeclaration(moduleName: string, content: string): string { function getRunnableSignature( runnable: AppDiagnosticRunnable | undefined, - asyncMode: boolean + asyncMode: boolean, ): string { const returnType = asyncMode ? "Promise" : "Promise"; const parameter = getRunnableParameterSignature(runnable); return `${parameter} => ${returnType}`; } -function getRunnableParameterSignature(runnable: AppDiagnosticRunnable | undefined): string { +function getRunnableParameterSignature( + runnable: AppDiagnosticRunnable | undefined, +): string { const parameterInfo = getRunnableParameterInfo(runnable); if (!parameterInfo) { return "()"; @@ -525,9 +588,12 @@ function getRunnableParameterSignature(runnable: AppDiagnosticRunnable | undefin } function getRunnableParameterInfo( - runnable: AppDiagnosticRunnable | undefined + runnable: AppDiagnosticRunnable | undefined, ): { typeText?: string; optional: boolean } | null { - if (!runnable?.inlineScript?.content || !isTypeCheckableBackendRunnable(runnable)) { + if ( + !runnable?.inlineScript?.content || + !isTypeCheckableBackendRunnable(runnable) + ) { return { typeText: "any", optional: true }; } @@ -536,7 +602,7 @@ function getRunnableParameterInfo( runnable.inlineScript.content, ts.ScriptTarget.Latest, true, - getScriptKindForLanguage(runnable.inlineScript.language) + getScriptKindForLanguage(runnable.inlineScript.language), ); const mainDeclaration = findExportedMainDeclaration(sourceFile); @@ -545,7 +611,8 @@ function getRunnableParameterInfo( } const [parameter] = mainDeclaration.parameters; - const optional = Boolean(parameter.questionToken) || Boolean(parameter.initializer); + const optional = + Boolean(parameter.questionToken) || Boolean(parameter.initializer); if (!parameter.type || !isPortableTypeNode(parameter.type)) { return { typeText: "any", optional: true }; @@ -558,10 +625,14 @@ function getRunnableParameterInfo( } function findExportedMainDeclaration( - sourceFile: ts.SourceFile + sourceFile: ts.SourceFile, ): ts.SignatureDeclarationBase | null { for (const statement of sourceFile.statements) { - if (ts.isFunctionDeclaration(statement) && statement.name?.text === "main" && hasExportModifier(statement)) { + if ( + ts.isFunctionDeclaration(statement) && + statement.name?.text === "main" && + hasExportModifier(statement) + ) { return statement; } @@ -570,11 +641,18 @@ function findExportedMainDeclaration( } for (const declaration of statement.declarationList.declarations) { - if (!ts.isIdentifier(declaration.name) || declaration.name.text !== "main") { + if ( + !ts.isIdentifier(declaration.name) || + declaration.name.text !== "main" + ) { continue; } const initializer = declaration.initializer; - if (initializer && (ts.isArrowFunction(initializer) || ts.isFunctionExpression(initializer))) { + if ( + initializer && + (ts.isArrowFunction(initializer) || + ts.isFunctionExpression(initializer)) + ) { return initializer; } } @@ -584,7 +662,14 @@ function findExportedMainDeclaration( } function hasExportModifier(node: ts.Node): boolean { - return Boolean(node.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword)); + const modifiers = ts.canHaveModifiers(node) + ? ts.getModifiers(node) + : undefined; + return Boolean( + modifiers?.some( + (modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword, + ), + ); } function isPortableTypeNode(node: ts.TypeNode): boolean { @@ -607,10 +692,15 @@ function isPortableTypeNode(node: ts.TypeNode): boolean { } if (ts.isTypeReferenceNode(node)) { - if (!ts.isIdentifier(node.typeName) || !SAFE_TYPE_REFERENCE_NAMES.has(node.typeName.text)) { + if ( + !ts.isIdentifier(node.typeName) || + !SAFE_TYPE_REFERENCE_NAMES.has(node.typeName.text) + ) { return false; } - return (node.typeArguments ?? []).every((typeArgument) => isPortableTypeNode(typeArgument)); + return (node.typeArguments ?? []).every((typeArgument) => + isPortableTypeNode(typeArgument), + ); } return false; @@ -637,10 +727,17 @@ function isKeywordTypeNode(node: ts.TypeNode): boolean { function isFrontendCodeFile(filePath: string): boolean { const extension = path.posix.extname(filePath).toLowerCase(); - return extension === ".js" || extension === ".jsx" || extension === ".ts" || extension === ".tsx"; + return ( + extension === ".js" || + extension === ".jsx" || + extension === ".ts" || + extension === ".tsx" + ); } -function isTypeCheckableBackendRunnable(runnable: AppDiagnosticRunnable | undefined): boolean { +function isTypeCheckableBackendRunnable( + runnable: AppDiagnosticRunnable | undefined, +): boolean { if (!runnable || runnable.type !== "inline") { return false; } @@ -655,5 +752,7 @@ function getBackendFileExtension(language: string | undefined): string { function getScriptKindForLanguage(language: string | undefined): ts.ScriptKind { const normalizedLanguage = language?.toLowerCase() ?? ""; - return JS_LIKE_LANGUAGES.has(normalizedLanguage) ? ts.ScriptKind.JS : ts.ScriptKind.TS; + return JS_LIKE_LANGUAGES.has(normalizedLanguage) + ? ts.ScriptKind.JS + : ts.ScriptKind.TS; } diff --git a/ai_evals/core/backendValidation.ts b/ai_evals/core/backendValidation.ts index 464be8533a..87094fd23f 100644 --- a/ai_evals/core/backendValidation.ts +++ b/ai_evals/core/backendValidation.ts @@ -1,4 +1,8 @@ import type { EvalMode } from "./types"; +import { + parsePositiveInteger, + resolveWindmillBackendSettings, +} from "./windmillBackendSettings"; export const BACKEND_VALIDATION_MODES = ["off", "preview"] as const; @@ -16,10 +20,17 @@ export interface BackendValidationSettings { maxWaitMs: number; } -export function parseBackendValidationMode(value?: string | null): BackendValidationMode { +export function parseBackendValidationMode( + value?: string | null, +): BackendValidationMode { const normalized = value?.trim().toLowerCase(); - if (!normalized || normalized === "off" || normalized === "false" || normalized === "0") { + if ( + !normalized || + normalized === "off" || + normalized === "false" || + normalized === "0" + ) { return "off"; } @@ -28,7 +39,7 @@ export function parseBackendValidationMode(value?: string | null): BackendValida } throw new Error( - `Unsupported backend validation mode: ${value}. Use one of: ${BACKEND_VALIDATION_MODES.join(", ")}` + `Unsupported backend validation mode: ${value}. Use one of: ${BACKEND_VALIDATION_MODES.join(", ")}`, ); } @@ -37,68 +48,29 @@ export function resolveBackendValidationSettings(input: { requestedMode?: string | null; }): BackendValidationSettings { const mode = parseBackendValidationMode( - input.requestedMode ?? process.env.WMILL_AI_EVAL_BACKEND_VALIDATION + input.requestedMode ?? process.env.WMILL_AI_EVAL_BACKEND_VALIDATION, ); - if (mode !== "off" && input.evalMode !== "flow" && input.evalMode !== "script") { + if ( + mode !== "off" && + input.evalMode !== "flow" && + input.evalMode !== "script" + ) { throw new Error( - `Backend validation mode "${mode}" is only supported for flow and script evals` + `Backend validation mode "${mode}" is only supported for flow and script evals`, ); } return { mode, - baseUrl: normalizeBaseUrl( - process.env.WMILL_AI_EVAL_BACKEND_URL ?? - process.env.WINDMILL_URL ?? - process.env.WINDMILL_BASE_URL ?? - process.env.REMOTE ?? - "http://127.0.0.1:8000" - ), - email: process.env.WMILL_AI_EVAL_BACKEND_EMAIL ?? "admin@windmill.dev", - password: process.env.WMILL_AI_EVAL_BACKEND_PASSWORD ?? "changeme", - keepWorkspaces: isTruthy(process.env.WMILL_AI_EVAL_KEEP_WORKSPACES), - workspaceOverride: sanitizeOptionalWorkspaceId(process.env.WMILL_AI_EVAL_BACKEND_WORKSPACE), - workspacePrefix: sanitizeWorkspacePrefix( - process.env.WMILL_AI_EVAL_WORKSPACE_PREFIX ?? "ai-evals" - ), + ...resolveWindmillBackendSettings(), pollIntervalMs: parsePositiveInteger( process.env.WMILL_AI_EVAL_BACKEND_POLL_INTERVAL_MS, - 2000 + 2000, + ), + maxWaitMs: parsePositiveInteger( + process.env.WMILL_AI_EVAL_BACKEND_MAX_WAIT_MS, + 120000, ), - maxWaitMs: parsePositiveInteger(process.env.WMILL_AI_EVAL_BACKEND_MAX_WAIT_MS, 120000), }; } - -function normalizeBaseUrl(value: string): string { - return value.replace(/\/+$/, ""); -} - -function sanitizeWorkspacePrefix(value: string): string { - const sanitized = value - .trim() - .toLowerCase() - .replace(/[^a-z0-9-]+/g, "-") - .replace(/^-+|-+$/g, ""); - return sanitized.length > 0 ? sanitized : "ai-evals"; -} - -function sanitizeOptionalWorkspaceId(value: string | undefined): string | undefined { - const trimmed = value?.trim(); - return trimmed ? trimmed : undefined; -} - -function isTruthy(value: string | undefined): boolean { - if (!value) { - return false; - } - return ["1", "true", "yes", "on"].includes(value.trim().toLowerCase()); -} - -function parsePositiveInteger(value: string | undefined, fallback: number): number { - if (!value) { - return fallback; - } - const parsed = Number(value); - return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; -} diff --git a/ai_evals/core/frontendTransport.test.ts b/ai_evals/core/frontendTransport.test.ts new file mode 100644 index 0000000000..09ebd1b3a7 --- /dev/null +++ b/ai_evals/core/frontendTransport.test.ts @@ -0,0 +1,64 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { + parseFrontendEvalTransport, + resolveFrontendEvalTransportSettings, +} from "./frontendTransport"; + +const ORIGINAL_ENV = { + WMILL_AI_EVAL_BACKEND_URL: process.env.WMILL_AI_EVAL_BACKEND_URL, +}; + +afterEach(() => { + if (ORIGINAL_ENV.WMILL_AI_EVAL_BACKEND_URL === undefined) { + delete process.env.WMILL_AI_EVAL_BACKEND_URL; + } else { + process.env.WMILL_AI_EVAL_BACKEND_URL = + ORIGINAL_ENV.WMILL_AI_EVAL_BACKEND_URL; + } +}); + +describe("parseFrontendEvalTransport", () => { + it("defaults to direct when unset", () => { + expect(parseFrontendEvalTransport(undefined)).toBe("direct"); + }); + + it("accepts proxy explicitly", () => { + expect(parseFrontendEvalTransport("proxy")).toBe("proxy"); + }); + + it("rejects unsupported values", () => { + expect(() => parseFrontendEvalTransport("worker")).toThrow( + "Unsupported frontend eval transport: worker", + ); + }); +}); + +describe("resolveFrontendEvalTransportSettings", () => { + it("includes backend settings for proxy transport", () => { + process.env.WMILL_AI_EVAL_BACKEND_URL = "http://127.0.0.1:8000/"; + + expect( + resolveFrontendEvalTransportSettings({ + evalMode: "app", + requestedTransport: "proxy", + }), + ).toMatchObject({ + transport: "proxy", + backend: { + baseUrl: "http://127.0.0.1:8000", + }, + }); + }); + + it("keeps direct transport for cli runs", () => { + expect( + resolveFrontendEvalTransportSettings({ + evalMode: "cli", + requestedTransport: "direct", + }), + ).toEqual({ + transport: "direct", + backend: undefined, + }); + }); +}); diff --git a/ai_evals/core/frontendTransport.ts b/ai_evals/core/frontendTransport.ts new file mode 100644 index 0000000000..fa78505113 --- /dev/null +++ b/ai_evals/core/frontendTransport.ts @@ -0,0 +1,49 @@ +import type { EvalMode } from "./types"; +import type { WindmillBackendSettings } from "./windmillBackendSettings"; +import { resolveWindmillBackendSettings } from "./windmillBackendSettings"; + +export const FRONTEND_EVAL_TRANSPORTS = ["direct", "proxy"] as const; + +export type FrontendEvalTransport = (typeof FRONTEND_EVAL_TRANSPORTS)[number]; + +export interface FrontendEvalTransportSettings { + transport: FrontendEvalTransport; + backend?: WindmillBackendSettings; +} + +export function parseFrontendEvalTransport( + value?: string | null, +): FrontendEvalTransport { + const normalized = value?.trim().toLowerCase(); + + if (!normalized || normalized === "direct") { + return "direct"; + } + + if (normalized === "proxy") { + return "proxy"; + } + + throw new Error( + `Unsupported frontend eval transport: ${value}. Use one of: ${FRONTEND_EVAL_TRANSPORTS.join(", ")}`, + ); +} + +export function resolveFrontendEvalTransportSettings(input: { + evalMode: EvalMode; + requestedTransport?: string | null; +}): FrontendEvalTransportSettings { + const transport = parseFrontendEvalTransport(input.requestedTransport); + + if (transport === "proxy" && input.evalMode === "cli") { + throw new Error( + 'Frontend eval transport "proxy" is only supported for flow, script, and app evals', + ); + } + + return { + transport, + backend: + transport === "proxy" ? resolveWindmillBackendSettings() : undefined, + }; +} diff --git a/ai_evals/core/results.ts b/ai_evals/core/results.ts index f2ea5b7dc4..c5f6b8749b 100644 --- a/ai_evals/core/results.ts +++ b/ai_evals/core/results.ts @@ -12,26 +12,34 @@ import type { export async function writeRunResult( result: BenchmarkRunResult, - outputPath?: string + outputPath?: string, ): Promise { const targetPath = resolveRunOutputPath(result.mode, outputPath); await mkdir(path.dirname(targetPath), { recursive: true }); - await writeFile(targetPath, JSON.stringify(toSerializableRunResult(result), null, 2) + "\n", "utf8"); + await writeFile( + targetPath, + JSON.stringify(toSerializableRunResult(result), null, 2) + "\n", + "utf8", + ); return targetPath; } export async function appendHistoryRecord( result: BenchmarkRunResult, - historyPath = resolveHistoryPath(result.mode) + historyPath = resolveHistoryPath(result.mode), ): Promise { await mkdir(path.dirname(historyPath), { recursive: true }); - await appendFile(historyPath, JSON.stringify(toHistoryRecord(result)) + "\n", "utf8"); + await appendFile( + historyPath, + JSON.stringify(toHistoryRecord(result)) + "\n", + "utf8", + ); return historyPath; } export async function writeRunArtifacts( result: BenchmarkRunResult, - outputPath?: string + outputPath?: string, ): Promise { const targetPath = resolveRunOutputPath(result.mode, outputPath); const artifactRoot = defaultArtifactsRoot(targetPath); @@ -47,7 +55,11 @@ export async function writeRunArtifacts( continue; } - const attemptDir = path.join(artifactRoot, caseResult.id, `attempt-${attempt.attempt}`); + const attemptDir = path.join( + artifactRoot, + caseResult.id, + `attempt-${attempt.attempt}`, + ); await writeArtifactFiles(attemptDir, artifactFiles); attempt.artifactsPath = attemptDir; wroteArtifacts = true; @@ -62,17 +74,24 @@ export function buildRunResult(input: { mode: EvalMode; runs: number; runModel: string | null; + transport?: BenchmarkRunResult["transport"]; judgeModel: string | null; caseResults: BenchmarkCaseResult[]; }): BenchmarkRunResult { - const attemptCount = input.caseResults.reduce((sum, entry) => sum + entry.attempts.length, 0); + const attemptCount = input.caseResults.reduce( + (sum, entry) => sum + entry.attempts.length, + 0, + ); const passedAttempts = input.caseResults.reduce( - (sum, entry) => sum + entry.attempts.filter((attempt) => attempt.passed).length, - 0 + (sum, entry) => + sum + entry.attempts.filter((attempt) => attempt.passed).length, + 0, ); const durationTotal = input.caseResults.reduce( - (sum, entry) => sum + entry.attempts.reduce((inner, attempt) => inner + attempt.durationMs, 0), - 0 + (sum, entry) => + sum + + entry.attempts.reduce((inner, attempt) => inner + attempt.durationMs, 0), + 0, ); const tokenUsageTotal = input.caseResults.reduce( (sum, entry) => { @@ -87,7 +106,7 @@ export function buildRunResult(input: { } return sum; }, - null + null, ); return { @@ -97,6 +116,7 @@ export function buildRunResult(input: { gitSha: getGitSha(), runs: input.runs, runModel: input.runModel, + transport: input.transport ?? null, judgeModel: input.judgeModel, caseCount: input.caseResults.length, attemptCount, @@ -122,6 +142,9 @@ export function formatRunSummary(result: BenchmarkRunResult): string { `Pass rate: ${formatPercent(result.passRate)} (${result.passedAttempts}/${result.attemptCount})`, `Average duration: ${Math.round(result.averageDurationMs)}ms`, ]; + if (result.transport) { + lines.splice(1, 0, `Transport: ${result.transport}`); + } const failures = collectFailures(result); if (failures.length > 0) { @@ -142,9 +165,11 @@ function collectFailures(result: BenchmarkRunResult): string[] { if (attempt.passed) { continue; } - const failedChecks = attempt.checks.filter((check) => !check.passed).map((check) => check.name); + const failedChecks = attempt.checks + .filter((check) => !check.passed) + .map((check) => check.name); failures.push( - `${caseResult.id} attempt ${attempt.attempt}: ${failedChecks.join(", ") || attempt.error || "failed"}` + `${caseResult.id} attempt ${attempt.attempt}: ${failedChecks.join(", ") || attempt.error || "failed"}`, ); } } @@ -156,8 +181,13 @@ function defaultFileName(mode: EvalMode): string { return `${new Date().toISOString().replaceAll(":", "-")}__${mode}.json`; } -export function resolveRunOutputPath(mode: EvalMode, outputPath?: string): string { - return outputPath ?? path.join(getAiEvalsRoot(), "results", defaultFileName(mode)); +export function resolveRunOutputPath( + mode: EvalMode, + outputPath?: string, +): string { + return ( + outputPath ?? path.join(getAiEvalsRoot(), "results", defaultFileName(mode)) + ); } export function resolveHistoryPath(mode: EvalMode): string { @@ -172,7 +202,7 @@ function defaultArtifactsRoot(resultPath: string): string { async function writeArtifactFiles( rootDir: string, - files: BenchmarkArtifactFile[] + files: BenchmarkArtifactFile[], ): Promise { for (const file of files) { const relativePath = normalizeArtifactPath(file.path); @@ -185,18 +215,25 @@ async function writeArtifactFiles( function normalizeArtifactPath(filePath: string): string { const normalized = filePath.replaceAll("\\", "/").replace(/^\/+/, ""); const parts = normalized.split("/").filter(Boolean); - if (parts.length === 0 || parts.some((part) => part === "." || part === "..")) { + if ( + parts.length === 0 || + parts.some((part) => part === "." || part === "..") + ) { throw new Error(`Invalid artifact path: ${filePath}`); } return parts.join("/"); } -function toSerializableRunResult(result: BenchmarkRunResult): BenchmarkRunResult { +function toSerializableRunResult( + result: BenchmarkRunResult, +): BenchmarkRunResult { return { ...result, cases: result.cases.map((caseResult) => ({ ...caseResult, - attempts: caseResult.attempts.map(({ artifactFiles, ...attempt }) => attempt), + attempts: caseResult.attempts.map( + ({ artifactFiles, ...attempt }) => attempt, + ), })), }; } @@ -204,8 +241,8 @@ function toSerializableRunResult(result: BenchmarkRunResult): BenchmarkRunResult function toHistoryRecord(result: BenchmarkRunResult) { const judgeScores = result.cases.flatMap((caseResult) => caseResult.attempts.flatMap((attempt) => - typeof attempt.judgeScore === "number" ? [attempt.judgeScore] : [] - ) + typeof attempt.judgeScore === "number" ? [attempt.judgeScore] : [], + ), ); return { @@ -214,6 +251,7 @@ function toHistoryRecord(result: BenchmarkRunResult) { mode: result.mode, runs: result.runs, runModel: result.runModel, + transport: result.transport, judgeModel: result.judgeModel, caseCount: result.caseCount, attemptCount: result.attemptCount, @@ -223,49 +261,57 @@ function toHistoryRecord(result: BenchmarkRunResult) { averageJudgeScore: judgeScores.length === 0 ? null - : judgeScores.reduce((sum, score) => sum + score, 0) / judgeScores.length, + : judgeScores.reduce((sum, score) => sum + score, 0) / + judgeScores.length, averageTokenUsagePerAttempt: result.averageTokenUsagePerAttempt ?? null, failedCaseIds: Array.from( new Set( result.cases - .filter((caseResult) => caseResult.attempts.some((attempt) => !attempt.passed)) - .map((caseResult) => caseResult.id) - ) + .filter((caseResult) => + caseResult.attempts.some((attempt) => !attempt.passed), + ) + .map((caseResult) => caseResult.id), + ), ), cases: result.cases.map((caseResult) => { const attemptCount = caseResult.attempts.length; - const passedAttempts = caseResult.attempts.filter((attempt) => attempt.passed).length; + const passedAttempts = caseResult.attempts.filter( + (attempt) => attempt.passed, + ).length; const totalDurationMs = caseResult.attempts.reduce( (sum, attempt) => sum + attempt.durationMs, - 0 + 0, ); const judgeScores = caseResult.attempts.flatMap((attempt) => - typeof attempt.judgeScore === "number" ? [attempt.judgeScore] : [] + typeof attempt.judgeScore === "number" ? [attempt.judgeScore] : [], ); - const totalTokenUsage = caseResult.attempts.reduce( - (sum, attempt) => { - if (!attempt.tokenUsage) { + const totalTokenUsage = + caseResult.attempts.reduce( + (sum, attempt) => { + if (!attempt.tokenUsage) { + return sum; + } + sum ??= { prompt: 0, completion: 0, total: 0 }; + sum.prompt += attempt.tokenUsage.prompt; + sum.completion += attempt.tokenUsage.completion; + sum.total += attempt.tokenUsage.total; return sum; - } - sum ??= { prompt: 0, completion: 0, total: 0 }; - sum.prompt += attempt.tokenUsage.prompt; - sum.completion += attempt.tokenUsage.completion; - sum.total += attempt.tokenUsage.total; - return sum; - }, - null - ); + }, + null, + ); return { id: caseResult.id, attemptCount, passedAttempts, passRate: attemptCount === 0 ? 0 : passedAttempts / attemptCount, - averageDurationMs: attemptCount === 0 ? 0 : totalDurationMs / attemptCount, + averageDurationMs: + attemptCount === 0 ? 0 : totalDurationMs / attemptCount, averageJudgeScore: judgeScores.length === 0 ? null - : judgeScores.reduce((sum, score) => sum + score, 0) / judgeScores.length, + : judgeScores.reduce((sum, score) => sum + score, 0) / + judgeScores.length, averageTokenUsagePerAttempt: attemptCount === 0 || !totalTokenUsage ? null diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 0b4ea539e8..9c71dea9ee 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -1,6 +1,7 @@ export const EVAL_MODES = ["cli", "flow", "script", "app"] as const; export type EvalMode = (typeof EVAL_MODES)[number]; +export type FrontendEvalTransport = "direct" | "proxy"; export interface EvalCaseRuntimeBackendPreview { args?: Record; @@ -151,7 +152,7 @@ export interface ModeRunner { run( prompt: string, initial: TInitial | undefined, - context: ModeRunContext + context: ModeRunContext, ): Promise>; validate(input: { evalCase: EvalCase; @@ -205,6 +206,7 @@ export interface BenchmarkRunResult { gitSha: string | null; runs: number; runModel: string | null; + transport: FrontendEvalTransport | null; judgeModel: string | null; caseCount: number; attemptCount: number; diff --git a/ai_evals/core/windmillBackendSettings.ts b/ai_evals/core/windmillBackendSettings.ts new file mode 100644 index 0000000000..c3a0a52d47 --- /dev/null +++ b/ai_evals/core/windmillBackendSettings.ts @@ -0,0 +1,67 @@ +export interface WindmillBackendSettings { + baseUrl: string; + email: string; + password: string; + keepWorkspaces: boolean; + workspaceOverride?: string; + workspacePrefix: string; +} + +export function resolveWindmillBackendSettings(): WindmillBackendSettings { + return { + baseUrl: normalizeBaseUrl( + process.env.WMILL_AI_EVAL_BACKEND_URL ?? + process.env.WINDMILL_URL ?? + process.env.WINDMILL_BASE_URL ?? + process.env.REMOTE ?? + "http://127.0.0.1:8000", + ), + email: process.env.WMILL_AI_EVAL_BACKEND_EMAIL ?? "admin@windmill.dev", + password: process.env.WMILL_AI_EVAL_BACKEND_PASSWORD ?? "changeme", + keepWorkspaces: isTruthy(process.env.WMILL_AI_EVAL_KEEP_WORKSPACES), + workspaceOverride: sanitizeOptionalWorkspaceId( + process.env.WMILL_AI_EVAL_BACKEND_WORKSPACE, + ), + workspacePrefix: sanitizeWorkspacePrefix( + process.env.WMILL_AI_EVAL_WORKSPACE_PREFIX ?? "ai-evals", + ), + }; +} + +export function parsePositiveInteger( + value: string | undefined, + fallback: number, +): number { + if (!value) { + return fallback; + } + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function normalizeBaseUrl(value: string): string { + return value.replace(/\/+$/, ""); +} + +function sanitizeWorkspacePrefix(value: string): string { + const sanitized = value + .trim() + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-+|-+$/g, ""); + return sanitized.length > 0 ? sanitized : "ai-evals"; +} + +function sanitizeOptionalWorkspaceId( + value: string | undefined, +): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +function isTruthy(value: string | undefined): boolean { + if (!value) { + return false; + } + return ["1", "true", "yes", "on"].includes(value.trim().toLowerCase()); +} diff --git a/ai_evals/modes/app.ts b/ai_evals/modes/app.ts index eba20f4090..c69e6ffc48 100644 --- a/ai_evals/modes/app.ts +++ b/ai_evals/modes/app.ts @@ -5,31 +5,42 @@ import type { FrontendEvalModelConfig } from "../core/models"; import { validateAppState, type AppFilesState } from "../core/validators"; import type { BenchmarkArtifactFile, ModeRunner } from "../core/types"; import { runAppEval } from "../adapters/frontend/core/app/appEvalRunner"; -import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon"; +import { + DEFAULT_FRONTEND_EVAL_MODEL, + getFrontendApiKey, +} from "./frontendCommon"; +import type { FrontendEvalTransportSettings } from "../core/frontendTransport"; export function createAppModeRunner( - modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL + modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL, + transportSettings?: FrontendEvalTransportSettings, ): ModeRunner { return { mode: "app", concurrency: 5, judgeThreshold: 80, async loadInitial(path) { - return path ? (await loadAppFixture(path)) : undefined; + return path ? await loadAppFixture(path) : undefined; }, async loadExpected(path) { - return path ? (await loadAppFixture(path)) : undefined; + return path ? await loadAppFixture(path) : undefined; }, async run(prompt, initial, context) { - const result = await runAppEval(prompt, getFrontendApiKey(modelConfig.provider), { - initialFrontend: initial?.frontend, - initialBackend: initial?.backend, - initialDatatables: initial?.datatables, - maxIterations: context.evalCase?.runtime?.maxTurns, - provider: modelConfig.provider, - model: modelConfig.model, - runContext: context, - }); + const result = await runAppEval( + prompt, + getFrontendApiKey(modelConfig.provider), + { + initialFrontend: initial?.frontend, + initialBackend: initial?.backend, + initialDatatables: initial?.datatables, + maxIterations: context.evalCase?.runtime?.maxTurns, + provider: modelConfig.provider, + model: modelConfig.model, + transport: transportSettings?.transport, + backend: transportSettings?.backend, + runContext: context, + }, + ); return { success: result.success, diff --git a/ai_evals/modes/flow.ts b/ai_evals/modes/flow.ts index 2001cd47e4..e641c2a67c 100644 --- a/ai_evals/modes/flow.ts +++ b/ai_evals/modes/flow.ts @@ -7,7 +7,11 @@ import type { BenchmarkArtifactFile, ModeRunner } from "../core/types"; import { runFlowEval } from "../adapters/frontend/core/flow/flowEvalRunner"; import type { FlowWorkspaceFixtures } from "../adapters/frontend/core/flow/fileHelpers"; import { BackendPreviewClient } from "../adapters/frontend/backendPreview"; -import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon"; +import { + DEFAULT_FRONTEND_EVAL_MODEL, + getFrontendApiKey, +} from "./frontendCommon"; +import type { FrontendEvalTransportSettings } from "../core/frontendTransport"; import { normalizeFlowInitialFixture, normalizeFlowStateFixture, @@ -16,7 +20,8 @@ import { export function createFlowModeRunner( modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL, - backendValidation?: BackendValidationSettings + backendValidation?: BackendValidationSettings, + transportSettings?: FrontendEvalTransportSettings, ): ModeRunner { return { mode: "flow", @@ -35,14 +40,20 @@ export function createFlowModeRunner( return normalizeFlowStateFixture(await readJsonFile(path)); }, async run(prompt, initial, context) { - const result = await runFlowEval(prompt, getFrontendApiKey(modelConfig.provider), { - initialFlow: initial?.flowFixture, - workspaceFixtures: initial?.workspace, - maxIterations: context.evalCase?.runtime?.maxTurns, - provider: modelConfig.provider, - model: modelConfig.model, - runContext: context, - }); + const result = await runFlowEval( + prompt, + getFrontendApiKey(modelConfig.provider), + { + initialFlow: initial?.flowFixture, + workspaceFixtures: initial?.workspace, + maxIterations: context.evalCase?.runtime?.maxTurns, + provider: modelConfig.provider, + model: modelConfig.model, + transport: transportSettings?.transport, + backend: transportSettings?.backend, + runContext: context, + }, + ); return { success: result.success, @@ -64,7 +75,10 @@ export function createFlowModeRunner( }); }, async backendValidate({ evalCase, initial, actual, context }) { - if (backendValidation?.mode !== "preview" || !evalCase.runtime?.backendPreview) { + if ( + backendValidation?.mode !== "preview" || + !evalCase.runtime?.backendPreview + ) { return null; } @@ -81,46 +95,54 @@ export function createFlowModeRunner( } const previewClient = new BackendPreviewClient(backendValidation); - return await previewClient.withWorkspace(evalCase.id, context.attempt, async (workspaceId) => { - await seedWorkspaceFixtures(previewClient, workspaceId, initial?.workspace); + return await previewClient.withWorkspace( + evalCase.id, + context.attempt, + async (workspaceId) => { + await seedWorkspaceFixtures( + previewClient, + workspaceId, + initial?.workspace, + ); - const completedJob = await previewClient.runFlowPreview({ - workspaceId, - value: actual.value as Record, - args: evalCase.runtime?.backendPreview?.args ?? {}, - timeoutSeconds: evalCase.runtime?.backendPreview?.timeoutSeconds, - }); + const completedJob = await previewClient.runFlowPreview({ + workspaceId, + value: actual.value as Record, + args: evalCase.runtime?.backendPreview?.args ?? {}, + timeoutSeconds: evalCase.runtime?.backendPreview?.timeoutSeconds, + }); - return { - checks: [ - { - name: "backend flow preview succeeded", - passed: completedJob.success, - details: completedJob.success - ? `workspace=${workspaceId}` - : `workspace=${workspaceId}; job=${completedJob.id}`, - }, - ], - artifactFiles: [ - { - path: "backend-preview.json", - content: - JSON.stringify( - { - workspaceId, - jobId: completedJob.id, - success: completedJob.success, - result: completedJob.result, - logs: completedJob.logs, - completedJob: completedJob.raw, - }, - null, - 2 - ) + "\n", - }, - ], - }; - }); + return { + checks: [ + { + name: "backend flow preview succeeded", + passed: completedJob.success, + details: completedJob.success + ? `workspace=${workspaceId}` + : `workspace=${workspaceId}; job=${completedJob.id}`, + }, + ], + artifactFiles: [ + { + path: "backend-preview.json", + content: + JSON.stringify( + { + workspaceId, + jobId: completedJob.id, + success: completedJob.success, + result: completedJob.result, + logs: completedJob.logs, + completedJob: completedJob.raw, + }, + null, + 2, + ) + "\n", + }, + ], + }; + }, + ); }, buildArtifacts(actual): BenchmarkArtifactFile[] { return [ @@ -136,7 +158,7 @@ export function createFlowModeRunner( async function seedWorkspaceFixtures( previewClient: BackendPreviewClient, workspaceId: string, - fixtures?: FlowWorkspaceFixtures + fixtures?: FlowWorkspaceFixtures, ): Promise { for (const script of fixtures?.scripts ?? []) { await previewClient.createScript({ diff --git a/ai_evals/modes/script.ts b/ai_evals/modes/script.ts index cf6739a9bf..d668040f8e 100644 --- a/ai_evals/modes/script.ts +++ b/ai_evals/modes/script.ts @@ -6,11 +6,16 @@ import type { BenchmarkArtifactFile, ModeRunner } from "../core/types"; import { BackendPreviewClient } from "../adapters/frontend/backendPreview"; import { runScriptEval } from "../adapters/frontend/core/script/scriptEvalRunner"; import type { ScriptEvalState } from "../adapters/frontend/core/script/fileHelpers"; -import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon"; +import { + DEFAULT_FRONTEND_EVAL_MODEL, + getFrontendApiKey, +} from "./frontendCommon"; +import type { FrontendEvalTransportSettings } from "../core/frontendTransport"; export function createScriptModeRunner( modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL, - backendValidation?: BackendValidationSettings + backendValidation?: BackendValidationSettings, + transportSettings?: FrontendEvalTransportSettings, ): ModeRunner { return { mode: "script", @@ -27,13 +32,19 @@ export function createScriptModeRunner( throw new Error("Script evals require an initial script fixture"); } - const result = await runScriptEval(prompt, getFrontendApiKey(modelConfig.provider), { - initialScript: initial, - maxIterations: context.evalCase?.runtime?.maxTurns, - provider: modelConfig.provider, - model: modelConfig.model, - runContext: context, - }); + const result = await runScriptEval( + prompt, + getFrontendApiKey(modelConfig.provider), + { + initialScript: initial, + maxIterations: context.evalCase?.runtime?.maxTurns, + provider: modelConfig.provider, + model: modelConfig.model, + transport: transportSettings?.transport, + backend: transportSettings?.backend, + runContext: context, + }, + ); return { success: result.success, @@ -55,50 +66,56 @@ export function createScriptModeRunner( } const previewClient = new BackendPreviewClient(backendValidation); - return await previewClient.withWorkspace(evalCase.id, context.attempt, async (workspaceId) => { - const completedJob = await previewClient.runScriptPreview({ - workspaceId, - content: actual.code, - args: - (evalCase.runtime?.backendPreview?.args as Record | undefined) ?? - actual.args ?? - initial?.args ?? - {}, - language: normalizePreviewLanguage(actual.lang), - path: toPreviewScriptPath(actual.path), - timeoutSeconds: evalCase.runtime?.backendPreview?.timeoutSeconds, - }); + return await previewClient.withWorkspace( + evalCase.id, + context.attempt, + async (workspaceId) => { + const completedJob = await previewClient.runScriptPreview({ + workspaceId, + content: actual.code, + args: + (evalCase.runtime?.backendPreview?.args as + | Record + | undefined) ?? + actual.args ?? + initial?.args ?? + {}, + language: normalizePreviewLanguage(actual.lang), + path: toPreviewScriptPath(actual.path), + timeoutSeconds: evalCase.runtime?.backendPreview?.timeoutSeconds, + }); - return { - checks: [ - { - name: "backend script preview succeeded", - passed: completedJob.success, - details: completedJob.success - ? `workspace=${workspaceId}` - : `workspace=${workspaceId}; job=${completedJob.id}`, - }, - ], - artifactFiles: [ - { - path: "backend-preview.json", - content: - JSON.stringify( - { - workspaceId, - jobId: completedJob.id, - success: completedJob.success, - result: completedJob.result, - logs: completedJob.logs, - completedJob: completedJob.raw, - }, - null, - 2 - ) + "\n", - }, - ], - }; - }); + return { + checks: [ + { + name: "backend script preview succeeded", + passed: completedJob.success, + details: completedJob.success + ? `workspace=${workspaceId}` + : `workspace=${workspaceId}; job=${completedJob.id}`, + }, + ], + artifactFiles: [ + { + path: "backend-preview.json", + content: + JSON.stringify( + { + workspaceId, + jobId: completedJob.id, + success: completedJob.success, + result: completedJob.result, + logs: completedJob.logs, + completedJob: completedJob.raw, + }, + null, + 2, + ) + "\n", + }, + ], + }; + }, + ); }, buildArtifacts(actual): BenchmarkArtifactFile[] { return [ diff --git a/backend/windmill-common/src/ai_google.rs b/backend/windmill-common/src/ai_google.rs index 84d75cf216..a707543011 100644 --- a/backend/windmill-common/src/ai_google.rs +++ b/backend/windmill-common/src/ai_google.rs @@ -643,6 +643,7 @@ pub fn gemini_event_to_openai_sse_chunks( pub fn sanitize_schema_for_google(value: &mut serde_json::Value) { const UNSUPPORTED: &[&str] = &[ "additionalProperties", + "propertyNames", "strict", "$schema", "default", @@ -747,8 +748,8 @@ fn openai_tool_call_json( #[cfg(test)] mod tests { use super::{ - gemini_event_to_openai_sse_chunks, gemini_response_to_openai, GeminiParsedEvent, - GeminiToolCallEvent, + gemini_event_to_openai_sse_chunks, gemini_response_to_openai, sanitize_schema_for_google, + GeminiParsedEvent, GeminiToolCallEvent, }; #[test] @@ -814,4 +815,23 @@ mod tests { ); assert_eq!(tool_call["index"], 0); } + + #[test] + fn sanitize_schema_for_google_removes_property_names() { + let mut schema = serde_json::json!({ + "type": "object", + "properties": { + "staticInputs": { + "type": "object", + "propertyNames": { "type": "string" }, + "additionalProperties": true + } + } + }); + + sanitize_schema_for_google(&mut schema); + + assert!(schema["properties"]["staticInputs"]["propertyNames"].is_null()); + assert!(schema["properties"]["staticInputs"]["additionalProperties"].is_null()); + } }