mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-21 00:02:23 +00:00
fix: add proxy eval coverage for gemini schemas (#8897)
* feat: add proxy transport for ai evals Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * fix: strip propertyNames for gemini schemas Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * fix: require explicit eval transport Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
centdix
GitHub
+12
-1
@@ -55,6 +55,7 @@ bun run cli -- run flow flow-test4-order-processing-loop --model opus
|
||||
bun run cli -- run flow flow-test0-sum-two-numbers --models haiku,opus,4o
|
||||
bun run cli -- run flow flow-test0-sum-two-numbers --runs 3 --verbose
|
||||
bun run cli -- run flow --record
|
||||
GEMINI_API_KEY=... bun run cli -- run app app-test1-counter-create --model gemini-pro --transport proxy
|
||||
WMILL_AI_EVAL_BACKEND_URL=http://127.0.0.1:8000 bun run cli -- run flow --backend-validation preview
|
||||
bun run cli -- run cli bun-hello-script
|
||||
```
|
||||
@@ -71,6 +72,7 @@ Public CLI surface:
|
||||
- `--output <path>`: custom result JSON path
|
||||
- `--model <alias>`: choose the model under test
|
||||
- `--models <a,b,c>`: run the same cases sequentially against several model aliases
|
||||
- `--transport <mode>`: frontend request transport (`direct` by default, `proxy` to exercise `/api/w/{workspace}/ai/proxy`)
|
||||
- `--verbose`: stream assistant output for frontend runs
|
||||
- `--record`: append a compact tracked summary line to `ai_evals/history/<mode>.jsonl` for full-suite runs only
|
||||
- `--backend-validation <mode>`: optional backend smoke validation (`off` or `preview`) for `script` and `flow` evals
|
||||
@@ -155,6 +157,15 @@ Supported backend validation env vars:
|
||||
- `WMILL_AI_EVAL_KEEP_WORKSPACES=1`
|
||||
- `WMILL_AI_EVAL_WORKSPACE_PREFIX=ai-evals`
|
||||
|
||||
Frontend proxy transport uses the same backend auth/workspace env vars.
|
||||
|
||||
When `--transport proxy` is set:
|
||||
|
||||
- `ai_evals` creates or reuses a backend workspace
|
||||
- it upserts a provider resource under `f/evals/ai/<provider>`
|
||||
- frontend requests go through `/api/w/{workspace}/ai/proxy`
|
||||
- result JSON and history records include `transport` so direct vs proxy runs stay distinguishable
|
||||
|
||||
## Results And Artifacts
|
||||
|
||||
Every run writes:
|
||||
@@ -171,7 +182,7 @@ If `--record` is used, the CLI also appends one compact JSON line to:
|
||||
|
||||
Each recorded line contains:
|
||||
|
||||
- run metadata (`createdAt`, `gitSha`, `mode`, `runModel`, `judgeModel`)
|
||||
- run metadata (`createdAt`, `gitSha`, `mode`, `runModel`, `transport`, `judgeModel`)
|
||||
- suite totals (`caseCount`, `attemptCount`, `passedAttempts`, `passRate`, `averageDurationMs`, `averageJudgeScore`)
|
||||
- average token usage (`averageTokenUsagePerAttempt`)
|
||||
- per-case metrics under `cases[]` (`averageDurationMs`, `averageJudgeScore`, `averageTokenUsagePerAttempt`, pass rate)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { loadSelectedCases } from "../../core/cases";
|
||||
import { resolveBackendValidationSettings } from "../../core/backendValidation";
|
||||
import { resolveFrontendEvalTransportSettings } from "../../core/frontendTransport";
|
||||
import {
|
||||
formatRunModelLabel,
|
||||
getFrontendEvalModel,
|
||||
@@ -18,18 +19,35 @@ export type FrontendBenchmarkMode = "flow" | "app" | "script";
|
||||
|
||||
export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult> {
|
||||
const mode = parseMode(process.env.WMILL_FRONTEND_AI_EVAL_MODE);
|
||||
const caseIds = parseOptionalJsonStringArray(process.env.WMILL_FRONTEND_AI_EVAL_CASE_IDS);
|
||||
const runs = parsePositiveInteger(process.env.WMILL_FRONTEND_AI_EVAL_RUNS, "WMILL_FRONTEND_AI_EVAL_RUNS");
|
||||
const caseIds = parseOptionalJsonStringArray(
|
||||
process.env.WMILL_FRONTEND_AI_EVAL_CASE_IDS,
|
||||
);
|
||||
const runs = parsePositiveInteger(
|
||||
process.env.WMILL_FRONTEND_AI_EVAL_RUNS,
|
||||
"WMILL_FRONTEND_AI_EVAL_RUNS",
|
||||
);
|
||||
const emitProgress = process.env.WMILL_FRONTEND_AI_EVAL_PROGRESS === "1";
|
||||
const verbose = process.env.WMILL_FRONTEND_AI_EVAL_VERBOSE === "1";
|
||||
const model = resolveEvalModel(mode, process.env.WMILL_FRONTEND_AI_EVAL_MODEL);
|
||||
const model = resolveEvalModel(
|
||||
mode,
|
||||
process.env.WMILL_FRONTEND_AI_EVAL_MODEL,
|
||||
);
|
||||
const backendValidation = resolveBackendValidationSettings({
|
||||
evalMode: mode,
|
||||
requestedMode: process.env.WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION,
|
||||
});
|
||||
const transportSettings = resolveFrontendEvalTransportSettings({
|
||||
evalMode: mode,
|
||||
requestedTransport: process.env.WMILL_FRONTEND_AI_EVAL_TRANSPORT,
|
||||
});
|
||||
|
||||
const selectedCases = await loadSelectedCases(mode, caseIds);
|
||||
const modeRunner = getModeRunner(mode, getFrontendEvalModel(model), backendValidation);
|
||||
const modeRunner = getModeRunner(
|
||||
mode,
|
||||
getFrontendEvalModel(model),
|
||||
backendValidation,
|
||||
transportSettings,
|
||||
);
|
||||
const runModel = formatRunModelLabel(mode, model);
|
||||
const caseResults = await runSuite({
|
||||
modeRunner,
|
||||
@@ -39,13 +57,16 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
|
||||
judgeModel: DEFAULT_JUDGE_MODEL,
|
||||
concurrency: verbose ? 1 : undefined,
|
||||
verbose,
|
||||
onProgress: emitProgress ? (event) => emitFrontendBenchmarkProgress(event) : undefined,
|
||||
onProgress: emitProgress
|
||||
? (event) => emitFrontendBenchmarkProgress(event)
|
||||
: undefined,
|
||||
});
|
||||
|
||||
return buildRunResult({
|
||||
mode,
|
||||
runs,
|
||||
runModel,
|
||||
transport: transportSettings.transport,
|
||||
judgeModel: DEFAULT_JUDGE_MODEL,
|
||||
caseResults,
|
||||
});
|
||||
@@ -54,15 +75,20 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
|
||||
function getModeRunner(
|
||||
mode: FrontendBenchmarkMode,
|
||||
model: ReturnType<typeof getFrontendEvalModel>,
|
||||
backendValidation: ReturnType<typeof resolveBackendValidationSettings>
|
||||
backendValidation: ReturnType<typeof resolveBackendValidationSettings>,
|
||||
transportSettings: ReturnType<typeof resolveFrontendEvalTransportSettings>,
|
||||
): ModeRunner<any, any, any> {
|
||||
switch (mode) {
|
||||
case "flow":
|
||||
return createFlowModeRunner(model, backendValidation);
|
||||
return createFlowModeRunner(model, backendValidation, transportSettings);
|
||||
case "app":
|
||||
return createAppModeRunner(model);
|
||||
return createAppModeRunner(model, transportSettings);
|
||||
case "script":
|
||||
return createScriptModeRunner(model, backendValidation);
|
||||
return createScriptModeRunner(
|
||||
model,
|
||||
backendValidation,
|
||||
transportSettings,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,13 +104,21 @@ function parseOptionalJsonStringArray(value: string | undefined): string[] {
|
||||
return [];
|
||||
}
|
||||
const parsed = JSON.parse(value) as unknown;
|
||||
if (!Array.isArray(parsed) || parsed.some((entry) => typeof entry !== "string")) {
|
||||
throw new Error("WMILL_FRONTEND_AI_EVAL_CASE_IDS must be a JSON string array");
|
||||
if (
|
||||
!Array.isArray(parsed) ||
|
||||
parsed.some((entry) => typeof entry !== "string")
|
||||
) {
|
||||
throw new Error(
|
||||
"WMILL_FRONTEND_AI_EVAL_CASE_IDS must be a JSON string array",
|
||||
);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value: string | undefined, envName: string): number {
|
||||
function parsePositiveInteger(
|
||||
value: string | undefined,
|
||||
envName: string,
|
||||
): number {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
throw new Error(`${envName} must be a positive integer`);
|
||||
|
||||
@@ -1,95 +1,106 @@
|
||||
import { mkdtemp } from 'fs/promises'
|
||||
import { tmpdir } from 'os'
|
||||
import { join } from 'path'
|
||||
import { mkdtemp } from "fs/promises";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import type {
|
||||
BackendRunnable,
|
||||
AppAIChatHelpers
|
||||
} from '../../../../../frontend/src/lib/components/copilot/chat/app/core'
|
||||
BackendRunnable,
|
||||
AppAIChatHelpers,
|
||||
} from "../../../../../frontend/src/lib/components/copilot/chat/app/core";
|
||||
import {
|
||||
getAppTools,
|
||||
prepareAppSystemMessage,
|
||||
prepareAppUserMessage
|
||||
} from '../../../../../frontend/src/lib/components/copilot/chat/app/core'
|
||||
import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared'
|
||||
import { createAppFileHelpers } from './fileHelpers'
|
||||
import { runEval } from '../shared'
|
||||
import type { AIProvider } from '$lib/gen/types.gen'
|
||||
import type { ModeRunContext } from '../../../../core/types'
|
||||
import type { TokenUsage } from '../shared/types'
|
||||
import type { AppFilesState } from '../../../../core/validators'
|
||||
getAppTools,
|
||||
prepareAppSystemMessage,
|
||||
prepareAppUserMessage,
|
||||
} from "../../../../../frontend/src/lib/components/copilot/chat/app/core";
|
||||
import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared";
|
||||
import { createAppFileHelpers } from "./fileHelpers";
|
||||
import { runEval } from "../shared";
|
||||
import type { AIProvider } from "$lib/gen/types.gen";
|
||||
import type { ModeRunContext } from "../../../../core/types";
|
||||
import type { TokenUsage } from "../shared/types";
|
||||
import type { AppFilesState } from "../../../../core/validators";
|
||||
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
|
||||
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
|
||||
|
||||
export interface AppEvalResult {
|
||||
success: boolean
|
||||
files: AppFilesState
|
||||
error?: string
|
||||
assistantMessageCount: number
|
||||
toolCallCount: number
|
||||
toolsUsed: string[]
|
||||
tokenUsage: TokenUsage
|
||||
success: boolean;
|
||||
files: AppFilesState;
|
||||
error?: string;
|
||||
assistantMessageCount: number;
|
||||
toolCallCount: number;
|
||||
toolsUsed: string[];
|
||||
tokenUsage: TokenUsage;
|
||||
}
|
||||
|
||||
export interface AppEvalOptions {
|
||||
initialFrontend?: Record<string, string>
|
||||
initialBackend?: Record<string, BackendRunnable>
|
||||
initialDatatables?: AppFilesState['datatables']
|
||||
model?: string
|
||||
maxIterations?: number
|
||||
provider?: AIProvider
|
||||
workspaceRoot?: string
|
||||
runContext?: ModeRunContext
|
||||
initialFrontend?: Record<string, string>;
|
||||
initialBackend?: AppFilesState["backend"];
|
||||
initialDatatables?: AppFilesState["datatables"];
|
||||
model?: string;
|
||||
maxIterations?: number;
|
||||
provider?: AIProvider;
|
||||
transport?: FrontendEvalTransport;
|
||||
backend?: WindmillBackendSettings;
|
||||
workspaceRoot?: string;
|
||||
runContext?: ModeRunContext;
|
||||
}
|
||||
|
||||
export async function runAppEval(
|
||||
userPrompt: string,
|
||||
apiKey: string,
|
||||
options?: AppEvalOptions
|
||||
userPrompt: string,
|
||||
apiKey: string,
|
||||
options?: AppEvalOptions,
|
||||
): Promise<AppEvalResult> {
|
||||
const workspaceRoot =
|
||||
options?.workspaceRoot ??
|
||||
(await mkdtemp(join(tmpdir(), 'wmill-frontend-app-benchmark-')))
|
||||
const { helpers, getEvalState, cleanup } = await createAppFileHelpers(
|
||||
options?.initialFrontend ?? {},
|
||||
options?.initialBackend ?? {},
|
||||
options?.initialDatatables ?? [],
|
||||
workspaceRoot
|
||||
)
|
||||
const workspaceRoot =
|
||||
options?.workspaceRoot ??
|
||||
(await mkdtemp(join(tmpdir(), "wmill-frontend-app-benchmark-")));
|
||||
const { helpers, getEvalState, cleanup } = await createAppFileHelpers(
|
||||
options?.initialFrontend ?? {},
|
||||
(options?.initialBackend ?? {}) as Record<string, BackendRunnable>,
|
||||
options?.initialDatatables ?? [],
|
||||
workspaceRoot,
|
||||
);
|
||||
|
||||
try {
|
||||
const systemMessage = prepareAppSystemMessage()
|
||||
const tools = getAppTools() as ProductionTool<AppAIChatHelpers>[]
|
||||
const model = options?.model ?? 'claude-haiku-4-5-20251001'
|
||||
const userMessage = prepareAppUserMessage(userPrompt, helpers.getSelectedContext())
|
||||
try {
|
||||
const systemMessage = prepareAppSystemMessage();
|
||||
const tools = getAppTools() as ProductionTool<AppAIChatHelpers>[];
|
||||
const model = options?.model ?? "claude-haiku-4-5-20251001";
|
||||
const userMessage = prepareAppUserMessage(
|
||||
userPrompt,
|
||||
helpers.getSelectedContext(),
|
||||
);
|
||||
|
||||
const rawResult = await runEval({
|
||||
userPrompt,
|
||||
systemMessage,
|
||||
userMessage,
|
||||
tools,
|
||||
helpers,
|
||||
apiKey,
|
||||
getOutput: getEvalState,
|
||||
onAssistantMessageStart: options?.runContext?.onAssistantMessageStart,
|
||||
onAssistantToken: options?.runContext?.onAssistantChunk,
|
||||
onAssistantMessageEnd: options?.runContext?.onAssistantMessageEnd,
|
||||
onToolCall: options?.runContext?.onToolCall,
|
||||
options: {
|
||||
maxIterations: options?.maxIterations,
|
||||
model,
|
||||
workspace: workspaceRoot,
|
||||
provider: options?.provider
|
||||
}
|
||||
})
|
||||
const rawResult = await runEval({
|
||||
userPrompt,
|
||||
systemMessage,
|
||||
userMessage,
|
||||
tools,
|
||||
helpers,
|
||||
apiKey,
|
||||
getOutput: getEvalState,
|
||||
onAssistantMessageStart: options?.runContext?.onAssistantMessageStart,
|
||||
onAssistantToken: options?.runContext?.onAssistantChunk,
|
||||
onAssistantMessageEnd: options?.runContext?.onAssistantMessageEnd,
|
||||
onToolCall: options?.runContext?.onToolCall,
|
||||
options: {
|
||||
maxIterations: options?.maxIterations,
|
||||
model,
|
||||
workspace: workspaceRoot,
|
||||
provider: options?.provider,
|
||||
transport: options?.transport,
|
||||
backend: options?.backend,
|
||||
proxyCaseId: options?.runContext?.caseId,
|
||||
proxyAttempt: options?.runContext?.attempt,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
files: rawResult.output,
|
||||
success: rawResult.success,
|
||||
error: rawResult.error,
|
||||
assistantMessageCount: rawResult.iterations,
|
||||
toolCallCount: rawResult.toolCallsCount,
|
||||
toolsUsed: rawResult.toolsCalled,
|
||||
tokenUsage: rawResult.tokenUsage
|
||||
}
|
||||
} finally {
|
||||
await cleanup()
|
||||
}
|
||||
return {
|
||||
files: rawResult.output,
|
||||
success: rawResult.success,
|
||||
error: rawResult.error,
|
||||
assistantMessageCount: rawResult.iterations,
|
||||
toolCallCount: rawResult.toolCallsCount,
|
||||
toolsUsed: rawResult.toolsCalled,
|
||||
tokenUsage: rawResult.tokenUsage,
|
||||
};
|
||||
} finally {
|
||||
await cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
import type {
|
||||
BackendRunnable,
|
||||
DataTableSchema,
|
||||
InlineScript
|
||||
} from '../../../../../frontend/src/lib/components/copilot/chat/app/core'
|
||||
import type { AppFilesState } from '../../../../core/validators'
|
||||
BackendRunnable,
|
||||
DataTableSchema,
|
||||
InlineScript,
|
||||
} from "../../../../../frontend/src/lib/components/copilot/chat/app/core";
|
||||
import type { AppFilesState } from "../../../../core/validators";
|
||||
|
||||
/**
|
||||
* Backend runnable metadata stored in meta.json files.
|
||||
*/
|
||||
interface BackendMeta {
|
||||
name: string
|
||||
language: 'bun' | 'python3'
|
||||
name: string;
|
||||
language: "bun" | "python3";
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -18,49 +18,53 @@ interface BackendMeta {
|
||||
* File paths are relative to the base directory with a leading '/'.
|
||||
*/
|
||||
async function readFilesRecursively(
|
||||
dir: string,
|
||||
basePath: string = ''
|
||||
dir: string,
|
||||
basePath: string = "",
|
||||
): Promise<Record<string, string>> {
|
||||
// @ts-ignore - Node.js fs/promises
|
||||
const { readdir, readFile } = await import('fs/promises')
|
||||
// @ts-ignore - Node.js path
|
||||
const { join } = await import('path')
|
||||
// @ts-ignore - Node.js fs/promises
|
||||
const { readdir, readFile } = await import("fs/promises");
|
||||
// @ts-ignore - Node.js path
|
||||
const { join } = await import("path");
|
||||
|
||||
const result: Record<string, string> = {}
|
||||
const entries = await readdir(dir, { withFileTypes: true })
|
||||
const result: Record<string, string> = {};
|
||||
const entries = await readdir(dir, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = join(dir, entry.name)
|
||||
const relativePath = basePath ? `${basePath}/${entry.name}` : `/${entry.name}`
|
||||
for (const entry of entries) {
|
||||
const fullPath = join(dir, entry.name);
|
||||
const relativePath = basePath
|
||||
? `${basePath}/${entry.name}`
|
||||
: `/${entry.name}`;
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
const subFiles = await readFilesRecursively(fullPath, relativePath)
|
||||
Object.assign(result, subFiles)
|
||||
} else {
|
||||
const content = await readFile(fullPath, 'utf-8')
|
||||
result[relativePath] = content
|
||||
}
|
||||
}
|
||||
if (entry.isDirectory()) {
|
||||
const subFiles = await readFilesRecursively(fullPath, relativePath);
|
||||
Object.assign(result, subFiles);
|
||||
} else {
|
||||
const content = await readFile(fullPath, "utf-8");
|
||||
result[relativePath] = content;
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads frontend files from a directory.
|
||||
* All files are read recursively and paths become keys with leading '/'.
|
||||
*/
|
||||
async function loadFrontend(frontendPath: string): Promise<Record<string, string>> {
|
||||
// @ts-ignore - Node.js fs/promises
|
||||
const { access } = await import('fs/promises')
|
||||
async function loadFrontend(
|
||||
frontendPath: string,
|
||||
): Promise<Record<string, string>> {
|
||||
// @ts-ignore - Node.js fs/promises
|
||||
const { access } = await import("fs/promises");
|
||||
|
||||
try {
|
||||
await access(frontendPath)
|
||||
} catch {
|
||||
// Directory doesn't exist, return empty
|
||||
return {}
|
||||
}
|
||||
try {
|
||||
await access(frontendPath);
|
||||
} catch {
|
||||
// Directory doesn't exist, return empty
|
||||
return {};
|
||||
}
|
||||
|
||||
return readFilesRecursively(frontendPath)
|
||||
return readFilesRecursively(frontendPath);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -69,78 +73,89 @@ async function loadFrontend(frontendPath: string): Promise<Record<string, string
|
||||
* - main.ts or main.py: The code content
|
||||
* - meta.json: Metadata { name, language }
|
||||
*/
|
||||
async function loadBackend(backendPath: string): Promise<Record<string, BackendRunnable>> {
|
||||
// @ts-ignore - Node.js fs/promises
|
||||
const { readdir, readFile, access } = await import('fs/promises')
|
||||
// @ts-ignore - Node.js path
|
||||
const { join } = await import('path')
|
||||
async function loadBackend(
|
||||
backendPath: string,
|
||||
): Promise<Record<string, BackendRunnable>> {
|
||||
// @ts-ignore - Node.js fs/promises
|
||||
const { readdir, readFile, access } = await import("fs/promises");
|
||||
// @ts-ignore - Node.js path
|
||||
const { join } = await import("path");
|
||||
|
||||
try {
|
||||
await access(backendPath)
|
||||
} catch {
|
||||
// Directory doesn't exist, return empty
|
||||
return {}
|
||||
}
|
||||
try {
|
||||
await access(backendPath);
|
||||
} catch {
|
||||
// Directory doesn't exist, return empty
|
||||
return {};
|
||||
}
|
||||
|
||||
const result: Record<string, BackendRunnable> = {}
|
||||
const entries = await readdir(backendPath, { withFileTypes: true })
|
||||
const result: Record<string, BackendRunnable> = {};
|
||||
const entries = await readdir(backendPath, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
|
||||
const runnableKey = entry.name
|
||||
const runnablePath = join(backendPath, entry.name)
|
||||
const runnableKey = entry.name;
|
||||
const runnablePath = join(backendPath, entry.name);
|
||||
|
||||
// Read meta.json
|
||||
const metaPath = join(runnablePath, 'meta.json')
|
||||
let meta: BackendMeta
|
||||
try {
|
||||
const metaContent = await readFile(metaPath, 'utf-8')
|
||||
meta = JSON.parse(metaContent)
|
||||
} catch {
|
||||
console.warn(`Missing or invalid meta.json for runnable '${runnableKey}', skipping`)
|
||||
continue
|
||||
}
|
||||
// Read meta.json
|
||||
const metaPath = join(runnablePath, "meta.json");
|
||||
let meta: BackendMeta;
|
||||
try {
|
||||
const metaContent = await readFile(metaPath, "utf-8");
|
||||
meta = JSON.parse(metaContent);
|
||||
} catch {
|
||||
console.warn(
|
||||
`Missing or invalid meta.json for runnable '${runnableKey}', skipping`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Find and read the main file (main.ts or main.py)
|
||||
const runnableFiles = await readdir(runnablePath)
|
||||
const mainFile = runnableFiles.find((f) => f === 'main.ts' || f === 'main.py')
|
||||
// Find and read the main file (main.ts or main.py)
|
||||
const runnableFiles = await readdir(runnablePath);
|
||||
const mainFile = runnableFiles.find(
|
||||
(f) => f === "main.ts" || f === "main.py",
|
||||
);
|
||||
|
||||
if (!mainFile) {
|
||||
console.warn(`No main.ts or main.py found for runnable '${runnableKey}', skipping`)
|
||||
continue
|
||||
}
|
||||
if (!mainFile) {
|
||||
console.warn(
|
||||
`No main.ts or main.py found for runnable '${runnableKey}', skipping`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const content = await readFile(join(runnablePath, mainFile), 'utf-8')
|
||||
const content = await readFile(join(runnablePath, mainFile), "utf-8");
|
||||
|
||||
const inlineScript: InlineScript = {
|
||||
language: meta.language,
|
||||
content
|
||||
}
|
||||
const inlineScript: InlineScript = {
|
||||
language: meta.language,
|
||||
content,
|
||||
};
|
||||
|
||||
result[runnableKey] = {
|
||||
name: meta.name,
|
||||
type: 'inline',
|
||||
inlineScript
|
||||
}
|
||||
}
|
||||
result[runnableKey] = {
|
||||
name: meta.name,
|
||||
type: "inline",
|
||||
inlineScript,
|
||||
};
|
||||
}
|
||||
|
||||
return result
|
||||
return result;
|
||||
}
|
||||
|
||||
async function loadDatatables(fixturePath: string): Promise<DataTableSchema[]> {
|
||||
// @ts-ignore - Node.js fs/promises
|
||||
const { readFile } = await import('fs/promises')
|
||||
// @ts-ignore - Node.js path
|
||||
const { join } = await import('path')
|
||||
// @ts-ignore - Node.js fs/promises
|
||||
const { readFile } = await import("fs/promises");
|
||||
// @ts-ignore - Node.js path
|
||||
const { join } = await import("path");
|
||||
|
||||
try {
|
||||
const content = await readFile(join(fixturePath, 'datatables.json'), 'utf-8')
|
||||
const parsed = JSON.parse(content)
|
||||
return Array.isArray(parsed) ? (parsed as DataTableSchema[]) : []
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
try {
|
||||
const content = await readFile(
|
||||
join(fixturePath, "datatables.json"),
|
||||
"utf-8",
|
||||
);
|
||||
const parsed = JSON.parse(content);
|
||||
return Array.isArray(parsed) ? (parsed as DataTableSchema[]) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -162,32 +177,32 @@ async function loadDatatables(fixturePath: string): Promise<DataTableSchema[]> {
|
||||
* @param fixturePath - Path to the fixture directory
|
||||
* @returns AppFiles object with frontend and backend
|
||||
*/
|
||||
export async function loadAppFixture(fixturePath: string): Promise<AppFilesState> {
|
||||
// @ts-ignore - Node.js path
|
||||
const { join } = await import('path')
|
||||
export async function loadAppFixture(
|
||||
fixturePath: string,
|
||||
): Promise<AppFilesState> {
|
||||
// @ts-ignore - Node.js path
|
||||
const { join } = await import("path");
|
||||
|
||||
const frontend = await loadFrontend(join(fixturePath, 'frontend'))
|
||||
const backend = await loadBackend(join(fixturePath, 'backend'))
|
||||
const datatables = await loadDatatables(fixturePath)
|
||||
const frontend = await loadFrontend(join(fixturePath, "frontend"));
|
||||
const backend = await loadBackend(join(fixturePath, "backend"));
|
||||
const datatables = await loadDatatables(fixturePath);
|
||||
|
||||
return { frontend, backend, datatables }
|
||||
return { frontend, backend, datatables };
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads an app fixture and returns the separate frontend and backend objects.
|
||||
* Convenience function for use with runAppEval options.
|
||||
*/
|
||||
export async function loadAppFixtureForEval(
|
||||
fixturePath: string
|
||||
): Promise<{
|
||||
initialFrontend: Record<string, string>
|
||||
initialBackend: Record<string, BackendRunnable>
|
||||
initialDatatables: DataTableSchema[]
|
||||
export async function loadAppFixtureForEval(fixturePath: string): Promise<{
|
||||
initialFrontend: Record<string, string>;
|
||||
initialBackend: AppFilesState["backend"];
|
||||
initialDatatables: DataTableSchema[];
|
||||
}> {
|
||||
const { frontend, backend, datatables } = await loadAppFixture(fixturePath)
|
||||
return {
|
||||
initialFrontend: frontend,
|
||||
initialBackend: backend,
|
||||
initialDatatables: datatables
|
||||
}
|
||||
const { frontend, backend, datatables } = await loadAppFixture(fixturePath);
|
||||
return {
|
||||
initialFrontend: frontend,
|
||||
initialBackend: backend,
|
||||
initialDatatables: datatables,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,108 +1,119 @@
|
||||
import { mkdtemp } from 'fs/promises'
|
||||
import { tmpdir } from 'os'
|
||||
import { join } from 'path'
|
||||
import type { FlowModule } from '$lib/gen'
|
||||
import type { AIProvider } from '$lib/gen/types.gen'
|
||||
import type { ExtendedOpenFlow } from '$lib/components/flows/types'
|
||||
import { mkdtemp } from "fs/promises";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import type { FlowModule } from "$lib/gen";
|
||||
import type { AIProvider } from "$lib/gen/types.gen";
|
||||
import type { ExtendedOpenFlow } from "$lib/components/flows/types";
|
||||
import {
|
||||
flowTools,
|
||||
prepareFlowSystemMessage,
|
||||
prepareFlowUserMessage,
|
||||
type FlowAIChatHelpers
|
||||
} from '../../../../../frontend/src/lib/components/copilot/chat/flow/core'
|
||||
import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared'
|
||||
import { createFlowFileHelpers, type FlowWorkspaceFixtures } from './fileHelpers'
|
||||
import { runEval } from '../shared'
|
||||
import type { ModeRunContext } from '../../../../core/types'
|
||||
import type { TokenUsage } from '../shared/types'
|
||||
flowTools,
|
||||
prepareFlowSystemMessage,
|
||||
prepareFlowUserMessage,
|
||||
type FlowAIChatHelpers,
|
||||
} from "../../../../../frontend/src/lib/components/copilot/chat/flow/core";
|
||||
import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared";
|
||||
import {
|
||||
createFlowFileHelpers,
|
||||
type FlowWorkspaceFixtures,
|
||||
} from "./fileHelpers";
|
||||
import { runEval } from "../shared";
|
||||
import type { ModeRunContext } from "../../../../core/types";
|
||||
import type { TokenUsage } from "../shared/types";
|
||||
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
|
||||
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
|
||||
|
||||
export interface FlowFixture {
|
||||
value?: {
|
||||
modules?: FlowModule[]
|
||||
preprocessor_module?: FlowModule
|
||||
failure_module?: FlowModule
|
||||
}
|
||||
schema?: Record<string, unknown>
|
||||
value?: {
|
||||
modules?: FlowModule[];
|
||||
preprocessor_module?: FlowModule;
|
||||
failure_module?: FlowModule;
|
||||
};
|
||||
schema?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface FlowEvalResult {
|
||||
success: boolean
|
||||
flow: ExtendedOpenFlow
|
||||
error?: string
|
||||
assistantMessageCount: number
|
||||
toolCallCount: number
|
||||
toolsUsed: string[]
|
||||
tokenUsage: TokenUsage
|
||||
success: boolean;
|
||||
flow: ExtendedOpenFlow;
|
||||
error?: string;
|
||||
assistantMessageCount: number;
|
||||
toolCallCount: number;
|
||||
toolsUsed: string[];
|
||||
tokenUsage: TokenUsage;
|
||||
}
|
||||
|
||||
export interface FlowEvalOptions {
|
||||
initialFlow?: FlowFixture
|
||||
workspaceFixtures?: FlowWorkspaceFixtures
|
||||
model?: string
|
||||
maxIterations?: number
|
||||
provider?: AIProvider
|
||||
workspaceRoot?: string
|
||||
runContext?: ModeRunContext
|
||||
initialFlow?: FlowFixture;
|
||||
workspaceFixtures?: FlowWorkspaceFixtures;
|
||||
model?: string;
|
||||
maxIterations?: number;
|
||||
provider?: AIProvider;
|
||||
transport?: FrontendEvalTransport;
|
||||
backend?: WindmillBackendSettings;
|
||||
workspaceRoot?: string;
|
||||
runContext?: ModeRunContext;
|
||||
}
|
||||
|
||||
export async function runFlowEval(
|
||||
userPrompt: string,
|
||||
apiKey: string,
|
||||
options?: FlowEvalOptions
|
||||
userPrompt: string,
|
||||
apiKey: string,
|
||||
options?: FlowEvalOptions,
|
||||
): Promise<FlowEvalResult> {
|
||||
const workspaceRoot =
|
||||
options?.workspaceRoot ??
|
||||
(await mkdtemp(join(tmpdir(), 'wmill-frontend-flow-benchmark-')))
|
||||
const { helpers, getFlow, cleanup } = await createFlowFileHelpers(
|
||||
options?.initialFlow?.value?.modules ?? [],
|
||||
options?.initialFlow?.schema,
|
||||
options?.initialFlow?.value?.preprocessor_module,
|
||||
options?.initialFlow?.value?.failure_module,
|
||||
workspaceRoot,
|
||||
options?.workspaceFixtures
|
||||
)
|
||||
const workspaceRoot =
|
||||
options?.workspaceRoot ??
|
||||
(await mkdtemp(join(tmpdir(), "wmill-frontend-flow-benchmark-")));
|
||||
const { helpers, getFlow, cleanup } = await createFlowFileHelpers(
|
||||
options?.initialFlow?.value?.modules ?? [],
|
||||
options?.initialFlow?.schema,
|
||||
options?.initialFlow?.value?.preprocessor_module,
|
||||
options?.initialFlow?.value?.failure_module,
|
||||
workspaceRoot,
|
||||
options?.workspaceFixtures,
|
||||
);
|
||||
|
||||
try {
|
||||
const systemMessage = prepareFlowSystemMessage()
|
||||
const tools = flowTools as ProductionTool<FlowAIChatHelpers>[]
|
||||
const model = options?.model ?? 'claude-haiku-4-5-20251001'
|
||||
const userMessage = prepareFlowUserMessage(
|
||||
userPrompt,
|
||||
helpers.getFlowAndSelectedId(),
|
||||
[],
|
||||
helpers.inlineScriptSession
|
||||
)
|
||||
try {
|
||||
const systemMessage = prepareFlowSystemMessage();
|
||||
const tools = flowTools as ProductionTool<FlowAIChatHelpers>[];
|
||||
const model = options?.model ?? "claude-haiku-4-5-20251001";
|
||||
const userMessage = prepareFlowUserMessage(
|
||||
userPrompt,
|
||||
helpers.getFlowAndSelectedId(),
|
||||
[],
|
||||
helpers.inlineScriptSession,
|
||||
);
|
||||
|
||||
const rawResult = await runEval({
|
||||
userPrompt,
|
||||
systemMessage,
|
||||
userMessage,
|
||||
tools,
|
||||
helpers,
|
||||
apiKey,
|
||||
getOutput: getFlow,
|
||||
onAssistantMessageStart: options?.runContext?.onAssistantMessageStart,
|
||||
onAssistantToken: options?.runContext?.onAssistantChunk,
|
||||
onAssistantMessageEnd: options?.runContext?.onAssistantMessageEnd,
|
||||
onToolCall: options?.runContext?.onToolCall,
|
||||
options: {
|
||||
maxIterations: options?.maxIterations,
|
||||
model,
|
||||
workspace: workspaceRoot,
|
||||
provider: options?.provider
|
||||
}
|
||||
})
|
||||
const rawResult = await runEval({
|
||||
userPrompt,
|
||||
systemMessage,
|
||||
userMessage,
|
||||
tools,
|
||||
helpers,
|
||||
apiKey,
|
||||
getOutput: getFlow,
|
||||
onAssistantMessageStart: options?.runContext?.onAssistantMessageStart,
|
||||
onAssistantToken: options?.runContext?.onAssistantChunk,
|
||||
onAssistantMessageEnd: options?.runContext?.onAssistantMessageEnd,
|
||||
onToolCall: options?.runContext?.onToolCall,
|
||||
options: {
|
||||
maxIterations: options?.maxIterations,
|
||||
model,
|
||||
workspace: workspaceRoot,
|
||||
provider: options?.provider,
|
||||
transport: options?.transport,
|
||||
backend: options?.backend,
|
||||
proxyCaseId: options?.runContext?.caseId,
|
||||
proxyAttempt: options?.runContext?.attempt,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
flow: rawResult.output,
|
||||
success: rawResult.success,
|
||||
error: rawResult.error,
|
||||
assistantMessageCount: rawResult.iterations,
|
||||
toolCallCount: rawResult.toolCallsCount,
|
||||
toolsUsed: rawResult.toolsCalled,
|
||||
tokenUsage: rawResult.tokenUsage
|
||||
}
|
||||
} finally {
|
||||
await cleanup()
|
||||
}
|
||||
return {
|
||||
flow: rawResult.output,
|
||||
success: rawResult.success,
|
||||
error: rawResult.error,
|
||||
assistantMessageCount: rawResult.iterations,
|
||||
toolCallCount: rawResult.toolCallsCount,
|
||||
toolsUsed: rawResult.toolsCalled,
|
||||
tokenUsage: rawResult.tokenUsage,
|
||||
};
|
||||
} finally {
|
||||
await cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,110 +1,119 @@
|
||||
import { mkdtemp } from 'fs/promises'
|
||||
import { tmpdir } from 'os'
|
||||
import { join } from 'path'
|
||||
import type { AIProvider, AIProviderModel } from '$lib/gen/types.gen'
|
||||
import type { ContextElement } from '../../../../../frontend/src/lib/components/copilot/chat/context'
|
||||
import { mkdtemp } from "fs/promises";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import type { AIProvider, AIProviderModel } from "$lib/gen/types.gen";
|
||||
import type { ContextElement } from "../../../../../frontend/src/lib/components/copilot/chat/context";
|
||||
import {
|
||||
prepareScriptSystemMessage,
|
||||
prepareScriptTools,
|
||||
prepareScriptUserMessage,
|
||||
type ScriptChatHelpers
|
||||
} from '../../../../../frontend/src/lib/components/copilot/chat/script/core'
|
||||
import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared'
|
||||
import { createScriptFileHelpers, type ScriptEvalState } from './fileHelpers'
|
||||
import { runEval } from '../shared'
|
||||
import type { ModeRunContext } from '../../../../core/types'
|
||||
import type { TokenUsage } from '../shared/types'
|
||||
prepareScriptSystemMessage,
|
||||
prepareScriptTools,
|
||||
prepareScriptUserMessage,
|
||||
type ScriptChatHelpers,
|
||||
} from "../../../../../frontend/src/lib/components/copilot/chat/script/core";
|
||||
import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared";
|
||||
import { createScriptFileHelpers, type ScriptEvalState } from "./fileHelpers";
|
||||
import { runEval } from "../shared";
|
||||
import type { ModeRunContext } from "../../../../core/types";
|
||||
import type { TokenUsage } from "../shared/types";
|
||||
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
|
||||
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
|
||||
|
||||
export interface ScriptEvalResult {
|
||||
success: boolean
|
||||
script: ScriptEvalState
|
||||
error?: string
|
||||
assistantMessageCount: number
|
||||
toolCallCount: number
|
||||
toolsUsed: string[]
|
||||
tokenUsage: TokenUsage
|
||||
success: boolean;
|
||||
script: ScriptEvalState;
|
||||
error?: string;
|
||||
assistantMessageCount: number;
|
||||
toolCallCount: number;
|
||||
toolsUsed: string[];
|
||||
tokenUsage: TokenUsage;
|
||||
}
|
||||
|
||||
export interface ScriptEvalOptions {
|
||||
initialScript: ScriptEvalState
|
||||
model?: string
|
||||
maxIterations?: number
|
||||
provider?: AIProvider
|
||||
workspaceRoot?: string
|
||||
runContext?: ModeRunContext
|
||||
initialScript: ScriptEvalState;
|
||||
model?: string;
|
||||
maxIterations?: number;
|
||||
provider?: AIProvider;
|
||||
transport?: FrontendEvalTransport;
|
||||
backend?: WindmillBackendSettings;
|
||||
workspaceRoot?: string;
|
||||
runContext?: ModeRunContext;
|
||||
}
|
||||
|
||||
function resolveModelProvider(
|
||||
model: string,
|
||||
provider?: AIProvider
|
||||
model: string,
|
||||
provider?: AIProvider,
|
||||
): AIProviderModel {
|
||||
if (provider) {
|
||||
return { provider, model }
|
||||
}
|
||||
if (model.startsWith('claude')) {
|
||||
return { provider: 'anthropic', model }
|
||||
}
|
||||
return { provider: 'openai', model }
|
||||
if (provider) {
|
||||
return { provider, model };
|
||||
}
|
||||
if (model.startsWith("claude")) {
|
||||
return { provider: "anthropic", model };
|
||||
}
|
||||
return { provider: "openai", model };
|
||||
}
|
||||
|
||||
export async function runScriptEval(
|
||||
userPrompt: string,
|
||||
apiKey: string,
|
||||
options: ScriptEvalOptions
|
||||
userPrompt: string,
|
||||
apiKey: string,
|
||||
options: ScriptEvalOptions,
|
||||
): Promise<ScriptEvalResult> {
|
||||
const workspaceRoot =
|
||||
options.workspaceRoot ?? (await mkdtemp(join(tmpdir(), 'wmill-frontend-script-benchmark-')))
|
||||
const { helpers, getScript, cleanup } = await createScriptFileHelpers(
|
||||
options.initialScript,
|
||||
workspaceRoot
|
||||
)
|
||||
const workspaceRoot =
|
||||
options.workspaceRoot ??
|
||||
(await mkdtemp(join(tmpdir(), "wmill-frontend-script-benchmark-")));
|
||||
const { helpers, getScript, cleanup } = await createScriptFileHelpers(
|
||||
options.initialScript,
|
||||
workspaceRoot,
|
||||
);
|
||||
|
||||
try {
|
||||
const model = options.model ?? 'claude-haiku-4-5-20251001'
|
||||
const modelProvider = resolveModelProvider(model, options.provider)
|
||||
const selectedContext: ContextElement[] = []
|
||||
const systemMessage = prepareScriptSystemMessage(
|
||||
modelProvider,
|
||||
options.initialScript.lang,
|
||||
{}
|
||||
)
|
||||
const tools = prepareScriptTools(
|
||||
modelProvider,
|
||||
options.initialScript.lang,
|
||||
selectedContext
|
||||
) as ProductionTool<ScriptChatHelpers>[]
|
||||
const userMessage = prepareScriptUserMessage(userPrompt, selectedContext)
|
||||
try {
|
||||
const model = options.model ?? "claude-haiku-4-5-20251001";
|
||||
const modelProvider = resolveModelProvider(model, options.provider);
|
||||
const selectedContext: ContextElement[] = [];
|
||||
const systemMessage = prepareScriptSystemMessage(
|
||||
modelProvider,
|
||||
options.initialScript.lang,
|
||||
{},
|
||||
);
|
||||
const tools = prepareScriptTools(
|
||||
modelProvider,
|
||||
options.initialScript.lang,
|
||||
selectedContext,
|
||||
) as ProductionTool<ScriptChatHelpers>[];
|
||||
const userMessage = prepareScriptUserMessage(userPrompt, selectedContext);
|
||||
|
||||
const rawResult = await runEval({
|
||||
userPrompt,
|
||||
systemMessage,
|
||||
userMessage,
|
||||
tools,
|
||||
helpers,
|
||||
apiKey,
|
||||
getOutput: getScript,
|
||||
onAssistantMessageStart: options.runContext?.onAssistantMessageStart,
|
||||
onAssistantToken: options.runContext?.onAssistantChunk,
|
||||
onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd,
|
||||
onToolCall: options.runContext?.onToolCall,
|
||||
options: {
|
||||
maxIterations: options.maxIterations,
|
||||
model,
|
||||
workspace: workspaceRoot,
|
||||
provider: modelProvider.provider
|
||||
}
|
||||
})
|
||||
const rawResult = await runEval({
|
||||
userPrompt,
|
||||
systemMessage,
|
||||
userMessage,
|
||||
tools,
|
||||
helpers,
|
||||
apiKey,
|
||||
getOutput: getScript,
|
||||
onAssistantMessageStart: options.runContext?.onAssistantMessageStart,
|
||||
onAssistantToken: options.runContext?.onAssistantChunk,
|
||||
onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd,
|
||||
onToolCall: options.runContext?.onToolCall,
|
||||
options: {
|
||||
maxIterations: options.maxIterations,
|
||||
model,
|
||||
workspace: workspaceRoot,
|
||||
provider: modelProvider.provider,
|
||||
transport: options.transport,
|
||||
backend: options.backend,
|
||||
proxyCaseId: options.runContext?.caseId,
|
||||
proxyAttempt: options.runContext?.attempt,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
script: rawResult.output,
|
||||
success: rawResult.success,
|
||||
error: rawResult.error,
|
||||
assistantMessageCount: rawResult.iterations,
|
||||
toolCallCount: rawResult.toolCallsCount,
|
||||
toolsUsed: rawResult.toolsCalled,
|
||||
tokenUsage: rawResult.tokenUsage
|
||||
}
|
||||
} finally {
|
||||
await cleanup()
|
||||
}
|
||||
return {
|
||||
script: rawResult.output,
|
||||
success: rawResult.success,
|
||||
error: rawResult.error,
|
||||
assistantMessageCount: rawResult.iterations,
|
||||
toolCallCount: rawResult.toolCallsCount,
|
||||
toolsUsed: rawResult.toolsCalled,
|
||||
tokenUsage: rawResult.tokenUsage,
|
||||
};
|
||||
} finally {
|
||||
await cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,46 +1,51 @@
|
||||
import type {
|
||||
ChatCompletionMessageParam,
|
||||
ChatCompletionSystemMessageParam
|
||||
} from 'openai/resources/chat/completions.mjs'
|
||||
import type { AIProvider } from '$lib/gen/types.gen'
|
||||
import type { ToolCallDetail, EvalRunnerOptions, RawEvalResult } from './types'
|
||||
import { runChatLoop, type ChatClients } from '../../../../../frontend/src/lib/components/copilot/chat/chatLoop'
|
||||
import type {
|
||||
Tool as ProductionTool,
|
||||
ToolCallbacks
|
||||
} from '../../../../../frontend/src/lib/components/copilot/chat/shared'
|
||||
ChatCompletionMessageParam,
|
||||
ChatCompletionSystemMessageParam,
|
||||
} from "openai/resources/chat/completions.mjs";
|
||||
import type { AIProvider } from "$lib/gen/types.gen";
|
||||
import type { ToolCallDetail, EvalRunnerOptions, RawEvalResult } from "./types";
|
||||
import {
|
||||
createEvalClients,
|
||||
type FrontendEvalProvider,
|
||||
resolveEvalModelProvider
|
||||
} from './providerConfig'
|
||||
runChatLoop,
|
||||
type ChatClients,
|
||||
} from "../../../../../frontend/src/lib/components/copilot/chat/chatLoop";
|
||||
import type {
|
||||
Tool as ProductionTool,
|
||||
ToolCallbacks,
|
||||
} from "../../../../../frontend/src/lib/components/copilot/chat/shared";
|
||||
import {
|
||||
buildProxyResourcePath,
|
||||
createEvalClients,
|
||||
type FrontendEvalProvider,
|
||||
resolveEvalModelProvider,
|
||||
} from "./providerConfig";
|
||||
import { WindmillBackendClient } from "../../windmillBackend";
|
||||
|
||||
/**
|
||||
* Parameters for running a base evaluation.
|
||||
*/
|
||||
export interface RunEvalParams<THelpers, TOutput> {
|
||||
/** The user's prompt/instruction */
|
||||
userPrompt: string
|
||||
/** System message for the LLM */
|
||||
systemMessage: ChatCompletionSystemMessageParam
|
||||
/** User message for the LLM */
|
||||
userMessage: ChatCompletionMessageParam
|
||||
/** Tool definitions for the LLM API (unused — derived from tools) */
|
||||
toolDefs?: unknown
|
||||
/** Full tool implementations for execution */
|
||||
tools: ProductionTool<THelpers>[]
|
||||
/** Domain-specific helpers for tool execution */
|
||||
helpers: THelpers
|
||||
/** API key for the provider */
|
||||
apiKey: string
|
||||
/** Function to get the current output state */
|
||||
getOutput: () => TOutput
|
||||
/** Optional configuration */
|
||||
options?: EvalRunnerOptions
|
||||
onAssistantMessageStart?: () => void
|
||||
onAssistantToken?: (token: string) => void
|
||||
onAssistantMessageEnd?: () => void
|
||||
onToolCall?: (input: { toolName: string; argumentsText: string }) => void
|
||||
/** The user's prompt/instruction */
|
||||
userPrompt: string;
|
||||
/** System message for the LLM */
|
||||
systemMessage: ChatCompletionSystemMessageParam;
|
||||
/** User message for the LLM */
|
||||
userMessage: ChatCompletionMessageParam;
|
||||
/** Tool definitions for the LLM API (unused — derived from tools) */
|
||||
toolDefs?: unknown;
|
||||
/** Full tool implementations for execution */
|
||||
tools: ProductionTool<THelpers>[];
|
||||
/** Domain-specific helpers for tool execution */
|
||||
helpers: THelpers;
|
||||
/** API key for the provider */
|
||||
apiKey: string;
|
||||
/** Function to get the current output state */
|
||||
getOutput: () => TOutput;
|
||||
/** Optional configuration */
|
||||
options?: EvalRunnerOptions;
|
||||
onAssistantMessageStart?: () => void;
|
||||
onAssistantToken?: (token: string) => void;
|
||||
onAssistantMessageEnd?: () => void;
|
||||
onToolCall?: (input: { toolName: string; argumentsText: string }) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -48,152 +53,206 @@ export interface RunEvalParams<THelpers, TOutput> {
|
||||
* Uses streaming via real provider SDKs instead of OpenRouter non-streaming.
|
||||
*/
|
||||
export async function runEval<THelpers, TOutput>(
|
||||
params: RunEvalParams<THelpers, TOutput>
|
||||
params: RunEvalParams<THelpers, TOutput>,
|
||||
): Promise<RawEvalResult<TOutput>> {
|
||||
const {
|
||||
systemMessage,
|
||||
userMessage,
|
||||
tools,
|
||||
helpers,
|
||||
apiKey,
|
||||
getOutput,
|
||||
options,
|
||||
onAssistantMessageStart,
|
||||
onAssistantToken,
|
||||
onAssistantMessageEnd,
|
||||
onToolCall
|
||||
} = params
|
||||
let shouldEmitMessageStart = true
|
||||
const {
|
||||
systemMessage,
|
||||
userMessage,
|
||||
tools,
|
||||
helpers,
|
||||
apiKey,
|
||||
getOutput,
|
||||
options,
|
||||
onAssistantMessageStart,
|
||||
onAssistantToken,
|
||||
onAssistantMessageEnd,
|
||||
onToolCall,
|
||||
} = params;
|
||||
let shouldEmitMessageStart = true;
|
||||
|
||||
const model = options?.model ?? 'gpt-4o'
|
||||
const maxIterations = options?.maxIterations ?? 20
|
||||
const workspace = options?.workspace ?? 'test-workspace'
|
||||
const provider = toFrontendEvalProvider(options?.provider)
|
||||
const model = options?.model ?? "gpt-4o";
|
||||
const maxIterations = options?.maxIterations ?? 20;
|
||||
const workspace = options?.workspace ?? "test-workspace";
|
||||
const provider = toFrontendEvalProvider(options?.provider);
|
||||
|
||||
const modelProvider = resolveEvalModelProvider(model, provider)
|
||||
const clients = createEvalClients(modelProvider.provider, apiKey) as unknown as ChatClients
|
||||
const modelProvider = resolveEvalModelProvider(model, provider);
|
||||
|
||||
const messages: ChatCompletionMessageParam[] = [userMessage]
|
||||
let toolCallsCount = 0
|
||||
const toolsCalled: string[] = []
|
||||
const toolCallDetails: ToolCallDetail[] = []
|
||||
const messages: ChatCompletionMessageParam[] = [userMessage];
|
||||
let toolCallsCount = 0;
|
||||
const toolsCalled: string[] = [];
|
||||
const toolCallDetails: ToolCallDetail[] = [];
|
||||
|
||||
// Wrap tools to intercept fn calls for tracking.
|
||||
// Cast to ProductionTool since the eval Tool has a narrower toolCallbacks type
|
||||
// but the actual callbacks passed at runtime will satisfy both interfaces.
|
||||
const wrappedTools = tools.map((tool) => ({
|
||||
...tool,
|
||||
fn: async (p: any) => {
|
||||
toolCallsCount++
|
||||
toolsCalled.push(tool.def.function.name)
|
||||
let argumentsText = ''
|
||||
try {
|
||||
const args = typeof p.args === 'string' ? JSON.parse(p.args) : p.args
|
||||
toolCallDetails.push({ name: tool.def.function.name, arguments: args })
|
||||
argumentsText = JSON.stringify(args)
|
||||
} catch {
|
||||
toolCallDetails.push({
|
||||
name: tool.def.function.name,
|
||||
arguments: p.args
|
||||
})
|
||||
argumentsText = typeof p.args === 'string' ? p.args : JSON.stringify(p.args)
|
||||
}
|
||||
onToolCall?.({
|
||||
toolName: tool.def.function.name,
|
||||
argumentsText
|
||||
})
|
||||
return tool.fn(p)
|
||||
}
|
||||
}))
|
||||
// Wrap tools to intercept fn calls for tracking.
|
||||
// Cast to ProductionTool since the eval Tool has a narrower toolCallbacks type
|
||||
// but the actual callbacks passed at runtime will satisfy both interfaces.
|
||||
const wrappedTools = tools.map((tool) => ({
|
||||
...tool,
|
||||
fn: async (p: any) => {
|
||||
toolCallsCount++;
|
||||
toolsCalled.push(tool.def.function.name);
|
||||
let argumentsText = "";
|
||||
try {
|
||||
const args = typeof p.args === "string" ? JSON.parse(p.args) : p.args;
|
||||
toolCallDetails.push({ name: tool.def.function.name, arguments: args });
|
||||
argumentsText = JSON.stringify(args);
|
||||
} catch {
|
||||
toolCallDetails.push({
|
||||
name: tool.def.function.name,
|
||||
arguments: p.args,
|
||||
});
|
||||
argumentsText =
|
||||
typeof p.args === "string" ? p.args : JSON.stringify(p.args);
|
||||
}
|
||||
onToolCall?.({
|
||||
toolName: tool.def.function.name,
|
||||
argumentsText,
|
||||
});
|
||||
return tool.fn(p);
|
||||
},
|
||||
}));
|
||||
|
||||
// No-op callbacks for eval
|
||||
const callbacks: ToolCallbacks & {
|
||||
onNewToken: (token: string) => void
|
||||
onMessageEnd: () => void
|
||||
} = {
|
||||
setToolStatus: () => {},
|
||||
removeToolStatus: () => {},
|
||||
onNewToken: (token: string) => {
|
||||
if (shouldEmitMessageStart) {
|
||||
onAssistantMessageStart?.()
|
||||
shouldEmitMessageStart = false
|
||||
}
|
||||
onAssistantToken?.(token)
|
||||
},
|
||||
onMessageEnd: () => {
|
||||
if (!shouldEmitMessageStart) {
|
||||
onAssistantMessageEnd?.()
|
||||
}
|
||||
shouldEmitMessageStart = true
|
||||
}
|
||||
}
|
||||
// No-op callbacks for eval
|
||||
const callbacks: ToolCallbacks & {
|
||||
onNewToken: (token: string) => void;
|
||||
onMessageEnd: () => void;
|
||||
} = {
|
||||
setToolStatus: () => {},
|
||||
removeToolStatus: () => {},
|
||||
onNewToken: (token: string) => {
|
||||
if (shouldEmitMessageStart) {
|
||||
onAssistantMessageStart?.();
|
||||
shouldEmitMessageStart = false;
|
||||
}
|
||||
onAssistantToken?.(token);
|
||||
},
|
||||
onMessageEnd: () => {
|
||||
if (!shouldEmitMessageStart) {
|
||||
onAssistantMessageEnd?.();
|
||||
}
|
||||
shouldEmitMessageStart = true;
|
||||
},
|
||||
};
|
||||
|
||||
const abortController = new AbortController()
|
||||
const abortController = new AbortController();
|
||||
|
||||
try {
|
||||
const result = await runChatLoop({
|
||||
messages,
|
||||
systemMessage,
|
||||
tools: wrappedTools,
|
||||
helpers,
|
||||
abortController,
|
||||
callbacks,
|
||||
modelProvider,
|
||||
clients,
|
||||
workspace,
|
||||
maxIterations,
|
||||
skipResponsesApi: modelProvider.provider !== 'openai'
|
||||
})
|
||||
const executeChatLoop = async (clients: ChatClients) => {
|
||||
try {
|
||||
const result = await runChatLoop({
|
||||
messages,
|
||||
systemMessage,
|
||||
tools: wrappedTools,
|
||||
helpers,
|
||||
abortController,
|
||||
callbacks,
|
||||
modelProvider,
|
||||
clients,
|
||||
workspace,
|
||||
maxIterations,
|
||||
skipResponsesApi: modelProvider.provider !== "openai",
|
||||
});
|
||||
|
||||
if (result.hitMaxIterations) {
|
||||
return {
|
||||
success: false,
|
||||
output: getOutput(),
|
||||
error: `Reached max turns (${maxIterations})`,
|
||||
tokenUsage: result.tokenUsage,
|
||||
toolCallsCount,
|
||||
toolsCalled,
|
||||
toolCallDetails,
|
||||
iterations: Math.max(1, result.addedMessages.filter((m) => m.role === 'assistant').length),
|
||||
messages
|
||||
}
|
||||
}
|
||||
if (result.hitMaxIterations) {
|
||||
return {
|
||||
success: false,
|
||||
output: getOutput(),
|
||||
error: `Reached max turns (${maxIterations})`,
|
||||
tokenUsage: result.tokenUsage,
|
||||
toolCallsCount,
|
||||
toolsCalled,
|
||||
toolCallDetails,
|
||||
iterations: Math.max(
|
||||
1,
|
||||
result.addedMessages.filter((m) => m.role === "assistant").length,
|
||||
),
|
||||
messages,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: getOutput(),
|
||||
tokenUsage: result.tokenUsage,
|
||||
toolCallsCount,
|
||||
toolsCalled,
|
||||
toolCallDetails,
|
||||
iterations: Math.max(1, result.addedMessages.filter((m) => m.role === 'assistant').length),
|
||||
messages
|
||||
}
|
||||
} catch (err) {
|
||||
let errorMessage: string
|
||||
if (err instanceof Error) {
|
||||
errorMessage = err.stack ?? err.message
|
||||
} else {
|
||||
errorMessage = String(err)
|
||||
}
|
||||
return {
|
||||
success: true,
|
||||
output: getOutput(),
|
||||
tokenUsage: result.tokenUsage,
|
||||
toolCallsCount,
|
||||
toolsCalled,
|
||||
toolCallDetails,
|
||||
iterations: Math.max(
|
||||
1,
|
||||
result.addedMessages.filter((m) => m.role === "assistant").length,
|
||||
),
|
||||
messages,
|
||||
};
|
||||
} catch (err) {
|
||||
let errorMessage: string;
|
||||
if (err instanceof Error) {
|
||||
errorMessage = err.stack ?? err.message;
|
||||
} else {
|
||||
errorMessage = String(err);
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
output: getOutput(),
|
||||
error: errorMessage,
|
||||
tokenUsage: { prompt: 0, completion: 0, total: 0 },
|
||||
toolCallsCount,
|
||||
toolsCalled,
|
||||
toolCallDetails,
|
||||
iterations: 0,
|
||||
messages
|
||||
}
|
||||
}
|
||||
return {
|
||||
success: false,
|
||||
output: getOutput(),
|
||||
error: errorMessage,
|
||||
tokenUsage: { prompt: 0, completion: 0, total: 0 },
|
||||
toolCallsCount,
|
||||
toolsCalled,
|
||||
toolCallDetails,
|
||||
iterations: 0,
|
||||
messages,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
if (options?.transport === "proxy") {
|
||||
const backendSettings = options.backend;
|
||||
if (!backendSettings) {
|
||||
throw new Error("Missing backend settings for proxy transport");
|
||||
}
|
||||
|
||||
const backendClient = new WindmillBackendClient(backendSettings);
|
||||
return await backendClient.withWorkspace(
|
||||
options.proxyCaseId ?? "eval",
|
||||
options.proxyAttempt ?? 1,
|
||||
async (proxyWorkspaceId) => {
|
||||
const resourcePath = buildProxyResourcePath(modelProvider.provider);
|
||||
await backendClient.upsertResource({
|
||||
workspaceId: proxyWorkspaceId,
|
||||
path: resourcePath,
|
||||
resourceType: modelProvider.provider,
|
||||
value: { api_key: apiKey },
|
||||
});
|
||||
const token = await backendClient.getToken();
|
||||
const clients = createEvalClients({
|
||||
provider: modelProvider.provider,
|
||||
apiKey,
|
||||
transport: "proxy",
|
||||
proxy: {
|
||||
baseURL: `${backendSettings.baseUrl}/api/w/${encodeURIComponent(proxyWorkspaceId)}/ai/proxy`,
|
||||
bearerToken: token,
|
||||
resourcePath,
|
||||
},
|
||||
}) as unknown as ChatClients;
|
||||
return await executeChatLoop(clients);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const clients = createEvalClients({
|
||||
provider: modelProvider.provider,
|
||||
apiKey,
|
||||
}) as unknown as ChatClients;
|
||||
return await executeChatLoop(clients);
|
||||
}
|
||||
|
||||
function toFrontendEvalProvider(provider?: AIProvider): FrontendEvalProvider | undefined {
|
||||
if (provider === 'anthropic' || provider === 'openai' || provider === 'googleai') {
|
||||
return provider
|
||||
}
|
||||
return undefined
|
||||
function toFrontendEvalProvider(
|
||||
provider?: AIProvider,
|
||||
): FrontendEvalProvider | undefined {
|
||||
if (
|
||||
provider === "anthropic" ||
|
||||
provider === "openai" ||
|
||||
provider === "googleai"
|
||||
) {
|
||||
return provider;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
buildProxyHeaders,
|
||||
buildProxyResourcePath,
|
||||
buildOpenAICompatibleClientOptions,
|
||||
resolveEvalModelProvider,
|
||||
} from "./providerConfig";
|
||||
|
||||
describe("buildOpenAICompatibleClientOptions", () => {
|
||||
it("adds Gemini's OpenAI-compatible base URL and client header", () => {
|
||||
const options = buildOpenAICompatibleClientOptions("googleai", "gemini-test-key");
|
||||
const options = buildOpenAICompatibleClientOptions(
|
||||
"googleai",
|
||||
"gemini-test-key",
|
||||
);
|
||||
|
||||
expect(options).toMatchObject({
|
||||
apiKey: "gemini-test-key",
|
||||
@@ -18,12 +23,28 @@ describe("buildOpenAICompatibleClientOptions", () => {
|
||||
});
|
||||
|
||||
it("keeps the default OpenAI-compatible config for OpenAI", () => {
|
||||
expect(buildOpenAICompatibleClientOptions("openai", "openai-test-key")).toEqual({
|
||||
expect(
|
||||
buildOpenAICompatibleClientOptions("openai", "openai-test-key"),
|
||||
).toEqual({
|
||||
apiKey: "openai-test-key",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("proxy helpers", () => {
|
||||
it("builds provider-scoped proxy resource paths", () => {
|
||||
expect(buildProxyResourcePath("googleai")).toBe("f/evals/ai/googleai");
|
||||
expect(buildProxyResourcePath("anthropic")).toBe("f/evals/ai/anthropic");
|
||||
});
|
||||
|
||||
it("adds auth and resource headers for workspace proxy requests", () => {
|
||||
expect(buildProxyHeaders("token-123", "f/evals/ai/googleai")).toEqual({
|
||||
Authorization: "Bearer token-123",
|
||||
"X-Resource-Path": "f/evals/ai/googleai",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveEvalModelProvider", () => {
|
||||
it("infers googleai from Gemini model ids", () => {
|
||||
expect(resolveEvalModelProvider("gemini-2.5-flash")).toEqual({
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import Anthropic from "@anthropic-ai/sdk";
|
||||
import OpenAI from "openai";
|
||||
import type { FrontendEvalModelConfig } from "../../../../core/models";
|
||||
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
|
||||
|
||||
export type FrontendEvalProvider = FrontendEvalModelConfig["provider"];
|
||||
|
||||
@@ -14,12 +15,34 @@ export interface ResolvedEvalModelProvider {
|
||||
model: string;
|
||||
}
|
||||
|
||||
const GEMINI_OPENAI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai/";
|
||||
export interface EvalProxyClientConfig {
|
||||
baseURL: string;
|
||||
bearerToken: string;
|
||||
resourcePath: string;
|
||||
}
|
||||
|
||||
const GEMINI_OPENAI_BASE_URL =
|
||||
"https://generativelanguage.googleapis.com/v1beta/openai/";
|
||||
const GEMINI_GOOG_API_CLIENT = "windmill-ai-evals/1.0";
|
||||
const EVAL_PROXY_RESOURCE_PREFIX = "f/evals/ai";
|
||||
|
||||
export function buildProxyHeaders(
|
||||
bearerToken: string,
|
||||
resourcePath: string,
|
||||
): Record<string, string> {
|
||||
return {
|
||||
Authorization: `Bearer ${bearerToken}`,
|
||||
"X-Resource-Path": resourcePath,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildProxyResourcePath(provider: FrontendEvalProvider): string {
|
||||
return `${EVAL_PROXY_RESOURCE_PREFIX}/${provider}`;
|
||||
}
|
||||
|
||||
export function buildOpenAICompatibleClientOptions(
|
||||
provider: Exclude<FrontendEvalProvider, "anthropic">,
|
||||
apiKey: string
|
||||
apiKey: string,
|
||||
): ConstructorParameters<typeof OpenAI>[0] {
|
||||
if (provider === "googleai") {
|
||||
return {
|
||||
@@ -34,26 +57,71 @@ export function buildOpenAICompatibleClientOptions(
|
||||
return { apiKey };
|
||||
}
|
||||
|
||||
export function createEvalClients(
|
||||
provider: FrontendEvalProvider,
|
||||
apiKey: string
|
||||
): EvalClients {
|
||||
if (provider === "anthropic") {
|
||||
function buildProxyOpenAIClientOptions(
|
||||
proxy: EvalProxyClientConfig,
|
||||
): ConstructorParameters<typeof OpenAI>[0] {
|
||||
return {
|
||||
apiKey: "unused",
|
||||
baseURL: proxy.baseURL,
|
||||
defaultHeaders: buildProxyHeaders(proxy.bearerToken, proxy.resourcePath),
|
||||
};
|
||||
}
|
||||
|
||||
export function createEvalClients(input: {
|
||||
provider: FrontendEvalProvider;
|
||||
apiKey: string;
|
||||
transport?: FrontendEvalTransport;
|
||||
proxy?: EvalProxyClientConfig;
|
||||
}): EvalClients {
|
||||
const transport = input.transport ?? "direct";
|
||||
|
||||
if (input.provider === "anthropic") {
|
||||
if (transport === "proxy") {
|
||||
if (!input.proxy) {
|
||||
throw new Error(
|
||||
"Missing proxy client configuration for proxy transport",
|
||||
);
|
||||
}
|
||||
return {
|
||||
openai: new OpenAI({ apiKey: "unused" }),
|
||||
anthropic: new Anthropic({
|
||||
apiKey: "unused",
|
||||
baseURL: input.proxy.baseURL,
|
||||
defaultHeaders: buildProxyHeaders(
|
||||
input.proxy.bearerToken,
|
||||
input.proxy.resourcePath,
|
||||
),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
openai: new OpenAI({ apiKey: "unused" }),
|
||||
anthropic: new Anthropic({ apiKey }),
|
||||
anthropic: new Anthropic({ apiKey: input.apiKey }),
|
||||
};
|
||||
}
|
||||
|
||||
if (transport === "proxy") {
|
||||
if (!input.proxy) {
|
||||
throw new Error("Missing proxy client configuration for proxy transport");
|
||||
}
|
||||
return {
|
||||
openai: new OpenAI(buildProxyOpenAIClientOptions(input.proxy)),
|
||||
anthropic: new Anthropic({ apiKey: "unused" }),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
openai: new OpenAI(buildOpenAICompatibleClientOptions(provider, apiKey)),
|
||||
openai: new OpenAI(
|
||||
buildOpenAICompatibleClientOptions(input.provider, input.apiKey),
|
||||
),
|
||||
anthropic: new Anthropic({ apiKey: "unused" }),
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveEvalModelProvider(
|
||||
model: string,
|
||||
provider?: FrontendEvalProvider
|
||||
provider?: FrontendEvalProvider,
|
||||
): ResolvedEvalModelProvider {
|
||||
if (provider) {
|
||||
return { provider, model };
|
||||
|
||||
@@ -1,32 +1,38 @@
|
||||
import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs'
|
||||
import type { AIProvider } from '$lib/gen/types.gen'
|
||||
import type { ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs";
|
||||
import type { AIProvider } from "$lib/gen/types.gen";
|
||||
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
|
||||
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
|
||||
|
||||
export interface TokenUsage {
|
||||
prompt: number
|
||||
completion: number
|
||||
total: number
|
||||
prompt: number;
|
||||
completion: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface ToolCallDetail {
|
||||
name: string
|
||||
arguments: Record<string, unknown>
|
||||
name: string;
|
||||
arguments: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface EvalRunnerOptions {
|
||||
maxIterations?: number
|
||||
model?: string
|
||||
workspace?: string
|
||||
provider?: AIProvider
|
||||
maxIterations?: number;
|
||||
model?: string;
|
||||
workspace?: string;
|
||||
provider?: AIProvider;
|
||||
transport?: FrontendEvalTransport;
|
||||
backend?: WindmillBackendSettings;
|
||||
proxyCaseId?: string;
|
||||
proxyAttempt?: number;
|
||||
}
|
||||
|
||||
export interface RawEvalResult<TOutput> {
|
||||
success: boolean
|
||||
output: TOutput
|
||||
error?: string
|
||||
tokenUsage: TokenUsage
|
||||
toolCallsCount: number
|
||||
toolsCalled: string[]
|
||||
toolCallDetails: ToolCallDetail[]
|
||||
iterations: number
|
||||
messages: ChatCompletionMessageParam[]
|
||||
success: boolean;
|
||||
output: TOutput;
|
||||
error?: string;
|
||||
tokenUsage: TokenUsage;
|
||||
toolCallsCount: number;
|
||||
toolsCalled: string[];
|
||||
toolCallDetails: ToolCallDetail[];
|
||||
iterations: number;
|
||||
messages: ChatCompletionMessageParam[];
|
||||
}
|
||||
|
||||
@@ -1,213 +1,228 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { mkdtemp, readFile, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
formatFrontendBenchmarkProgressEvent,
|
||||
parseFrontendBenchmarkProgressLine
|
||||
} from './progress'
|
||||
import type { BenchmarkRunResult } from '../../core/types'
|
||||
formatFrontendBenchmarkProgressEvent,
|
||||
parseFrontendBenchmarkProgressLine,
|
||||
} from "./progress";
|
||||
import type { BenchmarkRunResult } from "../../core/types";
|
||||
|
||||
const REPO_ROOT = fileURLToPath(new URL('../../../', import.meta.url))
|
||||
const FRONTEND_DIR = path.join(REPO_ROOT, 'frontend')
|
||||
const FRONTEND_BENCHMARK_TEST = '../ai_evals/adapters/frontend/vitestAdapter.test.ts'
|
||||
const FRONTEND_BENCHMARK_CONFIG = '../ai_evals/adapters/frontend/vitest.config.ts'
|
||||
const REPO_ROOT = fileURLToPath(new URL("../../../", import.meta.url));
|
||||
const FRONTEND_DIR = path.join(REPO_ROOT, "frontend");
|
||||
const FRONTEND_BENCHMARK_TEST =
|
||||
"../ai_evals/adapters/frontend/vitestAdapter.test.ts";
|
||||
const FRONTEND_BENCHMARK_CONFIG =
|
||||
"../ai_evals/adapters/frontend/vitest.config.ts";
|
||||
|
||||
export type FrontendMode = 'flow' | 'app' | 'script'
|
||||
export type FrontendMode = "flow" | "app" | "script";
|
||||
|
||||
export async function runFrontendBenchmarkAdapter(input: {
|
||||
mode: FrontendMode
|
||||
caseIds: string[]
|
||||
runs: number
|
||||
model?: string
|
||||
verbose?: boolean
|
||||
backendValidation?: string
|
||||
mode: FrontendMode;
|
||||
caseIds: string[];
|
||||
runs: number;
|
||||
model?: string;
|
||||
transport?: string;
|
||||
verbose?: boolean;
|
||||
backendValidation?: string;
|
||||
}): Promise<BenchmarkRunResult> {
|
||||
const tempDir = await mkdtemp(path.join(tmpdir(), 'wmill-frontend-benchmark-'))
|
||||
const outputPath = path.join(tempDir, 'result.json')
|
||||
const tempDir = await mkdtemp(
|
||||
path.join(tmpdir(), "wmill-frontend-benchmark-"),
|
||||
);
|
||||
const outputPath = path.join(tempDir, "result.json");
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
BROWSERSLIST_IGNORE_OLD_DATA: "1",
|
||||
WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH: outputPath,
|
||||
WMILL_FRONTEND_AI_EVAL_MODE: input.mode,
|
||||
WMILL_FRONTEND_AI_EVAL_CASE_IDS: JSON.stringify(input.caseIds),
|
||||
WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs),
|
||||
WMILL_FRONTEND_AI_EVAL_MODEL: input.model ?? "",
|
||||
WMILL_FRONTEND_AI_EVAL_PROGRESS: "1",
|
||||
WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? "1" : "0",
|
||||
WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION: input.backendValidation ?? "",
|
||||
};
|
||||
|
||||
try {
|
||||
await runVitestBenchmark(
|
||||
path.join(FRONTEND_DIR, 'node_modules', '.bin', 'vitest'),
|
||||
[
|
||||
'run',
|
||||
FRONTEND_BENCHMARK_TEST,
|
||||
'--project',
|
||||
'server',
|
||||
'--config',
|
||||
FRONTEND_BENCHMARK_CONFIG
|
||||
],
|
||||
{
|
||||
cwd: FRONTEND_DIR,
|
||||
env: {
|
||||
...process.env,
|
||||
BROWSERSLIST_IGNORE_OLD_DATA: '1',
|
||||
WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH: outputPath,
|
||||
WMILL_FRONTEND_AI_EVAL_MODE: input.mode,
|
||||
WMILL_FRONTEND_AI_EVAL_CASE_IDS: JSON.stringify(input.caseIds),
|
||||
WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs),
|
||||
WMILL_FRONTEND_AI_EVAL_MODEL: input.model ?? "",
|
||||
WMILL_FRONTEND_AI_EVAL_PROGRESS: '1',
|
||||
WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? '1' : '0',
|
||||
WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION: input.backendValidation ?? ''
|
||||
}
|
||||
}
|
||||
)
|
||||
if (input.transport) {
|
||||
env.WMILL_FRONTEND_AI_EVAL_TRANSPORT = input.transport;
|
||||
}
|
||||
|
||||
const raw = await readFile(outputPath, 'utf8')
|
||||
return JSON.parse(raw) as BenchmarkRunResult
|
||||
} catch (error) {
|
||||
throw new Error(`Frontend benchmark adapter failed:\n${toErrorMessage(error)}`)
|
||||
} finally {
|
||||
await rm(tempDir, { recursive: true, force: true })
|
||||
}
|
||||
try {
|
||||
await runVitestBenchmark(
|
||||
path.join(FRONTEND_DIR, "node_modules", ".bin", "vitest"),
|
||||
[
|
||||
"run",
|
||||
FRONTEND_BENCHMARK_TEST,
|
||||
"--project",
|
||||
"server",
|
||||
"--config",
|
||||
FRONTEND_BENCHMARK_CONFIG,
|
||||
],
|
||||
{
|
||||
cwd: FRONTEND_DIR,
|
||||
env,
|
||||
},
|
||||
);
|
||||
|
||||
const raw = await readFile(outputPath, "utf8");
|
||||
return JSON.parse(raw) as BenchmarkRunResult;
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Frontend benchmark adapter failed:\n${toErrorMessage(error)}`,
|
||||
);
|
||||
} finally {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function runVitestBenchmark(
|
||||
command: string,
|
||||
args: string[],
|
||||
options: {
|
||||
cwd: string
|
||||
env: NodeJS.ProcessEnv
|
||||
}
|
||||
command: string,
|
||||
args: string[],
|
||||
options: {
|
||||
cwd: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
},
|
||||
): Promise<void> {
|
||||
const child = spawn(command, args, {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
stdio: ['ignore', 'pipe', 'pipe']
|
||||
})
|
||||
const child = spawn(command, args, {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
let stderrLineBuffer = ''
|
||||
let assistantStreamOpen = false
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let stderrLineBuffer = "";
|
||||
let assistantStreamOpen = false;
|
||||
|
||||
child.stdout?.setEncoding('utf8')
|
||||
child.stdout?.on('data', (chunk: string) => {
|
||||
stdout += chunk
|
||||
})
|
||||
child.stdout?.setEncoding("utf8");
|
||||
child.stdout?.on("data", (chunk: string) => {
|
||||
stdout += chunk;
|
||||
});
|
||||
|
||||
child.stderr?.setEncoding('utf8')
|
||||
child.stderr?.on('data', (chunk: string) => {
|
||||
stderrLineBuffer += chunk
|
||||
const { remainder, passthrough, nextAssistantStreamOpen } = drainProgressLines(
|
||||
stderrLineBuffer,
|
||||
assistantStreamOpen
|
||||
)
|
||||
stderrLineBuffer = remainder
|
||||
stderr += passthrough
|
||||
assistantStreamOpen = nextAssistantStreamOpen
|
||||
})
|
||||
child.stderr?.setEncoding("utf8");
|
||||
child.stderr?.on("data", (chunk: string) => {
|
||||
stderrLineBuffer += chunk;
|
||||
const { remainder, passthrough, nextAssistantStreamOpen } =
|
||||
drainProgressLines(stderrLineBuffer, assistantStreamOpen);
|
||||
stderrLineBuffer = remainder;
|
||||
stderr += passthrough;
|
||||
assistantStreamOpen = nextAssistantStreamOpen;
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
child.on('error', reject)
|
||||
child.on('close', (code) => {
|
||||
if (stderrLineBuffer.length > 0) {
|
||||
const {
|
||||
remainder,
|
||||
passthrough,
|
||||
nextAssistantStreamOpen
|
||||
} = drainProgressLines(`${stderrLineBuffer}\n`, assistantStreamOpen)
|
||||
stderrLineBuffer = remainder
|
||||
stderr += passthrough
|
||||
assistantStreamOpen = nextAssistantStreamOpen
|
||||
}
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (stderrLineBuffer.length > 0) {
|
||||
const { remainder, passthrough, nextAssistantStreamOpen } =
|
||||
drainProgressLines(`${stderrLineBuffer}\n`, assistantStreamOpen);
|
||||
stderrLineBuffer = remainder;
|
||||
stderr += passthrough;
|
||||
assistantStreamOpen = nextAssistantStreamOpen;
|
||||
}
|
||||
|
||||
if (code === 0) {
|
||||
if (assistantStreamOpen) {
|
||||
process.stderr.write('\n')
|
||||
}
|
||||
resolve()
|
||||
return
|
||||
}
|
||||
if (code === 0) {
|
||||
if (assistantStreamOpen) {
|
||||
process.stderr.write("\n");
|
||||
}
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
const details = [`vitest exited with code ${code}`, stdout, stderr].filter(Boolean).join('\n')
|
||||
reject(new Error(details))
|
||||
})
|
||||
})
|
||||
const details = [`vitest exited with code ${code}`, stdout, stderr]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
reject(new Error(details));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function drainProgressLines(
|
||||
buffer: string,
|
||||
initialAssistantStreamOpen: boolean
|
||||
buffer: string,
|
||||
initialAssistantStreamOpen: boolean,
|
||||
): {
|
||||
remainder: string
|
||||
passthrough: string
|
||||
nextAssistantStreamOpen: boolean
|
||||
remainder: string;
|
||||
passthrough: string;
|
||||
nextAssistantStreamOpen: boolean;
|
||||
} {
|
||||
let remainder = buffer
|
||||
let passthrough = ''
|
||||
let assistantStreamOpen = initialAssistantStreamOpen
|
||||
let remainder = buffer;
|
||||
let passthrough = "";
|
||||
let assistantStreamOpen = initialAssistantStreamOpen;
|
||||
|
||||
while (true) {
|
||||
const newlineIndex = remainder.indexOf('\n')
|
||||
if (newlineIndex === -1) {
|
||||
return { remainder, passthrough, nextAssistantStreamOpen: assistantStreamOpen }
|
||||
}
|
||||
while (true) {
|
||||
const newlineIndex = remainder.indexOf("\n");
|
||||
if (newlineIndex === -1) {
|
||||
return {
|
||||
remainder,
|
||||
passthrough,
|
||||
nextAssistantStreamOpen: assistantStreamOpen,
|
||||
};
|
||||
}
|
||||
|
||||
const line = remainder.slice(0, newlineIndex).replace(/\r$/, '')
|
||||
remainder = remainder.slice(newlineIndex + 1)
|
||||
const line = remainder.slice(0, newlineIndex).replace(/\r$/, "");
|
||||
remainder = remainder.slice(newlineIndex + 1);
|
||||
|
||||
const progressEvent = parseFrontendBenchmarkProgressLine(line)
|
||||
if (progressEvent) {
|
||||
if (progressEvent.type === 'assistant-message-start') {
|
||||
if (assistantStreamOpen) {
|
||||
process.stderr.write('\n')
|
||||
}
|
||||
process.stderr.write(
|
||||
`${formatCasePrefix(progressEvent.caseNumber, progressEvent.totalCases)} ${progressEvent.caseId} attempt ${progressEvent.attempt}/${progressEvent.runs} assistant:\n`
|
||||
)
|
||||
assistantStreamOpen = true
|
||||
continue
|
||||
}
|
||||
const progressEvent = parseFrontendBenchmarkProgressLine(line);
|
||||
if (progressEvent) {
|
||||
if (progressEvent.type === "assistant-message-start") {
|
||||
if (assistantStreamOpen) {
|
||||
process.stderr.write("\n");
|
||||
}
|
||||
process.stderr.write(
|
||||
`${formatCasePrefix(progressEvent.caseNumber, progressEvent.totalCases)} ${progressEvent.caseId} attempt ${progressEvent.attempt}/${progressEvent.runs} assistant:\n`,
|
||||
);
|
||||
assistantStreamOpen = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (progressEvent.type === 'assistant-chunk') {
|
||||
process.stderr.write(progressEvent.chunk)
|
||||
continue
|
||||
}
|
||||
if (progressEvent.type === "assistant-chunk") {
|
||||
process.stderr.write(progressEvent.chunk);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (progressEvent.type === 'assistant-message-end') {
|
||||
if (assistantStreamOpen) {
|
||||
process.stderr.write('\n')
|
||||
}
|
||||
assistantStreamOpen = false
|
||||
continue
|
||||
}
|
||||
if (progressEvent.type === "assistant-message-end") {
|
||||
if (assistantStreamOpen) {
|
||||
process.stderr.write("\n");
|
||||
}
|
||||
assistantStreamOpen = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (assistantStreamOpen) {
|
||||
process.stderr.write('\n')
|
||||
assistantStreamOpen = false
|
||||
}
|
||||
process.stderr.write(`${formatFrontendBenchmarkProgressEvent(progressEvent)}\n`)
|
||||
continue
|
||||
}
|
||||
if (assistantStreamOpen) {
|
||||
process.stderr.write("\n");
|
||||
assistantStreamOpen = false;
|
||||
}
|
||||
process.stderr.write(
|
||||
`${formatFrontendBenchmarkProgressEvent(progressEvent)}\n`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (shouldSuppressFrontendStderrLine(line)) {
|
||||
continue
|
||||
}
|
||||
if (shouldSuppressFrontendStderrLine(line)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
passthrough += `${line}\n`
|
||||
process.stderr.write(`${line}\n`)
|
||||
}
|
||||
passthrough += `${line}\n`;
|
||||
process.stderr.write(`${line}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
function formatCasePrefix(caseNumber: number, totalCases: number): string {
|
||||
return `[${caseNumber}/${totalCases}]`
|
||||
return `[${caseNumber}/${totalCases}]`;
|
||||
}
|
||||
|
||||
function shouldSuppressFrontendStderrLine(line: string): boolean {
|
||||
return (
|
||||
line.startsWith('[baseline-browser-mapping] ') ||
|
||||
line.startsWith('Browserslist: browsers data (caniuse-lite) is ') ||
|
||||
line.includes('update-browserslist-db@latest') ||
|
||||
line.includes('update-db#readme')
|
||||
)
|
||||
return (
|
||||
line.startsWith("[baseline-browser-mapping] ") ||
|
||||
line.startsWith("Browserslist: browsers data (caniuse-lite) is ") ||
|
||||
line.includes("update-browserslist-db@latest") ||
|
||||
line.includes("update-db#readme")
|
||||
);
|
||||
}
|
||||
|
||||
function toErrorMessage(error: unknown): string {
|
||||
if (error instanceof Error) {
|
||||
return error.message
|
||||
}
|
||||
return String(error)
|
||||
if (error instanceof Error) {
|
||||
return error.message;
|
||||
}
|
||||
return String(error);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { WindmillBackendSettings } from "../../core/windmillBackendSettings";
|
||||
|
||||
const tokenCache = new Map<string, Promise<string>>();
|
||||
const sharedWorkspaceQueue = new Map<string, Promise<void>>();
|
||||
|
||||
export class WindmillBackendClient {
|
||||
constructor(private readonly settings: WindmillBackendSettings) {}
|
||||
|
||||
async withWorkspace<T>(
|
||||
caseId: string,
|
||||
attempt: number,
|
||||
body: (workspaceId: string) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const workspaceId =
|
||||
this.settings.workspaceOverride ??
|
||||
buildWorkspaceId(this.settings.workspacePrefix, caseId, attempt);
|
||||
|
||||
const run = async () => {
|
||||
await this.ensureWorkspace(workspaceId);
|
||||
|
||||
try {
|
||||
return await body(workspaceId);
|
||||
} finally {
|
||||
if (!this.settings.keepWorkspaces && !this.settings.workspaceOverride) {
|
||||
await this.deleteWorkspace(workspaceId).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (this.settings.workspaceOverride) {
|
||||
return await withSharedWorkspaceLock(workspaceId, run);
|
||||
}
|
||||
|
||||
return await run();
|
||||
}
|
||||
|
||||
async request(path: string, init?: RequestInit): Promise<Response> {
|
||||
const token = await this.getToken();
|
||||
return await fetch(`${this.settings.baseUrl}/api${path}`, {
|
||||
...init,
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
...(init?.headers ?? {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async getToken(): Promise<string> {
|
||||
const cacheKey = `${this.settings.baseUrl}|${this.settings.email}`;
|
||||
let tokenPromise = tokenCache.get(cacheKey);
|
||||
if (!tokenPromise) {
|
||||
tokenPromise = this.login().catch((error) => {
|
||||
if (tokenCache.get(cacheKey) === tokenPromise) {
|
||||
tokenCache.delete(cacheKey);
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
tokenCache.set(cacheKey, tokenPromise);
|
||||
}
|
||||
return await tokenPromise;
|
||||
}
|
||||
|
||||
async upsertResource(input: {
|
||||
workspaceId: string;
|
||||
path: string;
|
||||
resourceType: string;
|
||||
value: Record<string, unknown>;
|
||||
}): Promise<void> {
|
||||
const response = await this.request(
|
||||
`/w/${encodeURIComponent(input.workspaceId)}/resources/create?update_if_exists=true`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
path: input.path,
|
||||
resource_type: input.resourceType,
|
||||
value: input.value,
|
||||
}),
|
||||
},
|
||||
);
|
||||
await expectOk(response, `upsert resource ${input.path}`);
|
||||
}
|
||||
|
||||
private async ensureWorkspace(workspaceId: string): Promise<void> {
|
||||
const existsResponse = await this.request("/workspaces/exists", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ id: workspaceId }),
|
||||
});
|
||||
await expectOk(existsResponse, `check workspace ${workspaceId}`);
|
||||
|
||||
if ((await existsResponse.text()).trim() === "true") {
|
||||
return;
|
||||
}
|
||||
|
||||
const createResponse = await this.request("/workspaces/create", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ id: workspaceId, name: workspaceId }),
|
||||
});
|
||||
try {
|
||||
await expectOk(createResponse, `create workspace ${workspaceId}`);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (message.includes("maximum number of workspaces")) {
|
||||
throw new Error(
|
||||
`${message}. Reuse an existing workspace with WMILL_AI_EVAL_BACKEND_WORKSPACE=<workspace-id>.`,
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async deleteWorkspace(workspaceId: string): Promise<void> {
|
||||
const response = await this.request(
|
||||
`/workspaces/delete/${encodeURIComponent(workspaceId)}`,
|
||||
{
|
||||
method: "DELETE",
|
||||
},
|
||||
);
|
||||
await expectOk(response, `delete workspace ${workspaceId}`);
|
||||
}
|
||||
|
||||
private async login(): Promise<string> {
|
||||
const response = await fetch(`${this.settings.baseUrl}/api/auth/login`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
email: this.settings.email,
|
||||
password: this.settings.password,
|
||||
}),
|
||||
});
|
||||
await expectOk(response, "login to Windmill backend");
|
||||
return (await response.text()).trim();
|
||||
}
|
||||
}
|
||||
|
||||
async function withSharedWorkspaceLock<T>(
|
||||
workspaceId: string,
|
||||
body: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const previous = sharedWorkspaceQueue.get(workspaceId) ?? Promise.resolve();
|
||||
let releaseCurrent: (() => void) | undefined;
|
||||
const current = new Promise<void>((resolve) => {
|
||||
releaseCurrent = resolve;
|
||||
});
|
||||
const tail = previous.catch(() => undefined).then(() => current);
|
||||
sharedWorkspaceQueue.set(workspaceId, tail);
|
||||
|
||||
await previous.catch(() => undefined);
|
||||
|
||||
try {
|
||||
return await body();
|
||||
} finally {
|
||||
releaseCurrent?.();
|
||||
if (sharedWorkspaceQueue.get(workspaceId) === tail) {
|
||||
sharedWorkspaceQueue.delete(workspaceId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function buildWorkspaceId(
|
||||
prefix: string,
|
||||
caseId: string,
|
||||
attempt: number,
|
||||
): string {
|
||||
const caseSlug = caseId
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9-]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 30);
|
||||
const suffix = randomUUID().slice(0, 8);
|
||||
return `${prefix}-${caseSlug || "case"}-a${attempt}-${suffix}`;
|
||||
}
|
||||
|
||||
async function expectOk(response: Response, context: string): Promise<void> {
|
||||
if (response.ok) {
|
||||
return;
|
||||
}
|
||||
throw new Error(
|
||||
`${context} failed: ${response.status} ${response.statusText} - ${await response.text()}`,
|
||||
);
|
||||
}
|
||||
+79
-21
@@ -27,11 +27,18 @@ import { EVAL_MODES, type EvalMode } from "../core/types";
|
||||
import { DEFAULT_JUDGE_MODEL } from "../core/judge";
|
||||
import { createCliModeRunner } from "../modes/cli";
|
||||
import { runFrontendBenchmarkAdapter } from "../adapters/frontend/runtime";
|
||||
import {
|
||||
FRONTEND_EVAL_TRANSPORTS,
|
||||
type FrontendEvalTransport,
|
||||
parseFrontendEvalTransport,
|
||||
} from "../core/frontendTransport";
|
||||
|
||||
async function main() {
|
||||
const program = new Command()
|
||||
.name("bun run cli --")
|
||||
.description("Run AI eval cases against the current production prompts and guidance")
|
||||
.description(
|
||||
"Run AI eval cases against the current production prompts and guidance",
|
||||
)
|
||||
.showHelpAfterError()
|
||||
.showSuggestionAfterError()
|
||||
.addHelpText(
|
||||
@@ -53,7 +60,7 @@ async function main() {
|
||||
"",
|
||||
"Models:",
|
||||
getEvalModelHelpText(),
|
||||
].join("\n")
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
program
|
||||
@@ -76,15 +83,33 @@ async function main() {
|
||||
.description("Run one benchmark mode")
|
||||
.argument("<mode>", "cli, flow, script, or app", parseMode)
|
||||
.argument("[caseIds...]", "specific case ids to run")
|
||||
.option("--runs <n>", "number of attempts per case", parsePositiveInteger, 1)
|
||||
.option(
|
||||
"--runs <n>",
|
||||
"number of attempts per case",
|
||||
parsePositiveInteger,
|
||||
1,
|
||||
)
|
||||
.option("--output <path>", "write the result JSON to this path")
|
||||
.option("--model <name>", `model alias (${EVAL_MODELS.map((entry) => entry.id).join(", ")})`)
|
||||
.option("--models <names>", "comma-separated model aliases to run sequentially")
|
||||
.option(
|
||||
"--model <name>",
|
||||
`model alias (${EVAL_MODELS.map((entry) => entry.id).join(", ")})`,
|
||||
)
|
||||
.option(
|
||||
"--models <names>",
|
||||
"comma-separated model aliases to run sequentially",
|
||||
)
|
||||
.option(
|
||||
"--transport <mode>",
|
||||
`frontend transport (${FRONTEND_EVAL_TRANSPORTS.join(", ")})`,
|
||||
)
|
||||
.option("--verbose", "stream assistant output during frontend runs")
|
||||
.option("--record", "append a compact summary line to ai_evals/history/<mode>.jsonl")
|
||||
.option(
|
||||
"--record",
|
||||
"append a compact summary line to ai_evals/history/<mode>.jsonl",
|
||||
)
|
||||
.option(
|
||||
"--backend-validation <mode>",
|
||||
`backend smoke validation (${BACKEND_VALIDATION_MODES.join(", ")})`
|
||||
`backend smoke validation (${BACKEND_VALIDATION_MODES.join(", ")})`,
|
||||
)
|
||||
.action(
|
||||
async (
|
||||
@@ -95,10 +120,11 @@ async function main() {
|
||||
output?: string;
|
||||
model?: string;
|
||||
models?: string;
|
||||
transport?: string;
|
||||
verbose?: boolean;
|
||||
record?: boolean;
|
||||
backendValidation?: string;
|
||||
}
|
||||
},
|
||||
) => {
|
||||
await handleRun({
|
||||
mode,
|
||||
@@ -107,11 +133,14 @@ async function main() {
|
||||
outputPath: options.output,
|
||||
model: options.model,
|
||||
models: options.models,
|
||||
transport: options.transport
|
||||
? parseFrontendEvalTransport(options.transport)
|
||||
: undefined,
|
||||
verbose: options.verbose ?? false,
|
||||
record: options.record ?? false,
|
||||
backendValidation: options.backendValidation,
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
await program.parseAsync(process.argv);
|
||||
@@ -137,7 +166,10 @@ function handleModels() {
|
||||
...(model.frontend ? ["flow", "script", "app"] : []),
|
||||
...(model.cli ? ["cli"] : []),
|
||||
];
|
||||
const aliases = [model.id, ...model.aliases.filter((alias) => alias !== model.id)];
|
||||
const aliases = [
|
||||
model.id,
|
||||
...model.aliases.filter((alias) => alias !== model.id),
|
||||
];
|
||||
process.stdout.write(`- ${model.id}: ${model.label}\n`);
|
||||
process.stdout.write(` aliases: ${aliases.join(", ")}\n`);
|
||||
process.stdout.write(` modes: ${supports.join(", ")}\n`);
|
||||
@@ -152,48 +184,72 @@ async function handleRun(input: {
|
||||
outputPath?: string;
|
||||
model?: string;
|
||||
models?: string;
|
||||
transport?: FrontendEvalTransport;
|
||||
verbose: boolean;
|
||||
record: boolean;
|
||||
backendValidation?: string;
|
||||
}) {
|
||||
if (input.record && input.caseIds.length > 0) {
|
||||
throw new Error("--record only supports full-suite runs; omit case ids to record history");
|
||||
throw new Error(
|
||||
"--record only supports full-suite runs; omit case ids to record history",
|
||||
);
|
||||
}
|
||||
if (input.model && input.models) {
|
||||
throw new Error("Use either --model or --models, not both");
|
||||
}
|
||||
if (input.mode === "cli" && input.transport === "proxy") {
|
||||
throw new Error(
|
||||
"--transport proxy is only supported for flow, script, and app modes",
|
||||
);
|
||||
}
|
||||
|
||||
const selectedCases = await loadSelectedCases(input.mode, input.caseIds);
|
||||
const models = resolveRequestedModels(input.mode, input.model, input.models);
|
||||
const backendValidation = parseBackendValidationMode(
|
||||
input.backendValidation ?? process.env.WMILL_AI_EVAL_BACKEND_VALIDATION
|
||||
input.backendValidation ?? process.env.WMILL_AI_EVAL_BACKEND_VALIDATION,
|
||||
);
|
||||
if (input.outputPath && models.length > 1) {
|
||||
throw new Error("--output only supports a single model run");
|
||||
}
|
||||
if (backendValidation !== "off" && input.mode !== "flow" && input.mode !== "script") {
|
||||
throw new Error("--backend-validation currently supports only flow and script modes");
|
||||
if (
|
||||
backendValidation !== "off" &&
|
||||
input.mode !== "flow" &&
|
||||
input.mode !== "script"
|
||||
) {
|
||||
throw new Error(
|
||||
"--backend-validation currently supports only flow and script modes",
|
||||
);
|
||||
}
|
||||
|
||||
const summaries: Array<{ label: string; passRate: number; averageDurationMs: number }> = [];
|
||||
const summaries: Array<{
|
||||
label: string;
|
||||
passRate: number;
|
||||
averageDurationMs: number;
|
||||
}> = [];
|
||||
|
||||
for (const [index, model] of models.entries()) {
|
||||
const runModel = formatRunModelLabel(input.mode, model);
|
||||
if (models.length > 1) {
|
||||
process.stdout.write(
|
||||
`${index > 0 ? "\n" : ""}=== ${input.mode} ${model.id} (${runModel}) ===\n`
|
||||
`${index > 0 ? "\n" : ""}=== ${input.mode} ${model.id} (${runModel}) ===\n`,
|
||||
);
|
||||
}
|
||||
process.stderr.write(`Starting ${input.mode} benchmark...\n`);
|
||||
|
||||
const result =
|
||||
input.mode === "cli"
|
||||
? await runCliBenchmark(selectedCases, input.runs, getCliEvalModel(model), runModel)
|
||||
? await runCliBenchmark(
|
||||
selectedCases,
|
||||
input.runs,
|
||||
getCliEvalModel(model),
|
||||
runModel,
|
||||
)
|
||||
: await runFrontendBenchmarkAdapter({
|
||||
mode: input.mode,
|
||||
caseIds: input.caseIds,
|
||||
runs: input.runs,
|
||||
model: model.id,
|
||||
transport: input.transport,
|
||||
verbose: input.verbose,
|
||||
backendValidation,
|
||||
});
|
||||
@@ -225,7 +281,7 @@ async function handleRun(input: {
|
||||
process.stdout.write("\nModel summary\n");
|
||||
for (const summary of summaries) {
|
||||
process.stdout.write(
|
||||
`- ${summary.label}: ${formatPercent(summary.passRate)} | ${Math.round(summary.averageDurationMs)}ms\n`
|
||||
`- ${summary.label}: ${formatPercent(summary.passRate)} | ${Math.round(summary.averageDurationMs)}ms\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -235,7 +291,7 @@ async function runCliBenchmark(
|
||||
cases: Awaited<ReturnType<typeof loadSelectedCases>>,
|
||||
runs: number,
|
||||
model: ReturnType<typeof getCliEvalModel>,
|
||||
runModel: string
|
||||
runModel: string,
|
||||
) {
|
||||
const caseResults = await runSuite({
|
||||
modeRunner: createCliModeRunner(model),
|
||||
@@ -258,7 +314,9 @@ function parseMode(value: string): EvalMode {
|
||||
if (EVAL_MODES.includes(value as EvalMode)) {
|
||||
return value as EvalMode;
|
||||
}
|
||||
throw new InvalidArgumentError(`mode must be one of: ${EVAL_MODES.join(", ")}`);
|
||||
throw new InvalidArgumentError(
|
||||
`mode must be one of: ${EVAL_MODES.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
function parseOptionalMode(value: string | undefined): EvalMode | undefined {
|
||||
@@ -276,7 +334,7 @@ function parsePositiveInteger(value: string): number {
|
||||
function resolveRequestedModels(
|
||||
mode: EvalMode,
|
||||
singleModel?: string,
|
||||
multipleModels?: string
|
||||
multipleModels?: string,
|
||||
): EvalModelSpec[] {
|
||||
if (!multipleModels) {
|
||||
return [resolveEvalModel(mode, singleModel)];
|
||||
|
||||
+153
-54
@@ -7,7 +7,14 @@ const BACKEND_ROOT = "/__ai_evals__/backend";
|
||||
const FRONTEND_REACT_SHIM_PATH = `${FRONTEND_ROOT}/__react_shim__.d.ts`;
|
||||
const FRONTEND_WMILL_TYPES_PATH = `${FRONTEND_ROOT}/wmill.d.ts`;
|
||||
const BACKEND_WINDMILL_CLIENT_SHIM_PATH = `${BACKEND_ROOT}/__windmill_client__.d.ts`;
|
||||
const TS_LIKE_LANGUAGES = new Set(["bun", "deno", "nativets", "bunnative", "ts", "typescript"]);
|
||||
const TS_LIKE_LANGUAGES = new Set([
|
||||
"bun",
|
||||
"deno",
|
||||
"nativets",
|
||||
"bunnative",
|
||||
"ts",
|
||||
"typescript",
|
||||
]);
|
||||
const JS_LIKE_LANGUAGES = new Set(["javascript", "js", "nodejs"]);
|
||||
const SAFE_TYPE_REFERENCE_NAMES = new Set([
|
||||
"Array",
|
||||
@@ -121,20 +128,26 @@ export interface AppDiagnosticsResult {
|
||||
}
|
||||
|
||||
export function buildAppWmillTypes(
|
||||
backend: Record<string, AppDiagnosticRunnable> = {}
|
||||
backend: Record<string, AppDiagnosticRunnable> = {},
|
||||
): string {
|
||||
return `// THIS FILE IS READ-ONLY
|
||||
// AND GENERATED AUTOMATICALLY FROM YOUR RUNNABLES
|
||||
|
||||
export declare const backend: {
|
||||
${Object.entries(backend)
|
||||
.map(([key, runnable]) => ` ${JSON.stringify(key)}: ${getRunnableSignature(runnable, false)};`)
|
||||
.map(
|
||||
([key, runnable]) =>
|
||||
` ${JSON.stringify(key)}: ${getRunnableSignature(runnable, false)};`,
|
||||
)
|
||||
.join("\n")}
|
||||
};
|
||||
|
||||
export declare const backendAsync: {
|
||||
${Object.entries(backend)
|
||||
.map(([key, runnable]) => ` ${JSON.stringify(key)}: ${getRunnableSignature(runnable, true)};`)
|
||||
.map(
|
||||
([key, runnable]) =>
|
||||
` ${JSON.stringify(key)}: ${getRunnableSignature(runnable, true)};`,
|
||||
)
|
||||
.join("\n")}
|
||||
};
|
||||
|
||||
@@ -165,16 +178,26 @@ export function collectAppDiagnostics(input: {
|
||||
frontend: Record<string, string>;
|
||||
backend: Record<string, AppDiagnosticRunnable>;
|
||||
}): AppDiagnosticsResult {
|
||||
const frontendDiagnostics = collectFrontendDiagnostics(input.frontend, input.backend);
|
||||
const frontendDiagnostics = collectFrontendDiagnostics(
|
||||
input.frontend,
|
||||
input.backend,
|
||||
);
|
||||
const backendDiagnostics = collectBackendDiagnostics(input.backend);
|
||||
const diagnostics = dedupeDiagnostics([...frontendDiagnostics, ...backendDiagnostics]).sort(compareDiagnostics);
|
||||
const diagnostics = dedupeDiagnostics([
|
||||
...frontendDiagnostics,
|
||||
...backendDiagnostics,
|
||||
]).sort(compareDiagnostics);
|
||||
|
||||
return {
|
||||
diagnostics,
|
||||
lintResult: {
|
||||
errors: {
|
||||
frontend: groupMessages(diagnostics.filter((diagnostic) => diagnostic.source === "frontend")),
|
||||
backend: groupMessages(diagnostics.filter((diagnostic) => diagnostic.source === "backend")),
|
||||
frontend: groupMessages(
|
||||
diagnostics.filter((diagnostic) => diagnostic.source === "frontend"),
|
||||
),
|
||||
backend: groupMessages(
|
||||
diagnostics.filter((diagnostic) => diagnostic.source === "backend"),
|
||||
),
|
||||
},
|
||||
warnings: {
|
||||
frontend: {},
|
||||
@@ -188,20 +211,26 @@ export function collectAppDiagnostics(input: {
|
||||
|
||||
function collectFrontendDiagnostics(
|
||||
frontend: Record<string, string>,
|
||||
backend: Record<string, AppDiagnosticRunnable>
|
||||
backend: Record<string, AppDiagnosticRunnable>,
|
||||
): AppStaticDiagnostic[] {
|
||||
const frontendFiles = Object.entries(frontend)
|
||||
.filter(([filePath]) => isFrontendCodeFile(filePath))
|
||||
.map(([filePath, content]) => [toFrontendVirtualPath(filePath), content] as const);
|
||||
.map(
|
||||
([filePath, content]) =>
|
||||
[toFrontendVirtualPath(filePath), content] as const,
|
||||
);
|
||||
|
||||
const virtualFiles = new Map<string, string>([
|
||||
[FRONTEND_REACT_SHIM_PATH, FRONTEND_REACT_SHIM],
|
||||
[FRONTEND_WMILL_TYPES_PATH, wrapModuleDeclaration("wmill", buildAppWmillTypes(backend))],
|
||||
[
|
||||
FRONTEND_WMILL_TYPES_PATH,
|
||||
wrapModuleDeclaration("wmill", buildAppWmillTypes(backend)),
|
||||
],
|
||||
...frontendFiles,
|
||||
]);
|
||||
const host = createVirtualCompilerHost(
|
||||
virtualFiles,
|
||||
getFrontendCompilerOptions()
|
||||
getFrontendCompilerOptions(),
|
||||
);
|
||||
const rootNames = [...virtualFiles.keys()];
|
||||
const program = ts.createProgram({
|
||||
@@ -210,9 +239,8 @@ function collectFrontendDiagnostics(
|
||||
host,
|
||||
});
|
||||
|
||||
return ts
|
||||
.getPreEmitDiagnostics(program)
|
||||
.flatMap((diagnostic) => mapTypeScriptDiagnostic({
|
||||
return ts.getPreEmitDiagnostics(program).flatMap((diagnostic) =>
|
||||
mapTypeScriptDiagnostic({
|
||||
diagnostic,
|
||||
source: "frontend",
|
||||
toTarget(fileName) {
|
||||
@@ -228,18 +256,22 @@ function collectFrontendDiagnostics(
|
||||
}
|
||||
return normalized.slice(FRONTEND_ROOT.length);
|
||||
},
|
||||
}));
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function collectBackendDiagnostics(
|
||||
backend: Record<string, AppDiagnosticRunnable>
|
||||
backend: Record<string, AppDiagnosticRunnable>,
|
||||
): AppStaticDiagnostic[] {
|
||||
const backendFiles = Object.entries(backend)
|
||||
.filter(([, runnable]) => isTypeCheckableBackendRunnable(runnable))
|
||||
.map(([key, runnable]) => [
|
||||
`${BACKEND_ROOT}/${key}/main.${getBackendFileExtension(runnable.inlineScript?.language)}`,
|
||||
runnable.inlineScript?.content ?? "",
|
||||
] as const);
|
||||
.map(
|
||||
([key, runnable]) =>
|
||||
[
|
||||
`${BACKEND_ROOT}/${key}/main.${getBackendFileExtension(runnable.inlineScript?.language)}`,
|
||||
runnable.inlineScript?.content ?? "",
|
||||
] as const,
|
||||
);
|
||||
|
||||
if (backendFiles.length === 0) {
|
||||
return [];
|
||||
@@ -251,7 +283,7 @@ function collectBackendDiagnostics(
|
||||
]);
|
||||
const host = createVirtualCompilerHost(
|
||||
virtualFiles,
|
||||
getBackendCompilerOptions()
|
||||
getBackendCompilerOptions(),
|
||||
);
|
||||
const rootNames = [...virtualFiles.keys()];
|
||||
const program = ts.createProgram({
|
||||
@@ -260,9 +292,8 @@ function collectBackendDiagnostics(
|
||||
host,
|
||||
});
|
||||
|
||||
return ts
|
||||
.getPreEmitDiagnostics(program)
|
||||
.flatMap((diagnostic) => mapTypeScriptDiagnostic({
|
||||
return ts.getPreEmitDiagnostics(program).flatMap((diagnostic) =>
|
||||
mapTypeScriptDiagnostic({
|
||||
diagnostic,
|
||||
source: "backend",
|
||||
toTarget(fileName) {
|
||||
@@ -277,7 +308,8 @@ function collectBackendDiagnostics(
|
||||
const runnableKey = relativePath.split("/")[0];
|
||||
return runnableKey || null;
|
||||
},
|
||||
}));
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function getFrontendCompilerOptions(): ts.CompilerOptions {
|
||||
@@ -317,25 +349,37 @@ function getBackendCompilerOptions(): ts.CompilerOptions {
|
||||
|
||||
function createVirtualCompilerHost(
|
||||
files: Map<string, string>,
|
||||
options: ts.CompilerOptions
|
||||
options: ts.CompilerOptions,
|
||||
): ts.CompilerHost {
|
||||
const originalHost = ts.createCompilerHost(options, true);
|
||||
const originalGetSourceFile = originalHost.getSourceFile.bind(originalHost);
|
||||
const originalReadFile = originalHost.readFile.bind(originalHost);
|
||||
const originalFileExists = originalHost.fileExists.bind(originalHost);
|
||||
const originalDirectoryExists = originalHost.directoryExists?.bind(originalHost);
|
||||
const originalGetDirectories = originalHost.getDirectories?.bind(originalHost);
|
||||
const originalDirectoryExists =
|
||||
originalHost.directoryExists?.bind(originalHost);
|
||||
const originalGetDirectories =
|
||||
originalHost.getDirectories?.bind(originalHost);
|
||||
|
||||
return {
|
||||
...originalHost,
|
||||
getCurrentDirectory: () => "/",
|
||||
getSourceFile(fileName, languageVersion, onError, shouldCreateNewSourceFile) {
|
||||
getSourceFile(
|
||||
fileName,
|
||||
languageVersion,
|
||||
onError,
|
||||
shouldCreateNewSourceFile,
|
||||
) {
|
||||
const normalized = normalizeFileName(fileName);
|
||||
const content = files.get(normalized);
|
||||
if (content !== undefined) {
|
||||
return ts.createSourceFile(fileName, content, languageVersion, true);
|
||||
}
|
||||
return originalGetSourceFile(fileName, languageVersion, onError, shouldCreateNewSourceFile);
|
||||
return originalGetSourceFile(
|
||||
fileName,
|
||||
languageVersion,
|
||||
onError,
|
||||
shouldCreateNewSourceFile,
|
||||
);
|
||||
},
|
||||
readFile(fileName) {
|
||||
const normalized = normalizeFileName(fileName);
|
||||
@@ -347,7 +391,11 @@ function createVirtualCompilerHost(
|
||||
},
|
||||
directoryExists(dirName) {
|
||||
const normalized = normalizeFileName(dirName);
|
||||
return hasVirtualDirectory(files, normalized) || originalDirectoryExists?.(dirName) || false;
|
||||
return (
|
||||
hasVirtualDirectory(files, normalized) ||
|
||||
originalDirectoryExists?.(dirName) ||
|
||||
false
|
||||
);
|
||||
},
|
||||
getDirectories(dirName) {
|
||||
const normalized = normalizeFileName(dirName);
|
||||
@@ -386,7 +434,9 @@ function mapTypeScriptDiagnostic(input: {
|
||||
{
|
||||
source,
|
||||
target,
|
||||
message: ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n").trim(),
|
||||
message: ts
|
||||
.flattenDiagnosticMessageText(diagnostic.messageText, "\n")
|
||||
.trim(),
|
||||
line: position ? position.line + 1 : undefined,
|
||||
column: position ? position.character + 1 : undefined,
|
||||
code: diagnostic.code,
|
||||
@@ -395,7 +445,7 @@ function mapTypeScriptDiagnostic(input: {
|
||||
}
|
||||
|
||||
function groupMessages(
|
||||
diagnostics: AppStaticDiagnostic[]
|
||||
diagnostics: AppStaticDiagnostic[],
|
||||
): Record<string, string[]> {
|
||||
const grouped: Record<string, string[]> = {};
|
||||
|
||||
@@ -414,7 +464,9 @@ function formatLintMessage(diagnostic: AppStaticDiagnostic): string {
|
||||
return diagnostic.message;
|
||||
}
|
||||
|
||||
function dedupeDiagnostics(diagnostics: AppStaticDiagnostic[]): AppStaticDiagnostic[] {
|
||||
function dedupeDiagnostics(
|
||||
diagnostics: AppStaticDiagnostic[],
|
||||
): AppStaticDiagnostic[] {
|
||||
const uniqueDiagnostics = new Map<string, AppStaticDiagnostic>();
|
||||
|
||||
for (const diagnostic of diagnostics) {
|
||||
@@ -434,7 +486,10 @@ function dedupeDiagnostics(diagnostics: AppStaticDiagnostic[]): AppStaticDiagnos
|
||||
return [...uniqueDiagnostics.values()];
|
||||
}
|
||||
|
||||
function compareDiagnostics(a: AppStaticDiagnostic, b: AppStaticDiagnostic): number {
|
||||
function compareDiagnostics(
|
||||
a: AppStaticDiagnostic,
|
||||
b: AppStaticDiagnostic,
|
||||
): number {
|
||||
if (a.source !== b.source) {
|
||||
return a.source.localeCompare(b.source);
|
||||
}
|
||||
@@ -464,7 +519,10 @@ function normalizeFileName(fileName: string): string {
|
||||
return path.posix.normalize(fileName.replace(/\\/g, "/"));
|
||||
}
|
||||
|
||||
function hasVirtualDirectory(files: Map<string, string>, dirName: string): boolean {
|
||||
function hasVirtualDirectory(
|
||||
files: Map<string, string>,
|
||||
dirName: string,
|
||||
): boolean {
|
||||
const normalizedDirectory = dirName.endsWith("/") ? dirName : `${dirName}/`;
|
||||
for (const fileName of files.keys()) {
|
||||
if (fileName === dirName || fileName.startsWith(normalizedDirectory)) {
|
||||
@@ -474,7 +532,10 @@ function hasVirtualDirectory(files: Map<string, string>, dirName: string): boole
|
||||
return false;
|
||||
}
|
||||
|
||||
function listVirtualDirectories(files: Map<string, string>, dirName: string): string[] {
|
||||
function listVirtualDirectories(
|
||||
files: Map<string, string>,
|
||||
dirName: string,
|
||||
): string[] {
|
||||
const normalizedDirectory = dirName.endsWith("/") ? dirName : `${dirName}/`;
|
||||
const directories = new Set<string>();
|
||||
|
||||
@@ -504,14 +565,16 @@ function wrapModuleDeclaration(moduleName: string, content: string): string {
|
||||
|
||||
function getRunnableSignature(
|
||||
runnable: AppDiagnosticRunnable | undefined,
|
||||
asyncMode: boolean
|
||||
asyncMode: boolean,
|
||||
): string {
|
||||
const returnType = asyncMode ? "Promise<string>" : "Promise<any>";
|
||||
const parameter = getRunnableParameterSignature(runnable);
|
||||
return `${parameter} => ${returnType}`;
|
||||
}
|
||||
|
||||
function getRunnableParameterSignature(runnable: AppDiagnosticRunnable | undefined): string {
|
||||
function getRunnableParameterSignature(
|
||||
runnable: AppDiagnosticRunnable | undefined,
|
||||
): string {
|
||||
const parameterInfo = getRunnableParameterInfo(runnable);
|
||||
if (!parameterInfo) {
|
||||
return "()";
|
||||
@@ -525,9 +588,12 @@ function getRunnableParameterSignature(runnable: AppDiagnosticRunnable | undefin
|
||||
}
|
||||
|
||||
function getRunnableParameterInfo(
|
||||
runnable: AppDiagnosticRunnable | undefined
|
||||
runnable: AppDiagnosticRunnable | undefined,
|
||||
): { typeText?: string; optional: boolean } | null {
|
||||
if (!runnable?.inlineScript?.content || !isTypeCheckableBackendRunnable(runnable)) {
|
||||
if (
|
||||
!runnable?.inlineScript?.content ||
|
||||
!isTypeCheckableBackendRunnable(runnable)
|
||||
) {
|
||||
return { typeText: "any", optional: true };
|
||||
}
|
||||
|
||||
@@ -536,7 +602,7 @@ function getRunnableParameterInfo(
|
||||
runnable.inlineScript.content,
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
getScriptKindForLanguage(runnable.inlineScript.language)
|
||||
getScriptKindForLanguage(runnable.inlineScript.language),
|
||||
);
|
||||
const mainDeclaration = findExportedMainDeclaration(sourceFile);
|
||||
|
||||
@@ -545,7 +611,8 @@ function getRunnableParameterInfo(
|
||||
}
|
||||
|
||||
const [parameter] = mainDeclaration.parameters;
|
||||
const optional = Boolean(parameter.questionToken) || Boolean(parameter.initializer);
|
||||
const optional =
|
||||
Boolean(parameter.questionToken) || Boolean(parameter.initializer);
|
||||
|
||||
if (!parameter.type || !isPortableTypeNode(parameter.type)) {
|
||||
return { typeText: "any", optional: true };
|
||||
@@ -558,10 +625,14 @@ function getRunnableParameterInfo(
|
||||
}
|
||||
|
||||
function findExportedMainDeclaration(
|
||||
sourceFile: ts.SourceFile
|
||||
sourceFile: ts.SourceFile,
|
||||
): ts.SignatureDeclarationBase | null {
|
||||
for (const statement of sourceFile.statements) {
|
||||
if (ts.isFunctionDeclaration(statement) && statement.name?.text === "main" && hasExportModifier(statement)) {
|
||||
if (
|
||||
ts.isFunctionDeclaration(statement) &&
|
||||
statement.name?.text === "main" &&
|
||||
hasExportModifier(statement)
|
||||
) {
|
||||
return statement;
|
||||
}
|
||||
|
||||
@@ -570,11 +641,18 @@ function findExportedMainDeclaration(
|
||||
}
|
||||
|
||||
for (const declaration of statement.declarationList.declarations) {
|
||||
if (!ts.isIdentifier(declaration.name) || declaration.name.text !== "main") {
|
||||
if (
|
||||
!ts.isIdentifier(declaration.name) ||
|
||||
declaration.name.text !== "main"
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const initializer = declaration.initializer;
|
||||
if (initializer && (ts.isArrowFunction(initializer) || ts.isFunctionExpression(initializer))) {
|
||||
if (
|
||||
initializer &&
|
||||
(ts.isArrowFunction(initializer) ||
|
||||
ts.isFunctionExpression(initializer))
|
||||
) {
|
||||
return initializer;
|
||||
}
|
||||
}
|
||||
@@ -584,7 +662,14 @@ function findExportedMainDeclaration(
|
||||
}
|
||||
|
||||
function hasExportModifier(node: ts.Node): boolean {
|
||||
return Boolean(node.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword));
|
||||
const modifiers = ts.canHaveModifiers(node)
|
||||
? ts.getModifiers(node)
|
||||
: undefined;
|
||||
return Boolean(
|
||||
modifiers?.some(
|
||||
(modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function isPortableTypeNode(node: ts.TypeNode): boolean {
|
||||
@@ -607,10 +692,15 @@ function isPortableTypeNode(node: ts.TypeNode): boolean {
|
||||
}
|
||||
|
||||
if (ts.isTypeReferenceNode(node)) {
|
||||
if (!ts.isIdentifier(node.typeName) || !SAFE_TYPE_REFERENCE_NAMES.has(node.typeName.text)) {
|
||||
if (
|
||||
!ts.isIdentifier(node.typeName) ||
|
||||
!SAFE_TYPE_REFERENCE_NAMES.has(node.typeName.text)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return (node.typeArguments ?? []).every((typeArgument) => isPortableTypeNode(typeArgument));
|
||||
return (node.typeArguments ?? []).every((typeArgument) =>
|
||||
isPortableTypeNode(typeArgument),
|
||||
);
|
||||
}
|
||||
|
||||
return false;
|
||||
@@ -637,10 +727,17 @@ function isKeywordTypeNode(node: ts.TypeNode): boolean {
|
||||
|
||||
function isFrontendCodeFile(filePath: string): boolean {
|
||||
const extension = path.posix.extname(filePath).toLowerCase();
|
||||
return extension === ".js" || extension === ".jsx" || extension === ".ts" || extension === ".tsx";
|
||||
return (
|
||||
extension === ".js" ||
|
||||
extension === ".jsx" ||
|
||||
extension === ".ts" ||
|
||||
extension === ".tsx"
|
||||
);
|
||||
}
|
||||
|
||||
function isTypeCheckableBackendRunnable(runnable: AppDiagnosticRunnable | undefined): boolean {
|
||||
function isTypeCheckableBackendRunnable(
|
||||
runnable: AppDiagnosticRunnable | undefined,
|
||||
): boolean {
|
||||
if (!runnable || runnable.type !== "inline") {
|
||||
return false;
|
||||
}
|
||||
@@ -655,5 +752,7 @@ function getBackendFileExtension(language: string | undefined): string {
|
||||
|
||||
function getScriptKindForLanguage(language: string | undefined): ts.ScriptKind {
|
||||
const normalizedLanguage = language?.toLowerCase() ?? "";
|
||||
return JS_LIKE_LANGUAGES.has(normalizedLanguage) ? ts.ScriptKind.JS : ts.ScriptKind.TS;
|
||||
return JS_LIKE_LANGUAGES.has(normalizedLanguage)
|
||||
? ts.ScriptKind.JS
|
||||
: ts.ScriptKind.TS;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import type { EvalMode } from "./types";
|
||||
import {
|
||||
parsePositiveInteger,
|
||||
resolveWindmillBackendSettings,
|
||||
} from "./windmillBackendSettings";
|
||||
|
||||
export const BACKEND_VALIDATION_MODES = ["off", "preview"] as const;
|
||||
|
||||
@@ -16,10 +20,17 @@ export interface BackendValidationSettings {
|
||||
maxWaitMs: number;
|
||||
}
|
||||
|
||||
export function parseBackendValidationMode(value?: string | null): BackendValidationMode {
|
||||
export function parseBackendValidationMode(
|
||||
value?: string | null,
|
||||
): BackendValidationMode {
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
|
||||
if (!normalized || normalized === "off" || normalized === "false" || normalized === "0") {
|
||||
if (
|
||||
!normalized ||
|
||||
normalized === "off" ||
|
||||
normalized === "false" ||
|
||||
normalized === "0"
|
||||
) {
|
||||
return "off";
|
||||
}
|
||||
|
||||
@@ -28,7 +39,7 @@ export function parseBackendValidationMode(value?: string | null): BackendValida
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Unsupported backend validation mode: ${value}. Use one of: ${BACKEND_VALIDATION_MODES.join(", ")}`
|
||||
`Unsupported backend validation mode: ${value}. Use one of: ${BACKEND_VALIDATION_MODES.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -37,68 +48,29 @@ export function resolveBackendValidationSettings(input: {
|
||||
requestedMode?: string | null;
|
||||
}): BackendValidationSettings {
|
||||
const mode = parseBackendValidationMode(
|
||||
input.requestedMode ?? process.env.WMILL_AI_EVAL_BACKEND_VALIDATION
|
||||
input.requestedMode ?? process.env.WMILL_AI_EVAL_BACKEND_VALIDATION,
|
||||
);
|
||||
|
||||
if (mode !== "off" && input.evalMode !== "flow" && input.evalMode !== "script") {
|
||||
if (
|
||||
mode !== "off" &&
|
||||
input.evalMode !== "flow" &&
|
||||
input.evalMode !== "script"
|
||||
) {
|
||||
throw new Error(
|
||||
`Backend validation mode "${mode}" is only supported for flow and script evals`
|
||||
`Backend validation mode "${mode}" is only supported for flow and script evals`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
mode,
|
||||
baseUrl: normalizeBaseUrl(
|
||||
process.env.WMILL_AI_EVAL_BACKEND_URL ??
|
||||
process.env.WINDMILL_URL ??
|
||||
process.env.WINDMILL_BASE_URL ??
|
||||
process.env.REMOTE ??
|
||||
"http://127.0.0.1:8000"
|
||||
),
|
||||
email: process.env.WMILL_AI_EVAL_BACKEND_EMAIL ?? "admin@windmill.dev",
|
||||
password: process.env.WMILL_AI_EVAL_BACKEND_PASSWORD ?? "changeme",
|
||||
keepWorkspaces: isTruthy(process.env.WMILL_AI_EVAL_KEEP_WORKSPACES),
|
||||
workspaceOverride: sanitizeOptionalWorkspaceId(process.env.WMILL_AI_EVAL_BACKEND_WORKSPACE),
|
||||
workspacePrefix: sanitizeWorkspacePrefix(
|
||||
process.env.WMILL_AI_EVAL_WORKSPACE_PREFIX ?? "ai-evals"
|
||||
),
|
||||
...resolveWindmillBackendSettings(),
|
||||
pollIntervalMs: parsePositiveInteger(
|
||||
process.env.WMILL_AI_EVAL_BACKEND_POLL_INTERVAL_MS,
|
||||
2000
|
||||
2000,
|
||||
),
|
||||
maxWaitMs: parsePositiveInteger(
|
||||
process.env.WMILL_AI_EVAL_BACKEND_MAX_WAIT_MS,
|
||||
120000,
|
||||
),
|
||||
maxWaitMs: parsePositiveInteger(process.env.WMILL_AI_EVAL_BACKEND_MAX_WAIT_MS, 120000),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeBaseUrl(value: string): string {
|
||||
return value.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
function sanitizeWorkspacePrefix(value: string): string {
|
||||
const sanitized = value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9-]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "");
|
||||
return sanitized.length > 0 ? sanitized : "ai-evals";
|
||||
}
|
||||
|
||||
function sanitizeOptionalWorkspaceId(value: string | undefined): string | undefined {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function isTruthy(value: string | undefined): boolean {
|
||||
if (!value) {
|
||||
return false;
|
||||
}
|
||||
return ["1", "true", "yes", "on"].includes(value.trim().toLowerCase());
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value: string | undefined, fallback: number): number {
|
||||
if (!value) {
|
||||
return fallback;
|
||||
}
|
||||
const parsed = Number(value);
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import {
|
||||
parseFrontendEvalTransport,
|
||||
resolveFrontendEvalTransportSettings,
|
||||
} from "./frontendTransport";
|
||||
|
||||
const ORIGINAL_ENV = {
|
||||
WMILL_AI_EVAL_BACKEND_URL: process.env.WMILL_AI_EVAL_BACKEND_URL,
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
if (ORIGINAL_ENV.WMILL_AI_EVAL_BACKEND_URL === undefined) {
|
||||
delete process.env.WMILL_AI_EVAL_BACKEND_URL;
|
||||
} else {
|
||||
process.env.WMILL_AI_EVAL_BACKEND_URL =
|
||||
ORIGINAL_ENV.WMILL_AI_EVAL_BACKEND_URL;
|
||||
}
|
||||
});
|
||||
|
||||
describe("parseFrontendEvalTransport", () => {
|
||||
it("defaults to direct when unset", () => {
|
||||
expect(parseFrontendEvalTransport(undefined)).toBe("direct");
|
||||
});
|
||||
|
||||
it("accepts proxy explicitly", () => {
|
||||
expect(parseFrontendEvalTransport("proxy")).toBe("proxy");
|
||||
});
|
||||
|
||||
it("rejects unsupported values", () => {
|
||||
expect(() => parseFrontendEvalTransport("worker")).toThrow(
|
||||
"Unsupported frontend eval transport: worker",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveFrontendEvalTransportSettings", () => {
|
||||
it("includes backend settings for proxy transport", () => {
|
||||
process.env.WMILL_AI_EVAL_BACKEND_URL = "http://127.0.0.1:8000/";
|
||||
|
||||
expect(
|
||||
resolveFrontendEvalTransportSettings({
|
||||
evalMode: "app",
|
||||
requestedTransport: "proxy",
|
||||
}),
|
||||
).toMatchObject({
|
||||
transport: "proxy",
|
||||
backend: {
|
||||
baseUrl: "http://127.0.0.1:8000",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps direct transport for cli runs", () => {
|
||||
expect(
|
||||
resolveFrontendEvalTransportSettings({
|
||||
evalMode: "cli",
|
||||
requestedTransport: "direct",
|
||||
}),
|
||||
).toEqual({
|
||||
transport: "direct",
|
||||
backend: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { EvalMode } from "./types";
|
||||
import type { WindmillBackendSettings } from "./windmillBackendSettings";
|
||||
import { resolveWindmillBackendSettings } from "./windmillBackendSettings";
|
||||
|
||||
export const FRONTEND_EVAL_TRANSPORTS = ["direct", "proxy"] as const;
|
||||
|
||||
export type FrontendEvalTransport = (typeof FRONTEND_EVAL_TRANSPORTS)[number];
|
||||
|
||||
export interface FrontendEvalTransportSettings {
|
||||
transport: FrontendEvalTransport;
|
||||
backend?: WindmillBackendSettings;
|
||||
}
|
||||
|
||||
export function parseFrontendEvalTransport(
|
||||
value?: string | null,
|
||||
): FrontendEvalTransport {
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
|
||||
if (!normalized || normalized === "direct") {
|
||||
return "direct";
|
||||
}
|
||||
|
||||
if (normalized === "proxy") {
|
||||
return "proxy";
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Unsupported frontend eval transport: ${value}. Use one of: ${FRONTEND_EVAL_TRANSPORTS.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveFrontendEvalTransportSettings(input: {
|
||||
evalMode: EvalMode;
|
||||
requestedTransport?: string | null;
|
||||
}): FrontendEvalTransportSettings {
|
||||
const transport = parseFrontendEvalTransport(input.requestedTransport);
|
||||
|
||||
if (transport === "proxy" && input.evalMode === "cli") {
|
||||
throw new Error(
|
||||
'Frontend eval transport "proxy" is only supported for flow, script, and app evals',
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
transport,
|
||||
backend:
|
||||
transport === "proxy" ? resolveWindmillBackendSettings() : undefined,
|
||||
};
|
||||
}
|
||||
+89
-43
@@ -12,26 +12,34 @@ import type {
|
||||
|
||||
export async function writeRunResult(
|
||||
result: BenchmarkRunResult,
|
||||
outputPath?: string
|
||||
outputPath?: string,
|
||||
): Promise<string> {
|
||||
const targetPath = resolveRunOutputPath(result.mode, outputPath);
|
||||
await mkdir(path.dirname(targetPath), { recursive: true });
|
||||
await writeFile(targetPath, JSON.stringify(toSerializableRunResult(result), null, 2) + "\n", "utf8");
|
||||
await writeFile(
|
||||
targetPath,
|
||||
JSON.stringify(toSerializableRunResult(result), null, 2) + "\n",
|
||||
"utf8",
|
||||
);
|
||||
return targetPath;
|
||||
}
|
||||
|
||||
export async function appendHistoryRecord(
|
||||
result: BenchmarkRunResult,
|
||||
historyPath = resolveHistoryPath(result.mode)
|
||||
historyPath = resolveHistoryPath(result.mode),
|
||||
): Promise<string> {
|
||||
await mkdir(path.dirname(historyPath), { recursive: true });
|
||||
await appendFile(historyPath, JSON.stringify(toHistoryRecord(result)) + "\n", "utf8");
|
||||
await appendFile(
|
||||
historyPath,
|
||||
JSON.stringify(toHistoryRecord(result)) + "\n",
|
||||
"utf8",
|
||||
);
|
||||
return historyPath;
|
||||
}
|
||||
|
||||
export async function writeRunArtifacts(
|
||||
result: BenchmarkRunResult,
|
||||
outputPath?: string
|
||||
outputPath?: string,
|
||||
): Promise<string | null> {
|
||||
const targetPath = resolveRunOutputPath(result.mode, outputPath);
|
||||
const artifactRoot = defaultArtifactsRoot(targetPath);
|
||||
@@ -47,7 +55,11 @@ export async function writeRunArtifacts(
|
||||
continue;
|
||||
}
|
||||
|
||||
const attemptDir = path.join(artifactRoot, caseResult.id, `attempt-${attempt.attempt}`);
|
||||
const attemptDir = path.join(
|
||||
artifactRoot,
|
||||
caseResult.id,
|
||||
`attempt-${attempt.attempt}`,
|
||||
);
|
||||
await writeArtifactFiles(attemptDir, artifactFiles);
|
||||
attempt.artifactsPath = attemptDir;
|
||||
wroteArtifacts = true;
|
||||
@@ -62,17 +74,24 @@ export function buildRunResult(input: {
|
||||
mode: EvalMode;
|
||||
runs: number;
|
||||
runModel: string | null;
|
||||
transport?: BenchmarkRunResult["transport"];
|
||||
judgeModel: string | null;
|
||||
caseResults: BenchmarkCaseResult[];
|
||||
}): BenchmarkRunResult {
|
||||
const attemptCount = input.caseResults.reduce((sum, entry) => sum + entry.attempts.length, 0);
|
||||
const attemptCount = input.caseResults.reduce(
|
||||
(sum, entry) => sum + entry.attempts.length,
|
||||
0,
|
||||
);
|
||||
const passedAttempts = input.caseResults.reduce(
|
||||
(sum, entry) => sum + entry.attempts.filter((attempt) => attempt.passed).length,
|
||||
0
|
||||
(sum, entry) =>
|
||||
sum + entry.attempts.filter((attempt) => attempt.passed).length,
|
||||
0,
|
||||
);
|
||||
const durationTotal = input.caseResults.reduce(
|
||||
(sum, entry) => sum + entry.attempts.reduce((inner, attempt) => inner + attempt.durationMs, 0),
|
||||
0
|
||||
(sum, entry) =>
|
||||
sum +
|
||||
entry.attempts.reduce((inner, attempt) => inner + attempt.durationMs, 0),
|
||||
0,
|
||||
);
|
||||
const tokenUsageTotal = input.caseResults.reduce<BenchmarkTokenUsage | null>(
|
||||
(sum, entry) => {
|
||||
@@ -87,7 +106,7 @@ export function buildRunResult(input: {
|
||||
}
|
||||
return sum;
|
||||
},
|
||||
null
|
||||
null,
|
||||
);
|
||||
|
||||
return {
|
||||
@@ -97,6 +116,7 @@ export function buildRunResult(input: {
|
||||
gitSha: getGitSha(),
|
||||
runs: input.runs,
|
||||
runModel: input.runModel,
|
||||
transport: input.transport ?? null,
|
||||
judgeModel: input.judgeModel,
|
||||
caseCount: input.caseResults.length,
|
||||
attemptCount,
|
||||
@@ -122,6 +142,9 @@ export function formatRunSummary(result: BenchmarkRunResult): string {
|
||||
`Pass rate: ${formatPercent(result.passRate)} (${result.passedAttempts}/${result.attemptCount})`,
|
||||
`Average duration: ${Math.round(result.averageDurationMs)}ms`,
|
||||
];
|
||||
if (result.transport) {
|
||||
lines.splice(1, 0, `Transport: ${result.transport}`);
|
||||
}
|
||||
|
||||
const failures = collectFailures(result);
|
||||
if (failures.length > 0) {
|
||||
@@ -142,9 +165,11 @@ function collectFailures(result: BenchmarkRunResult): string[] {
|
||||
if (attempt.passed) {
|
||||
continue;
|
||||
}
|
||||
const failedChecks = attempt.checks.filter((check) => !check.passed).map((check) => check.name);
|
||||
const failedChecks = attempt.checks
|
||||
.filter((check) => !check.passed)
|
||||
.map((check) => check.name);
|
||||
failures.push(
|
||||
`${caseResult.id} attempt ${attempt.attempt}: ${failedChecks.join(", ") || attempt.error || "failed"}`
|
||||
`${caseResult.id} attempt ${attempt.attempt}: ${failedChecks.join(", ") || attempt.error || "failed"}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -156,8 +181,13 @@ function defaultFileName(mode: EvalMode): string {
|
||||
return `${new Date().toISOString().replaceAll(":", "-")}__${mode}.json`;
|
||||
}
|
||||
|
||||
export function resolveRunOutputPath(mode: EvalMode, outputPath?: string): string {
|
||||
return outputPath ?? path.join(getAiEvalsRoot(), "results", defaultFileName(mode));
|
||||
export function resolveRunOutputPath(
|
||||
mode: EvalMode,
|
||||
outputPath?: string,
|
||||
): string {
|
||||
return (
|
||||
outputPath ?? path.join(getAiEvalsRoot(), "results", defaultFileName(mode))
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveHistoryPath(mode: EvalMode): string {
|
||||
@@ -172,7 +202,7 @@ function defaultArtifactsRoot(resultPath: string): string {
|
||||
|
||||
async function writeArtifactFiles(
|
||||
rootDir: string,
|
||||
files: BenchmarkArtifactFile[]
|
||||
files: BenchmarkArtifactFile[],
|
||||
): Promise<void> {
|
||||
for (const file of files) {
|
||||
const relativePath = normalizeArtifactPath(file.path);
|
||||
@@ -185,18 +215,25 @@ async function writeArtifactFiles(
|
||||
function normalizeArtifactPath(filePath: string): string {
|
||||
const normalized = filePath.replaceAll("\\", "/").replace(/^\/+/, "");
|
||||
const parts = normalized.split("/").filter(Boolean);
|
||||
if (parts.length === 0 || parts.some((part) => part === "." || part === "..")) {
|
||||
if (
|
||||
parts.length === 0 ||
|
||||
parts.some((part) => part === "." || part === "..")
|
||||
) {
|
||||
throw new Error(`Invalid artifact path: ${filePath}`);
|
||||
}
|
||||
return parts.join("/");
|
||||
}
|
||||
|
||||
function toSerializableRunResult(result: BenchmarkRunResult): BenchmarkRunResult {
|
||||
function toSerializableRunResult(
|
||||
result: BenchmarkRunResult,
|
||||
): BenchmarkRunResult {
|
||||
return {
|
||||
...result,
|
||||
cases: result.cases.map((caseResult) => ({
|
||||
...caseResult,
|
||||
attempts: caseResult.attempts.map(({ artifactFiles, ...attempt }) => attempt),
|
||||
attempts: caseResult.attempts.map(
|
||||
({ artifactFiles, ...attempt }) => attempt,
|
||||
),
|
||||
})),
|
||||
};
|
||||
}
|
||||
@@ -204,8 +241,8 @@ function toSerializableRunResult(result: BenchmarkRunResult): BenchmarkRunResult
|
||||
function toHistoryRecord(result: BenchmarkRunResult) {
|
||||
const judgeScores = result.cases.flatMap((caseResult) =>
|
||||
caseResult.attempts.flatMap((attempt) =>
|
||||
typeof attempt.judgeScore === "number" ? [attempt.judgeScore] : []
|
||||
)
|
||||
typeof attempt.judgeScore === "number" ? [attempt.judgeScore] : [],
|
||||
),
|
||||
);
|
||||
|
||||
return {
|
||||
@@ -214,6 +251,7 @@ function toHistoryRecord(result: BenchmarkRunResult) {
|
||||
mode: result.mode,
|
||||
runs: result.runs,
|
||||
runModel: result.runModel,
|
||||
transport: result.transport,
|
||||
judgeModel: result.judgeModel,
|
||||
caseCount: result.caseCount,
|
||||
attemptCount: result.attemptCount,
|
||||
@@ -223,49 +261,57 @@ function toHistoryRecord(result: BenchmarkRunResult) {
|
||||
averageJudgeScore:
|
||||
judgeScores.length === 0
|
||||
? null
|
||||
: judgeScores.reduce((sum, score) => sum + score, 0) / judgeScores.length,
|
||||
: judgeScores.reduce((sum, score) => sum + score, 0) /
|
||||
judgeScores.length,
|
||||
averageTokenUsagePerAttempt: result.averageTokenUsagePerAttempt ?? null,
|
||||
failedCaseIds: Array.from(
|
||||
new Set(
|
||||
result.cases
|
||||
.filter((caseResult) => caseResult.attempts.some((attempt) => !attempt.passed))
|
||||
.map((caseResult) => caseResult.id)
|
||||
)
|
||||
.filter((caseResult) =>
|
||||
caseResult.attempts.some((attempt) => !attempt.passed),
|
||||
)
|
||||
.map((caseResult) => caseResult.id),
|
||||
),
|
||||
),
|
||||
cases: result.cases.map((caseResult) => {
|
||||
const attemptCount = caseResult.attempts.length;
|
||||
const passedAttempts = caseResult.attempts.filter((attempt) => attempt.passed).length;
|
||||
const passedAttempts = caseResult.attempts.filter(
|
||||
(attempt) => attempt.passed,
|
||||
).length;
|
||||
const totalDurationMs = caseResult.attempts.reduce(
|
||||
(sum, attempt) => sum + attempt.durationMs,
|
||||
0
|
||||
0,
|
||||
);
|
||||
const judgeScores = caseResult.attempts.flatMap((attempt) =>
|
||||
typeof attempt.judgeScore === "number" ? [attempt.judgeScore] : []
|
||||
typeof attempt.judgeScore === "number" ? [attempt.judgeScore] : [],
|
||||
);
|
||||
const totalTokenUsage = caseResult.attempts.reduce<BenchmarkTokenUsage | null>(
|
||||
(sum, attempt) => {
|
||||
if (!attempt.tokenUsage) {
|
||||
const totalTokenUsage =
|
||||
caseResult.attempts.reduce<BenchmarkTokenUsage | null>(
|
||||
(sum, attempt) => {
|
||||
if (!attempt.tokenUsage) {
|
||||
return sum;
|
||||
}
|
||||
sum ??= { prompt: 0, completion: 0, total: 0 };
|
||||
sum.prompt += attempt.tokenUsage.prompt;
|
||||
sum.completion += attempt.tokenUsage.completion;
|
||||
sum.total += attempt.tokenUsage.total;
|
||||
return sum;
|
||||
}
|
||||
sum ??= { prompt: 0, completion: 0, total: 0 };
|
||||
sum.prompt += attempt.tokenUsage.prompt;
|
||||
sum.completion += attempt.tokenUsage.completion;
|
||||
sum.total += attempt.tokenUsage.total;
|
||||
return sum;
|
||||
},
|
||||
null
|
||||
);
|
||||
},
|
||||
null,
|
||||
);
|
||||
|
||||
return {
|
||||
id: caseResult.id,
|
||||
attemptCount,
|
||||
passedAttempts,
|
||||
passRate: attemptCount === 0 ? 0 : passedAttempts / attemptCount,
|
||||
averageDurationMs: attemptCount === 0 ? 0 : totalDurationMs / attemptCount,
|
||||
averageDurationMs:
|
||||
attemptCount === 0 ? 0 : totalDurationMs / attemptCount,
|
||||
averageJudgeScore:
|
||||
judgeScores.length === 0
|
||||
? null
|
||||
: judgeScores.reduce((sum, score) => sum + score, 0) / judgeScores.length,
|
||||
: judgeScores.reduce((sum, score) => sum + score, 0) /
|
||||
judgeScores.length,
|
||||
averageTokenUsagePerAttempt:
|
||||
attemptCount === 0 || !totalTokenUsage
|
||||
? null
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
export const EVAL_MODES = ["cli", "flow", "script", "app"] as const;
|
||||
|
||||
export type EvalMode = (typeof EVAL_MODES)[number];
|
||||
export type FrontendEvalTransport = "direct" | "proxy";
|
||||
|
||||
export interface EvalCaseRuntimeBackendPreview {
|
||||
args?: Record<string, unknown>;
|
||||
@@ -151,7 +152,7 @@ export interface ModeRunner<TInitial, TExpected, TActual> {
|
||||
run(
|
||||
prompt: string,
|
||||
initial: TInitial | undefined,
|
||||
context: ModeRunContext
|
||||
context: ModeRunContext,
|
||||
): Promise<ModeRunOutput<TActual>>;
|
||||
validate(input: {
|
||||
evalCase: EvalCase;
|
||||
@@ -205,6 +206,7 @@ export interface BenchmarkRunResult {
|
||||
gitSha: string | null;
|
||||
runs: number;
|
||||
runModel: string | null;
|
||||
transport: FrontendEvalTransport | null;
|
||||
judgeModel: string | null;
|
||||
caseCount: number;
|
||||
attemptCount: number;
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
export interface WindmillBackendSettings {
|
||||
baseUrl: string;
|
||||
email: string;
|
||||
password: string;
|
||||
keepWorkspaces: boolean;
|
||||
workspaceOverride?: string;
|
||||
workspacePrefix: string;
|
||||
}
|
||||
|
||||
export function resolveWindmillBackendSettings(): WindmillBackendSettings {
|
||||
return {
|
||||
baseUrl: normalizeBaseUrl(
|
||||
process.env.WMILL_AI_EVAL_BACKEND_URL ??
|
||||
process.env.WINDMILL_URL ??
|
||||
process.env.WINDMILL_BASE_URL ??
|
||||
process.env.REMOTE ??
|
||||
"http://127.0.0.1:8000",
|
||||
),
|
||||
email: process.env.WMILL_AI_EVAL_BACKEND_EMAIL ?? "admin@windmill.dev",
|
||||
password: process.env.WMILL_AI_EVAL_BACKEND_PASSWORD ?? "changeme",
|
||||
keepWorkspaces: isTruthy(process.env.WMILL_AI_EVAL_KEEP_WORKSPACES),
|
||||
workspaceOverride: sanitizeOptionalWorkspaceId(
|
||||
process.env.WMILL_AI_EVAL_BACKEND_WORKSPACE,
|
||||
),
|
||||
workspacePrefix: sanitizeWorkspacePrefix(
|
||||
process.env.WMILL_AI_EVAL_WORKSPACE_PREFIX ?? "ai-evals",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function parsePositiveInteger(
|
||||
value: string | undefined,
|
||||
fallback: number,
|
||||
): number {
|
||||
if (!value) {
|
||||
return fallback;
|
||||
}
|
||||
const parsed = Number(value);
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function normalizeBaseUrl(value: string): string {
|
||||
return value.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
function sanitizeWorkspacePrefix(value: string): string {
|
||||
const sanitized = value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9-]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "");
|
||||
return sanitized.length > 0 ? sanitized : "ai-evals";
|
||||
}
|
||||
|
||||
function sanitizeOptionalWorkspaceId(
|
||||
value: string | undefined,
|
||||
): string | undefined {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function isTruthy(value: string | undefined): boolean {
|
||||
if (!value) {
|
||||
return false;
|
||||
}
|
||||
return ["1", "true", "yes", "on"].includes(value.trim().toLowerCase());
|
||||
}
|
||||
+24
-13
@@ -5,31 +5,42 @@ import type { FrontendEvalModelConfig } from "../core/models";
|
||||
import { validateAppState, type AppFilesState } from "../core/validators";
|
||||
import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
|
||||
import { runAppEval } from "../adapters/frontend/core/app/appEvalRunner";
|
||||
import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon";
|
||||
import {
|
||||
DEFAULT_FRONTEND_EVAL_MODEL,
|
||||
getFrontendApiKey,
|
||||
} from "./frontendCommon";
|
||||
import type { FrontendEvalTransportSettings } from "../core/frontendTransport";
|
||||
|
||||
export function createAppModeRunner(
|
||||
modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL
|
||||
modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL,
|
||||
transportSettings?: FrontendEvalTransportSettings,
|
||||
): ModeRunner<AppFilesState, AppFilesState, AppFilesState> {
|
||||
return {
|
||||
mode: "app",
|
||||
concurrency: 5,
|
||||
judgeThreshold: 80,
|
||||
async loadInitial(path) {
|
||||
return path ? (await loadAppFixture(path)) : undefined;
|
||||
return path ? await loadAppFixture(path) : undefined;
|
||||
},
|
||||
async loadExpected(path) {
|
||||
return path ? (await loadAppFixture(path)) : undefined;
|
||||
return path ? await loadAppFixture(path) : undefined;
|
||||
},
|
||||
async run(prompt, initial, context) {
|
||||
const result = await runAppEval(prompt, getFrontendApiKey(modelConfig.provider), {
|
||||
initialFrontend: initial?.frontend,
|
||||
initialBackend: initial?.backend,
|
||||
initialDatatables: initial?.datatables,
|
||||
maxIterations: context.evalCase?.runtime?.maxTurns,
|
||||
provider: modelConfig.provider,
|
||||
model: modelConfig.model,
|
||||
runContext: context,
|
||||
});
|
||||
const result = await runAppEval(
|
||||
prompt,
|
||||
getFrontendApiKey(modelConfig.provider),
|
||||
{
|
||||
initialFrontend: initial?.frontend,
|
||||
initialBackend: initial?.backend,
|
||||
initialDatatables: initial?.datatables,
|
||||
maxIterations: context.evalCase?.runtime?.maxTurns,
|
||||
provider: modelConfig.provider,
|
||||
model: modelConfig.model,
|
||||
transport: transportSettings?.transport,
|
||||
backend: transportSettings?.backend,
|
||||
runContext: context,
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
success: result.success,
|
||||
|
||||
+72
-50
@@ -7,7 +7,11 @@ import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
|
||||
import { runFlowEval } from "../adapters/frontend/core/flow/flowEvalRunner";
|
||||
import type { FlowWorkspaceFixtures } from "../adapters/frontend/core/flow/fileHelpers";
|
||||
import { BackendPreviewClient } from "../adapters/frontend/backendPreview";
|
||||
import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon";
|
||||
import {
|
||||
DEFAULT_FRONTEND_EVAL_MODEL,
|
||||
getFrontendApiKey,
|
||||
} from "./frontendCommon";
|
||||
import type { FrontendEvalTransportSettings } from "../core/frontendTransport";
|
||||
import {
|
||||
normalizeFlowInitialFixture,
|
||||
normalizeFlowStateFixture,
|
||||
@@ -16,7 +20,8 @@ import {
|
||||
|
||||
export function createFlowModeRunner(
|
||||
modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL,
|
||||
backendValidation?: BackendValidationSettings
|
||||
backendValidation?: BackendValidationSettings,
|
||||
transportSettings?: FrontendEvalTransportSettings,
|
||||
): ModeRunner<FlowInitialFixture, FlowState, FlowState> {
|
||||
return {
|
||||
mode: "flow",
|
||||
@@ -35,14 +40,20 @@ export function createFlowModeRunner(
|
||||
return normalizeFlowStateFixture(await readJsonFile<unknown>(path));
|
||||
},
|
||||
async run(prompt, initial, context) {
|
||||
const result = await runFlowEval(prompt, getFrontendApiKey(modelConfig.provider), {
|
||||
initialFlow: initial?.flowFixture,
|
||||
workspaceFixtures: initial?.workspace,
|
||||
maxIterations: context.evalCase?.runtime?.maxTurns,
|
||||
provider: modelConfig.provider,
|
||||
model: modelConfig.model,
|
||||
runContext: context,
|
||||
});
|
||||
const result = await runFlowEval(
|
||||
prompt,
|
||||
getFrontendApiKey(modelConfig.provider),
|
||||
{
|
||||
initialFlow: initial?.flowFixture,
|
||||
workspaceFixtures: initial?.workspace,
|
||||
maxIterations: context.evalCase?.runtime?.maxTurns,
|
||||
provider: modelConfig.provider,
|
||||
model: modelConfig.model,
|
||||
transport: transportSettings?.transport,
|
||||
backend: transportSettings?.backend,
|
||||
runContext: context,
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
success: result.success,
|
||||
@@ -64,7 +75,10 @@ export function createFlowModeRunner(
|
||||
});
|
||||
},
|
||||
async backendValidate({ evalCase, initial, actual, context }) {
|
||||
if (backendValidation?.mode !== "preview" || !evalCase.runtime?.backendPreview) {
|
||||
if (
|
||||
backendValidation?.mode !== "preview" ||
|
||||
!evalCase.runtime?.backendPreview
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -81,46 +95,54 @@ export function createFlowModeRunner(
|
||||
}
|
||||
|
||||
const previewClient = new BackendPreviewClient(backendValidation);
|
||||
return await previewClient.withWorkspace(evalCase.id, context.attempt, async (workspaceId) => {
|
||||
await seedWorkspaceFixtures(previewClient, workspaceId, initial?.workspace);
|
||||
return await previewClient.withWorkspace(
|
||||
evalCase.id,
|
||||
context.attempt,
|
||||
async (workspaceId) => {
|
||||
await seedWorkspaceFixtures(
|
||||
previewClient,
|
||||
workspaceId,
|
||||
initial?.workspace,
|
||||
);
|
||||
|
||||
const completedJob = await previewClient.runFlowPreview({
|
||||
workspaceId,
|
||||
value: actual.value as Record<string, unknown>,
|
||||
args: evalCase.runtime?.backendPreview?.args ?? {},
|
||||
timeoutSeconds: evalCase.runtime?.backendPreview?.timeoutSeconds,
|
||||
});
|
||||
const completedJob = await previewClient.runFlowPreview({
|
||||
workspaceId,
|
||||
value: actual.value as Record<string, unknown>,
|
||||
args: evalCase.runtime?.backendPreview?.args ?? {},
|
||||
timeoutSeconds: evalCase.runtime?.backendPreview?.timeoutSeconds,
|
||||
});
|
||||
|
||||
return {
|
||||
checks: [
|
||||
{
|
||||
name: "backend flow preview succeeded",
|
||||
passed: completedJob.success,
|
||||
details: completedJob.success
|
||||
? `workspace=${workspaceId}`
|
||||
: `workspace=${workspaceId}; job=${completedJob.id}`,
|
||||
},
|
||||
],
|
||||
artifactFiles: [
|
||||
{
|
||||
path: "backend-preview.json",
|
||||
content:
|
||||
JSON.stringify(
|
||||
{
|
||||
workspaceId,
|
||||
jobId: completedJob.id,
|
||||
success: completedJob.success,
|
||||
result: completedJob.result,
|
||||
logs: completedJob.logs,
|
||||
completedJob: completedJob.raw,
|
||||
},
|
||||
null,
|
||||
2
|
||||
) + "\n",
|
||||
},
|
||||
],
|
||||
};
|
||||
});
|
||||
return {
|
||||
checks: [
|
||||
{
|
||||
name: "backend flow preview succeeded",
|
||||
passed: completedJob.success,
|
||||
details: completedJob.success
|
||||
? `workspace=${workspaceId}`
|
||||
: `workspace=${workspaceId}; job=${completedJob.id}`,
|
||||
},
|
||||
],
|
||||
artifactFiles: [
|
||||
{
|
||||
path: "backend-preview.json",
|
||||
content:
|
||||
JSON.stringify(
|
||||
{
|
||||
workspaceId,
|
||||
jobId: completedJob.id,
|
||||
success: completedJob.success,
|
||||
result: completedJob.result,
|
||||
logs: completedJob.logs,
|
||||
completedJob: completedJob.raw,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
) + "\n",
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
);
|
||||
},
|
||||
buildArtifacts(actual): BenchmarkArtifactFile[] {
|
||||
return [
|
||||
@@ -136,7 +158,7 @@ export function createFlowModeRunner(
|
||||
async function seedWorkspaceFixtures(
|
||||
previewClient: BackendPreviewClient,
|
||||
workspaceId: string,
|
||||
fixtures?: FlowWorkspaceFixtures
|
||||
fixtures?: FlowWorkspaceFixtures,
|
||||
): Promise<void> {
|
||||
for (const script of fixtures?.scripts ?? []) {
|
||||
await previewClient.createScript({
|
||||
|
||||
+69
-52
@@ -6,11 +6,16 @@ import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
|
||||
import { BackendPreviewClient } from "../adapters/frontend/backendPreview";
|
||||
import { runScriptEval } from "../adapters/frontend/core/script/scriptEvalRunner";
|
||||
import type { ScriptEvalState } from "../adapters/frontend/core/script/fileHelpers";
|
||||
import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon";
|
||||
import {
|
||||
DEFAULT_FRONTEND_EVAL_MODEL,
|
||||
getFrontendApiKey,
|
||||
} from "./frontendCommon";
|
||||
import type { FrontendEvalTransportSettings } from "../core/frontendTransport";
|
||||
|
||||
export function createScriptModeRunner(
|
||||
modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL,
|
||||
backendValidation?: BackendValidationSettings
|
||||
backendValidation?: BackendValidationSettings,
|
||||
transportSettings?: FrontendEvalTransportSettings,
|
||||
): ModeRunner<ScriptEvalState, ScriptEvalState, ScriptEvalState> {
|
||||
return {
|
||||
mode: "script",
|
||||
@@ -27,13 +32,19 @@ export function createScriptModeRunner(
|
||||
throw new Error("Script evals require an initial script fixture");
|
||||
}
|
||||
|
||||
const result = await runScriptEval(prompt, getFrontendApiKey(modelConfig.provider), {
|
||||
initialScript: initial,
|
||||
maxIterations: context.evalCase?.runtime?.maxTurns,
|
||||
provider: modelConfig.provider,
|
||||
model: modelConfig.model,
|
||||
runContext: context,
|
||||
});
|
||||
const result = await runScriptEval(
|
||||
prompt,
|
||||
getFrontendApiKey(modelConfig.provider),
|
||||
{
|
||||
initialScript: initial,
|
||||
maxIterations: context.evalCase?.runtime?.maxTurns,
|
||||
provider: modelConfig.provider,
|
||||
model: modelConfig.model,
|
||||
transport: transportSettings?.transport,
|
||||
backend: transportSettings?.backend,
|
||||
runContext: context,
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
success: result.success,
|
||||
@@ -55,50 +66,56 @@ export function createScriptModeRunner(
|
||||
}
|
||||
|
||||
const previewClient = new BackendPreviewClient(backendValidation);
|
||||
return await previewClient.withWorkspace(evalCase.id, context.attempt, async (workspaceId) => {
|
||||
const completedJob = await previewClient.runScriptPreview({
|
||||
workspaceId,
|
||||
content: actual.code,
|
||||
args:
|
||||
(evalCase.runtime?.backendPreview?.args as Record<string, unknown> | undefined) ??
|
||||
actual.args ??
|
||||
initial?.args ??
|
||||
{},
|
||||
language: normalizePreviewLanguage(actual.lang),
|
||||
path: toPreviewScriptPath(actual.path),
|
||||
timeoutSeconds: evalCase.runtime?.backendPreview?.timeoutSeconds,
|
||||
});
|
||||
return await previewClient.withWorkspace(
|
||||
evalCase.id,
|
||||
context.attempt,
|
||||
async (workspaceId) => {
|
||||
const completedJob = await previewClient.runScriptPreview({
|
||||
workspaceId,
|
||||
content: actual.code,
|
||||
args:
|
||||
(evalCase.runtime?.backendPreview?.args as
|
||||
| Record<string, unknown>
|
||||
| undefined) ??
|
||||
actual.args ??
|
||||
initial?.args ??
|
||||
{},
|
||||
language: normalizePreviewLanguage(actual.lang),
|
||||
path: toPreviewScriptPath(actual.path),
|
||||
timeoutSeconds: evalCase.runtime?.backendPreview?.timeoutSeconds,
|
||||
});
|
||||
|
||||
return {
|
||||
checks: [
|
||||
{
|
||||
name: "backend script preview succeeded",
|
||||
passed: completedJob.success,
|
||||
details: completedJob.success
|
||||
? `workspace=${workspaceId}`
|
||||
: `workspace=${workspaceId}; job=${completedJob.id}`,
|
||||
},
|
||||
],
|
||||
artifactFiles: [
|
||||
{
|
||||
path: "backend-preview.json",
|
||||
content:
|
||||
JSON.stringify(
|
||||
{
|
||||
workspaceId,
|
||||
jobId: completedJob.id,
|
||||
success: completedJob.success,
|
||||
result: completedJob.result,
|
||||
logs: completedJob.logs,
|
||||
completedJob: completedJob.raw,
|
||||
},
|
||||
null,
|
||||
2
|
||||
) + "\n",
|
||||
},
|
||||
],
|
||||
};
|
||||
});
|
||||
return {
|
||||
checks: [
|
||||
{
|
||||
name: "backend script preview succeeded",
|
||||
passed: completedJob.success,
|
||||
details: completedJob.success
|
||||
? `workspace=${workspaceId}`
|
||||
: `workspace=${workspaceId}; job=${completedJob.id}`,
|
||||
},
|
||||
],
|
||||
artifactFiles: [
|
||||
{
|
||||
path: "backend-preview.json",
|
||||
content:
|
||||
JSON.stringify(
|
||||
{
|
||||
workspaceId,
|
||||
jobId: completedJob.id,
|
||||
success: completedJob.success,
|
||||
result: completedJob.result,
|
||||
logs: completedJob.logs,
|
||||
completedJob: completedJob.raw,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
) + "\n",
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
);
|
||||
},
|
||||
buildArtifacts(actual): BenchmarkArtifactFile[] {
|
||||
return [
|
||||
|
||||
@@ -643,6 +643,7 @@ pub fn gemini_event_to_openai_sse_chunks(
|
||||
pub fn sanitize_schema_for_google(value: &mut serde_json::Value) {
|
||||
const UNSUPPORTED: &[&str] = &[
|
||||
"additionalProperties",
|
||||
"propertyNames",
|
||||
"strict",
|
||||
"$schema",
|
||||
"default",
|
||||
@@ -747,8 +748,8 @@ fn openai_tool_call_json(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
gemini_event_to_openai_sse_chunks, gemini_response_to_openai, GeminiParsedEvent,
|
||||
GeminiToolCallEvent,
|
||||
gemini_event_to_openai_sse_chunks, gemini_response_to_openai, sanitize_schema_for_google,
|
||||
GeminiParsedEvent, GeminiToolCallEvent,
|
||||
};
|
||||
|
||||
#[test]
|
||||
@@ -814,4 +815,23 @@ mod tests {
|
||||
);
|
||||
assert_eq!(tool_call["index"], 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_schema_for_google_removes_property_names() {
|
||||
let mut schema = serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"staticInputs": {
|
||||
"type": "object",
|
||||
"propertyNames": { "type": "string" },
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
sanitize_schema_for_google(&mut schema);
|
||||
|
||||
assert!(schema["properties"]["staticInputs"]["propertyNames"].is_null());
|
||||
assert!(schema["properties"]["staticInputs"]["additionalProperties"].is_null());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user