From 2cbc854577d2b0b9b50b75467557d73e95ddeb95 Mon Sep 17 00:00:00 2001 From: centdix Date: Mon, 15 Jun 2026 11:56:50 +0200 Subject: [PATCH] test(ai-evals): add ask benchmark mode comparing inkeep vs llms.txt docs tools Co-Authored-By: Claude Fable 5 --- ai_evals/adapters/frontend/benchmarkRunner.ts | 20 +- .../frontend/core/ask/askEvalRunner.ts | 175 ++++++++ ai_evals/adapters/frontend/progress.ts | 2 +- ai_evals/adapters/frontend/runtime.ts | 5 +- .../adapters/frontend/vitestAdapter.test.ts | 3 +- ai_evals/cases/ask.yaml | 419 ++++++++++++++++++ ai_evals/cli/index.ts | 35 +- ai_evals/core/cases.test.ts | 29 ++ ai_evals/core/models.test.ts | 7 + ai_evals/core/models.ts | 2 +- ai_evals/core/runSuite.ts | 4 +- ai_evals/core/types.ts | 26 +- ai_evals/core/validators.test.ts | 74 ++++ ai_evals/core/validators.ts | 53 +++ ai_evals/modes/ask.ts | 90 ++++ 15 files changed, 929 insertions(+), 15 deletions(-) create mode 100644 ai_evals/adapters/frontend/core/ask/askEvalRunner.ts create mode 100644 ai_evals/cases/ask.yaml create mode 100644 ai_evals/modes/ask.ts diff --git a/ai_evals/adapters/frontend/benchmarkRunner.ts b/ai_evals/adapters/frontend/benchmarkRunner.ts index 1729df7170..ac7cad5dee 100644 --- a/ai_evals/adapters/frontend/benchmarkRunner.ts +++ b/ai_evals/adapters/frontend/benchmarkRunner.ts @@ -12,7 +12,7 @@ import { resolveWindmillBackendSettings } from "../../core/windmillBackendSettin import { emitFrontendBenchmarkProgress } from "./progress"; import { DEFAULT_JUDGE_MODEL } from "../../core/judge"; -export type FrontendBenchmarkMode = "flow" | "app" | "script" | "global"; +export type FrontendBenchmarkMode = "flow" | "app" | "script" | "global" | "ask"; export async function runFrontendBenchmarkFromEnv(): Promise { const mode = parseMode(process.env.WMILL_FRONTEND_AI_EVAL_MODE); @@ -42,7 +42,11 @@ export async function runFrontendBenchmarkFromEnv(): Promise backendValidation, backendSettings, ); - const runModel = formatRunModelLabel(mode, model); + const docsTool = + mode === "ask" ? process.env.WMILL_AI_EVAL_DOCS_TOOL?.trim() || "llmstxt" : undefined; + const runModel = docsTool + ? `${formatRunModelLabel(mode, model)} ask:${docsTool}` + : formatRunModelLabel(mode, model); const caseResults = await runSuite({ modeRunner, cases: selectedCases, @@ -92,11 +96,21 @@ async function getModeRunner( const { createGlobalModeRunner } = await import("../../modes/global"); return createGlobalModeRunner(model, backendSettings); } + case "ask": { + const { createAskModeRunner } = await import("../../modes/ask"); + return createAskModeRunner(model, backendSettings); + } } } function parseMode(value: string | undefined): FrontendBenchmarkMode { - if (value === "flow" || value === "app" || value === "script" || value === "global") { + if ( + value === "flow" || + value === "app" || + value === "script" || + value === "global" || + value === "ask" + ) { return value; } throw new Error(`Unsupported frontend benchmark mode: ${String(value)}`); diff --git a/ai_evals/adapters/frontend/core/ask/askEvalRunner.ts b/ai_evals/adapters/frontend/core/ask/askEvalRunner.ts new file mode 100644 index 0000000000..82384def85 --- /dev/null +++ b/ai_evals/adapters/frontend/core/ask/askEvalRunner.ts @@ -0,0 +1,175 @@ +import type { ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs"; +import type { AIProvider } from "$lib/gen/types.gen"; +import { + getAskTools, + prepareAskSystemMessage, + prepareAskUserMessage, + type DocsToolVariant, +} from "../../../../../frontend/src/lib/components/copilot/chat/ask/core"; +import type { ModeRunContext } from "../../../../core/types"; +import type { AskAnswerState } from "../../../../core/validators"; +import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; +import { WindmillBackendClient } from "../../windmillBackend"; +import { runEval } from "../shared"; +import type { TokenUsage, ToolCallDetail } from "../shared/types"; + +export interface AskEvalResult { + success: boolean; + state: AskAnswerState; + error?: string; + assistantMessageCount: number; + toolCallCount: number; + toolsUsed: string[]; + toolCallDetails: ToolCallDetail[]; + tokenUsage: TokenUsage; +} + +export interface AskEvalOptions { + variant: DocsToolVariant; + model?: string; + maxIterations?: number; + provider?: AIProvider; + backend: WindmillBackendSettings; + runContext?: ModeRunContext; +} + +const DOCS_TOOL_ENV = "WMILL_AI_EVAL_DOCS_TOOL"; + +/** + * Resolves which docs-tool arm to benchmark. Defaults to the new llms.txt arm. + */ +export function resolveDocsToolVariant( + value: string | undefined = process.env[DOCS_TOOL_ENV], +): DocsToolVariant { + return value === "inkeep" ? "inkeep" : "llmstxt"; +} + +export async function runAskEval( + userPrompt: string, + apiKey: string, + options: AskEvalOptions, +): Promise { + // The production inkeep tool calls fetch('/api/inkeep') with a relative URL, + // which has no origin under node. Install a process-wide shim that rewrites + // /api/* calls to the eval backend with an auth token. Only needed for the + // inkeep arm; the llms.txt arm fetches absolute windmill.dev URLs natively. + if (options.variant === "inkeep") { + await installInkeepFetchShim(options.backend); + } + + const model = options.model ?? "claude-haiku-4-5-20251001"; + + const rawResult = await runEval({ + userPrompt, + systemMessage: prepareAskSystemMessage(undefined, options.variant), + userMessage: prepareAskUserMessage(userPrompt), + tools: getAskTools(options.variant), + helpers: {}, + apiKey, + getOutput: () => ({ + answer: "", + docsTool: options.variant, + toolsUsed: [], + toolCallCount: 0, + }), + onAssistantMessageStart: options.runContext?.onAssistantMessageStart, + onAssistantToken: options.runContext?.onAssistantChunk, + onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd, + onToolCall: options.runContext?.onToolCall, + options: { + maxIterations: options.maxIterations, + model, + provider: options.provider, + backend: options.backend, + caseId: options.runContext?.caseId, + attempt: options.runContext?.attempt, + }, + }); + + const answer = extractFinalAnswer(rawResult.messages); + + return { + success: rawResult.success, + state: { + answer, + docsTool: options.variant, + toolsUsed: rawResult.toolsCalled, + toolCallCount: rawResult.toolCallsCount, + }, + error: rawResult.error, + assistantMessageCount: rawResult.iterations, + toolCallCount: rawResult.toolCallsCount, + toolsUsed: rawResult.toolsCalled, + toolCallDetails: rawResult.toolCallDetails, + tokenUsage: rawResult.tokenUsage, + }; +} + +/** + * The final answer is the last assistant message with non-empty string content. + */ +export function extractFinalAnswer( + messages: ChatCompletionMessageParam[], +): string { + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i]; + if (message.role !== "assistant") { + continue; + } + const content = message.content; + if (typeof content === "string" && content.trim().length > 0) { + return content; + } + if (Array.isArray(content)) { + const text = content + .map((part) => + part && typeof part === "object" && "text" in part + ? String((part as { text?: unknown }).text ?? "") + : "", + ) + .join("") + .trim(); + if (text.length > 0) { + return text; + } + } + } + return ""; +} + +let inkeepFetchShimInstalled = false; + +/** + * Installs (once) a process-wide fetch wrapper that rewrites relative /api/* + * requests to the eval backend, adding a bearer token. All other URLs are + * passed through untouched. + */ +async function installInkeepFetchShim( + backend: WindmillBackendSettings, +): Promise { + if (inkeepFetchShimInstalled) { + return; + } + inkeepFetchShimInstalled = true; + + const client = new WindmillBackendClient(backend); + const originalFetch = globalThis.fetch.bind(globalThis); + + globalThis.fetch = (async ( + input: RequestInfo | URL, + init?: RequestInit, + ): Promise => { + if (typeof input === "string" && input.startsWith("/api/")) { + const token = await client.getToken(); + const url = `${backend.baseUrl}${input}`; + return await originalFetch(url, { + ...init, + headers: { + ...(init?.headers ?? {}), + Authorization: `Bearer ${token}`, + }, + }); + } + return await originalFetch(input as any, init); + }) as typeof fetch; +} diff --git a/ai_evals/adapters/frontend/progress.ts b/ai_evals/adapters/frontend/progress.ts index 3a4810c4a3..0e15d5588f 100644 --- a/ai_evals/adapters/frontend/progress.ts +++ b/ai_evals/adapters/frontend/progress.ts @@ -1,4 +1,4 @@ -export type FrontendBenchmarkProgressSurface = 'flow' | 'app' | 'script' | 'global' +export type FrontendBenchmarkProgressSurface = 'flow' | 'app' | 'script' | 'global' | 'ask' export type FrontendBenchmarkProgressEvent = | { diff --git a/ai_evals/adapters/frontend/runtime.ts b/ai_evals/adapters/frontend/runtime.ts index 347e15191c..ac6ba04122 100644 --- a/ai_evals/adapters/frontend/runtime.ts +++ b/ai_evals/adapters/frontend/runtime.ts @@ -16,7 +16,7 @@ const FRONTEND_BENCHMARK_TEST = const FRONTEND_BENCHMARK_CONFIG = "../ai_evals/adapters/frontend/vitest.config.ts"; -export type FrontendMode = "flow" | "app" | "script" | "global"; +export type FrontendMode = "flow" | "app" | "script" | "global" | "ask"; export async function runFrontendBenchmarkAdapter(input: { mode: FrontendMode; @@ -25,6 +25,7 @@ export async function runFrontendBenchmarkAdapter(input: { model?: string; verbose?: boolean; backendValidation?: string; + docsTool?: string; }): Promise { const tempDir = await mkdtemp( path.join(tmpdir(), "wmill-frontend-benchmark-"), @@ -41,6 +42,8 @@ export async function runFrontendBenchmarkAdapter(input: { WMILL_FRONTEND_AI_EVAL_PROGRESS: "1", WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? "1" : "0", WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION: input.backendValidation ?? "", + // Only meaningful for ask mode; selects the docs-tool arm to benchmark. + WMILL_AI_EVAL_DOCS_TOOL: input.docsTool ?? process.env.WMILL_AI_EVAL_DOCS_TOOL ?? "", }; try { diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index ebbbac8d11..2739eedce2 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -434,5 +434,6 @@ benchmarkIt( resetBenchmarkMockBackend() } }, - 600_000 + // Full-suite runs (30+ cases at concurrency 2-3) routinely exceed 10 minutes. + 7_200_000 ) diff --git a/ai_evals/cases/ask.yaml b/ai_evals/cases/ask.yaml new file mode 100644 index 0000000000..0a26f68e6b --- /dev/null +++ b/ai_evals/cases/ask.yaml @@ -0,0 +1,419 @@ +# AI-chat documentation Q&A cases. Each case asks a realistic user question and +# is graded on answer quality (judge) plus a deterministic URL-citation check. +# Ground-truth URLs are derived from https://www.windmill.dev/llms.txt — all +# `answerIncludesAny` URLs are real docs pages (without the `.md` suffix). + +# --- Tier 1: direct lookups ------------------------------------------------- + +- id: ask-lookup-cron-schedule + prompt: How do I run a script automatically every Monday at 9am? + runtime: + maxTurns: 10 + judgeChecklist: + - explains that recurring runs use a schedule with a CRON expression + - mentions a concrete cron-like expression or the schedule configuration + - the answer is grounded in the Windmill scheduling documentation + validate: + answerIncludesAny: + - ["windmill.dev/docs/core_concepts/scheduling"] + +- id: ask-lookup-secret-in-python + prompt: How do I store a secret like an API key and use it inside a Python script? + runtime: + maxTurns: 10 + judgeChecklist: + - explains storing the value as a secret variable + - explains accessing it from a Python script (e.g. wmill.get_variable or get_resource) + - the answer is grounded in the variables and secrets documentation + validate: + answerIncludesAny: + - ["windmill.dev/docs/core_concepts/variables_and_secrets"] + +- id: ask-lookup-webhook-trigger + prompt: How can I trigger one of my scripts from an external service by calling a URL? + runtime: + maxTurns: 10 + judgeChecklist: + - explains that scripts expose webhook URLs + - mentions sync vs async webhook behavior or how to find the webhook URL + - the answer is grounded in the webhooks documentation + validate: + answerIncludesAny: + - ["windmill.dev/docs/core_concepts/webhooks"] + +- id: ask-lookup-retry-step + prompt: One step in my flow calls a flaky API. How do I make it automatically retry if it fails? + runtime: + maxTurns: 10 + judgeChecklist: + - explains configuring retries on a flow step + - mentions the number of attempts and/or delay between attempts + - the answer is grounded in the error handling or flow editor documentation + validate: + answerIncludesAny: + - [ + "windmill.dev/docs/flows/retries", + "windmill.dev/docs/core_concepts/error_handling", + "windmill.dev/docs/flows/flow_editor", + ] + +- id: ask-lookup-connect-database + prompt: How do I connect to my Postgres database so my scripts can query it? + runtime: + maxTurns: 10 + judgeChecklist: + - explains creating a resource holding the database connection details + - explains referencing that resource from a script + - the answer is grounded in the resources documentation + validate: + answerIncludesAny: + - [ + "windmill.dev/docs/core_concepts/resources_and_types", + "windmill.dev/docs/getting_started/scripts_quickstart/sql", + ] + +- id: ask-lookup-python-deps + prompt: How do I add a third-party pip package to a Python script in Windmill? + runtime: + maxTurns: 10 + judgeChecklist: + - explains that imports are auto-detected and dependencies resolved automatically + - mentions how to pin a version or the requirements mechanism + - the answer is grounded in the Python scripting documentation + validate: + answerIncludesAny: + - [ + "windmill.dev/docs/getting_started/scripts_quickstart/python", + "windmill.dev/docs/advanced/dependencies_in_python", + ] + +- id: ask-lookup-concurrency-limit + prompt: How do I stop a script from running too many times at once so I don't hit an API rate limit? + runtime: + maxTurns: 10 + judgeChecklist: + - explains setting a concurrency limit on the script + - mentions the max number of concurrent executions and/or time window + - the answer is grounded in the concurrency limits documentation + validate: + answerIncludesAny: + - [ + "windmill.dev/docs/core_concepts/concurrency_limits", + "windmill.dev/docs/script_editor/concurrency_limit", + ] + +- id: ask-lookup-app-table-button + prompt: How do I build a simple internal tool with a table and a button using Windmill? + runtime: + maxTurns: 10 + judgeChecklist: + - explains using the app editor with drag-and-drop components + - mentions wiring a component to a backend script/runnable + - the answer is grounded in the app editor or apps quickstart documentation + validate: + answerIncludesAny: + - [ + "windmill.dev/docs/apps/app_editor", + "windmill.dev/docs/getting_started/apps_quickstart", + ] + +# --- Tier 2: conceptual / vocabulary-mismatch ------------------------------- + +- id: ask-concept-flow-wait-for-approval + prompt: How can I make a flow pause and wait for a person to approve before it continues? + runtime: + maxTurns: 10 + judgeChecklist: + - identifies this as a suspend / approval step in a flow + - explains that the flow resumes once approved (e.g. via a resume link or form) + - the answer is grounded in the flow editor / approval documentation + validate: + answerIncludesAny: + - [ + "windmill.dev/docs/flows/flow_editor", + "windmill.dev/docs/flows/flow_approval", + ] + +- id: ask-concept-remember-value-between-runs + prompt: Can a script remember a value from its previous run, like the last timestamp it processed? + runtime: + maxTurns: 10 + judgeChecklist: + - identifies states as the mechanism (getState / setState) + - explains state persists between runs of the same script + - the answer is grounded in the states / within-windmill documentation + validate: + answerIncludesAny: + - ["windmill.dev/docs/core_concepts/persistent_storage/within_windmill"] + +- id: ask-concept-steps-at-same-time + prompt: How do I make several steps in my flow run at the same time instead of one after another? + runtime: + maxTurns: 10 + judgeChecklist: + - identifies running branches in parallel (branchall / parallel branches) + - explains the steps execute concurrently + - the answer is grounded in the flow editor documentation + validate: + answerIncludesAny: + - [ + "windmill.dev/docs/flows/flow_editor", + "windmill.dev/docs/flows/flow_branches", + ] + +- id: ask-concept-react-to-db-changes + prompt: I want something to happen automatically whenever a new row is inserted into my Postgres table. Is that possible? + runtime: + maxTurns: 10 + judgeChecklist: + - identifies Postgres triggers reacting to insert/update/delete + - mentions it listens to database change events (logical replication) + - the answer is grounded in the Postgres triggers documentation + validate: + answerIncludesAny: + - [ + "windmill.dev/docs/core_concepts/postgres_triggers", + "windmill.dev/docs/getting_started/triggers", + ] + +- id: ask-concept-reuse-config-everywhere + prompt: I keep pasting the same base URL and credentials into many scripts. Is there a cleaner way to manage that shared config? + runtime: + maxTurns: 10 + judgeChecklist: + - identifies variables (and/or resources) as the mechanism for reusable config + - explains that the config is defined once and referenced from scripts + - the answer is grounded in the variables / resources documentation + validate: + answerIncludesAny: + - [ + "windmill.dev/docs/core_concepts/variables_and_secrets", + "windmill.dev/docs/core_concepts/resources_and_types", + ] + +- id: ask-concept-let-llm-call-my-scripts + prompt: I want my AI assistant in Claude or Cursor to be able to run my Windmill scripts. How would I set that up? + runtime: + maxTurns: 10 + judgeChecklist: + - identifies MCP (Model Context Protocol) as the mechanism + - explains connecting an MCP client to Windmill to trigger scripts/flows + - the answer is grounded in the MCP documentation + validate: + answerIncludesAny: + - ["windmill.dev/docs/core_concepts/mcp"] + +- id: ask-concept-avoid-recompute + prompt: My script does an expensive computation with the same inputs a lot. Can Windmill avoid recomputing the same result every time? + runtime: + maxTurns: 10 + judgeChecklist: + - identifies caching of script/flow results + - explains that identical inputs reuse the cached result for a configured duration + - the answer is grounded in the caching documentation + validate: + answerIncludesAny: + - ["windmill.dev/docs/core_concepts/caching"] + +- id: ask-concept-agent-step + prompt: I want a flow step where an LLM decides which of my scripts to call based on the input. Does Windmill support that? + runtime: + maxTurns: 10 + judgeChecklist: + - identifies AI agent steps in a flow + - explains the agent can call tools/scripts based on the input + - the answer is grounded in the AI agents documentation + validate: + answerIncludesAny: + - ["windmill.dev/docs/core_concepts/ai_agents"] + +# --- Tier 3: multi-page synthesis ------------------------------------------- + +- id: ask-synthesis-variable-vs-resource + prompt: What's the difference between a variable and a resource in Windmill, and when should I use each? + runtime: + maxTurns: 10 + judgeChecklist: + - explains a variable holds a single reusable value (and secrets for sensitive ones) + - explains a resource holds a structured connection object (e.g. database/API credentials) + - gives reasonable guidance on when to use each + - the answer is grounded in the variables and resources documentation + validate: + answerIncludesAny: + - ["windmill.dev/docs/core_concepts/variables_and_secrets"] + - ["windmill.dev/docs/core_concepts/resources_and_types"] + +- id: ask-synthesis-trigger-options + prompt: What are all the different ways I can trigger a flow — both on a schedule and from external events? + runtime: + maxTurns: 10 + judgeChecklist: + - covers scheduled/cron triggering + - covers event-based triggering (e.g. webhooks, HTTP routes, or message queues) + - the answer is grounded in the triggers / scheduling documentation + validate: + answerIncludesAny: + - [ + "windmill.dev/docs/getting_started/triggers", + "windmill.dev/docs/core_concepts/scheduling", + ] + +- id: ask-synthesis-storage-options + prompt: Where should I store data in Windmill? I'm confused about all the storage options. + runtime: + maxTurns: 10 + judgeChecklist: + - distinguishes object storage (S3) from relational/structured storage and lightweight state/KV options + - gives reasonable guidance on which to use when + - the answer is grounded in the persistent storage documentation + validate: + answerIncludesAny: + - ["windmill.dev/docs/core_concepts/persistent_storage"] + +- id: ask-synthesis-deploy-to-prod + prompt: I'm building in a dev workspace and want a safe way to promote my scripts and flows to production. What's the recommended workflow? + runtime: + maxTurns: 10 + judgeChecklist: + - describes promoting from staging/dev to prod and/or git-based deployment + - mentions draft/deploy and/or git sync as part of the workflow + - the answer is grounded in the deploy-to-prod / staging-prod documentation + validate: + answerIncludesAny: + - [ + "windmill.dev/docs/advanced/deploy_to_prod", + "windmill.dev/docs/advanced/deploy_gh_gl", + "windmill.dev/docs/advanced/canonical_deployment_setups", + "windmill.dev/docs/core_concepts/staging_prod", + "windmill.dev/docs/advanced/git_sync", + ] + +- id: ask-synthesis-handle-failures + prompt: What are my options for dealing with steps that fail in a flow — both retrying and being notified when something breaks? + runtime: + maxTurns: 10 + judgeChecklist: + - covers retries on failing steps + - covers error handlers / failure notification + - the answer is grounded in the error handling documentation + validate: + answerIncludesAny: + - ["windmill.dev/docs/core_concepts/error_handling"] + +- id: ask-synthesis-etl-pipeline + prompt: I want to build an ETL pipeline that pulls data, transforms it, and writes results to S3. How does Windmill support that? + runtime: + maxTurns: 10 + judgeChecklist: + - describes building a DAG/flow of steps for extract-transform-load + - mentions S3 / object storage integration for the data + - the answer is grounded in the data pipelines / object storage documentation + validate: + answerIncludesAny: + - [ + "windmill.dev/docs/core_concepts/data_pipelines", + "windmill.dev/docs/core_concepts/object_storage_in_windmill", + ] + +# --- Tier 4: niche precise facts -------------------------------------------- + +- id: ask-niche-worker-tags-env + prompt: In the self-hosted community edition, how do I make a specific worker only pick up certain jobs? + runtime: + maxTurns: 10 + judgeChecklist: + - explains assigning tags to jobs and to workers + - mentions the WORKER_TAGS environment variable used to configure a worker's tags + - the answer is grounded in the worker groups documentation + validate: + answerIncludesAny: + - ["windmill.dev/docs/core_concepts/worker_groups"] + +- id: ask-niche-s3-object-storage + prompt: How do I connect my workspace to S3 so scripts can read and write large files there? + runtime: + maxTurns: 10 + judgeChecklist: + - explains configuring an S3 / object storage connection for the workspace + - mentions reading/writing files via the workspace object storage + - the answer is grounded in the object storage documentation + validate: + answerIncludesAny: + - ["windmill.dev/docs/core_concepts/object_storage_in_windmill"] + +- id: ask-niche-data-tables + prompt: Does Windmill have a built-in way to store relational data without me setting up my own Postgres database? + runtime: + maxTurns: 10 + judgeChecklist: + - identifies Windmill data tables as the built-in relational storage + - explains storing and querying relational data without an external database + - the answer is grounded in the data tables documentation + validate: + answerIncludesAny: + - [ + "windmill.dev/docs/core_concepts/persistent_storage/data_tables", + "windmill.dev/docs/core_concepts/persistent_storage", + ] + +- id: ask-niche-key-value-store + prompt: Can I use something like Redis or a key-value store from my Windmill scripts? + runtime: + maxTurns: 10 + judgeChecklist: + - confirms key-value / NoSQL stores (e.g. Redis, MongoDB, Upstash) are supported via resources + - explains connecting and using them from scripts + - the answer is grounded in the key value stores documentation + validate: + answerIncludesAny: + - ["windmill.dev/docs/core_concepts/persistent_storage/key_value_stores"] + +- id: ask-niche-secret-encryption-key + prompt: How are my workspace secrets encrypted at rest, and can I rotate the encryption key? + runtime: + maxTurns: 10 + judgeChecklist: + - explains workspace secrets are encrypted with a workspace encryption key + - explains the key can be updated/rotated + - the answer is grounded in the workspace secret encryption documentation + validate: + answerIncludesAny: + - ["windmill.dev/docs/core_concepts/workspace_secret_encryption"] + +# --- Tier 5: out-of-scope controls ------------------------------------------ + +- id: ask-nodocs-cobol-runtime + prompt: Does Windmill have a built-in COBOL runtime for running COBOL scripts directly? + runtime: + maxTurns: 10 + judgeChecklist: + - does NOT claim Windmill has a built-in COBOL runtime + - says it is not sure / not a supported language rather than fabricating one + - optionally suggests a workaround (e.g. Docker/Bash) or asking the Windmill team + validate: + answerNotIncludes: + - "Windmill has a built-in COBOL runtime" + +- id: ask-nodocs-onchain-payments + prompt: Can Windmill settle blockchain cryptocurrency payments on-chain natively as a built-in feature? + runtime: + maxTurns: 10 + judgeChecklist: + - does NOT claim Windmill has a native on-chain crypto payment feature + - says it is not sure / not a documented feature rather than fabricating one + - optionally suggests doing it in a normal script or asking the Windmill team + validate: + answerNotIncludes: + - "Windmill natively settles" + +- id: ask-nodocs-voice-assistant + prompt: Does Windmill ship a built-in voice assistant that I can talk to with my microphone to run flows by voice? + runtime: + maxTurns: 10 + judgeChecklist: + - does NOT claim Windmill ships a built-in microphone voice assistant + - says it is not sure / not a documented feature rather than fabricating one + - optionally points to real trigger mechanisms or asking the Windmill team + validate: + answerNotIncludes: + - "Windmill ships a built-in voice assistant" diff --git a/ai_evals/cli/index.ts b/ai_evals/cli/index.ts index f92d6d7027..09d9a3d75d 100644 --- a/ai_evals/cli/index.ts +++ b/ai_evals/cli/index.ts @@ -54,6 +54,8 @@ async function main() { " bun run cli -- run flow --backend-validation preview", " bun run cli -- run flow flow-test5-simple-modification --runs 3", " bun run cli -- run global global-test1-script-create", + " bun run cli -- run ask --docs-tool llmstxt", + " bun run cli -- run ask ask-lookup-cron-schedule --docs-tool inkeep", " bun run cli -- run cli bun-hello-script", "", "Models:", @@ -71,7 +73,7 @@ async function main() { program .command("cases") .description("List available cases") - .argument("[mode]", "cli, flow, script, app, or global", parseOptionalMode) + .argument("[mode]", "cli, flow, script, app, global, or ask", parseOptionalMode) .action(async (mode?: EvalMode) => { await handleCases(mode); }); @@ -79,7 +81,7 @@ async function main() { program .command("run") .description("Run one benchmark mode") - .argument("", "cli, flow, script, app, or global", parseMode) + .argument("", "cli, flow, script, app, global, or ask", parseMode) .argument("[caseIds...]", "specific case ids to run") .option( "--runs ", @@ -105,6 +107,11 @@ async function main() { "--backend-validation ", `backend smoke validation (${BACKEND_VALIDATION_MODES.join(", ")})`, ) + .option( + "--docs-tool ", + "docs-tool arm for ask mode (inkeep, llmstxt)", + parseDocsTool, + ) .action( async ( mode: EvalMode, @@ -117,6 +124,7 @@ async function main() { verbose?: boolean; record?: boolean; backendValidation?: string; + docsTool?: string; }, ) => { await handleRun({ @@ -129,6 +137,7 @@ async function main() { verbose: options.verbose ?? false, record: options.record ?? false, backendValidation: options.backendValidation, + docsTool: options.docsTool, }); }, ); @@ -153,7 +162,7 @@ function handleModels() { process.stdout.write("Available models\n"); for (const model of EVAL_MODELS) { const supports = [ - ...(model.frontend ? ["flow", "script", "app", "global"] : []), + ...(model.frontend ? ["flow", "script", "app", "global", "ask"] : []), ...(model.cli ? ["cli"] : []), ]; const aliases = [ @@ -177,6 +186,7 @@ async function handleRun(input: { verbose: boolean; record: boolean; backendValidation?: string; + docsTool?: string; }) { if (input.record && input.caseIds.length > 0) { throw new Error( @@ -187,6 +197,9 @@ async function handleRun(input: { throw new Error("Use either --model or --models, not both"); } + // The docs-tool arm only applies to ask mode; default to the new llms.txt arm. + const docsTool = input.mode === "ask" ? input.docsTool ?? "llmstxt" : undefined; + const selectedCases = await loadSelectedCases(input.mode, input.caseIds); const models = resolveRequestedModels(input.mode, input.model, input.models); const backendValidation = parseBackendValidationMode( @@ -215,13 +228,17 @@ async function handleRun(input: { }> = []; for (const [index, model] of models.entries()) { - const runModel = formatRunModelLabel(input.mode, model); + const baseRunModel = formatRunModelLabel(input.mode, model); + // Distinguish saved results / history by the docs-tool arm. + const runModel = docsTool ? `${baseRunModel} ask:${docsTool}` : baseRunModel; if (models.length > 1) { process.stdout.write( `${index > 0 ? "\n" : ""}=== ${input.mode} ${model.id} (${runModel}) ===\n`, ); } - process.stderr.write(`Starting ${input.mode} benchmark...\n`); + process.stderr.write( + `Starting ${input.mode} benchmark${docsTool ? ` (docs-tool: ${docsTool})` : ""}...\n`, + ); const result = input.mode === "cli" @@ -238,6 +255,7 @@ async function handleRun(input: { model: model.id, verbose: input.verbose, backendValidation, + docsTool, }); const resolvedOutputPath = @@ -317,6 +335,13 @@ function parsePositiveInteger(value: string): number { return parsed; } +function parseDocsTool(value: string): string { + if (value === "inkeep" || value === "llmstxt") { + return value; + } + throw new InvalidArgumentError("docs-tool must be one of: inkeep, llmstxt"); +} + function resolveRequestedModels( mode: EvalMode, singleModel?: string, diff --git a/ai_evals/core/cases.test.ts b/ai_evals/core/cases.test.ts index 05e2f1527b..0077d34ebc 100644 --- a/ai_evals/core/cases.test.ts +++ b/ai_evals/core/cases.test.ts @@ -246,6 +246,35 @@ describe("loadCases", () => { }); }); + it("loads ask docs Q&A cases with citation validation", async () => { + const askCases = await loadCases("ask"); + expect(askCases.length).toBeGreaterThanOrEqual(25); + + const lookupCase = askCases.find((entry) => entry.id === "ask-lookup-cron-schedule"); + expect(lookupCase?.runtime).toEqual({ maxTurns: 10 }); + expect(lookupCase?.validate).toEqual({ + answerIncludesAny: [["windmill.dev/docs/core_concepts/scheduling"]], + }); + expect(lookupCase?.judgeChecklist?.length).toBeGreaterThan(0); + + const noDocsCase = askCases.find((entry) => entry.id === "ask-nodocs-cobol-runtime"); + expect(noDocsCase?.validate).toEqual({ + answerNotIncludes: ["Windmill has a built-in COBOL runtime"], + }); + + // Every case must cap turns and either cite docs or assert a forbidden claim. + for (const entry of askCases) { + expect(entry.runtime?.maxTurns).toBe(10); + const validate = entry.validate as + | { answerIncludesAny?: string[][]; answerNotIncludes?: string[] } + | undefined; + expect( + (validate?.answerIncludesAny?.length ?? 0) > 0 || + (validate?.answerNotIncludes?.length ?? 0) > 0, + ).toBe(true); + } + }); + it("loads tool expectations for workspace mutation cases", async () => { const scriptCases = await loadCases("script"); const caseEntry = scriptCases.find( diff --git a/ai_evals/core/models.test.ts b/ai_evals/core/models.test.ts index ba53c24592..e6205f560b 100644 --- a/ai_evals/core/models.test.ts +++ b/ai_evals/core/models.test.ts @@ -48,4 +48,11 @@ describe("resolveEvalModel", () => { "Model gemini-3-flash-preview is not supported for cli mode", ); }); + + it("resolves frontend models for ask mode", () => { + expect(resolveEvalModel("ask", "sonnet").frontend).toEqual({ + provider: "anthropic", + model: "claude-sonnet-4-5-20250929", + }); + }); }); diff --git a/ai_evals/core/models.ts b/ai_evals/core/models.ts index 295cd36135..b6e9de1462 100644 --- a/ai_evals/core/models.ts +++ b/ai_evals/core/models.ts @@ -161,7 +161,7 @@ export function resolveEvalModel( export function getEvalModelHelpText(): string { return EVAL_MODELS.map((model) => { const modes = [ - ...(model.frontend ? ["flow", "script", "app", "global"] : []), + ...(model.frontend ? ["flow", "script", "app", "global", "ask"] : []), ...(model.cli ? ["cli"] : []), ]; return ` ${model.id.padEnd(8)} ${model.label} (${modes.join(", ")})`; diff --git a/ai_evals/core/runSuite.ts b/ai_evals/core/runSuite.ts index ed82d841cb..bb0f9b99a4 100644 --- a/ai_evals/core/runSuite.ts +++ b/ai_evals/core/runSuite.ts @@ -225,7 +225,9 @@ async function runCaseAttempts(input: { checklist: input.evalCase.judgeChecklist, initial, expected: input.modeRunner.mode === "cli" ? undefined : expected, - actual: run.actual, + actual: input.modeRunner.prepareJudgeActual + ? input.modeRunner.prepareJudgeActual(run.actual) + : run.actual, model: input.judgeModel, }); diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 27c2fcddac..1d1bade81d 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -1,4 +1,4 @@ -export const EVAL_MODES = ["cli", "flow", "script", "app", "global"] as const; +export const EVAL_MODES = ["cli", "flow", "script", "app", "global", "ask"] as const; export type EvalMode = (typeof EVAL_MODES)[number]; @@ -131,6 +131,18 @@ export interface GlobalValidationSpec { }>; } +export interface AskValidationSpec { + /** + * URL-citation / required-mention check. A list of groups; each group passes + * if ANY of its alternative substrings appears in the answer + * (case-insensitive). Used where several documentation URLs are acceptable + * answers to the same question. + */ + answerIncludesAny?: string[][]; + /** Substrings that must NOT appear in the answer (case-insensitive). */ + answerNotIncludes?: string[]; +} + export interface CliValidationSpec { requiredSkills?: string[]; forbiddenSkills?: string[]; @@ -172,7 +184,11 @@ export interface ToolValidationSpec { toolCallArgs?: ToolCallArgumentRule[]; } -export type EvalValidationSpec = FlowValidationSpec | AppValidationSpec | GlobalValidationSpec; +export type EvalValidationSpec = + | FlowValidationSpec + | AppValidationSpec + | GlobalValidationSpec + | AskValidationSpec; export interface EvalCase { id: string; @@ -294,6 +310,12 @@ export interface ModeRunner { context: ModeRunContext; }): Promise; buildArtifacts?(actual: TActual): BenchmarkArtifactFile[]; + /** + * Optional transform applied to `actual` before it is handed to the LLM judge. + * Use it to strip fields the judge must stay blind to (e.g. which docs-tool + * arm produced an answer). When omitted, the judge receives `actual` as-is. + */ + prepareJudgeActual?(actual: TActual): unknown; } export interface BenchmarkAttemptResult { diff --git a/ai_evals/core/validators.test.ts b/ai_evals/core/validators.test.ts index 7f0e841366..90770599f7 100644 --- a/ai_evals/core/validators.test.ts +++ b/ai_evals/core/validators.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "bun:test"; import { validateAppState, + validateAskAnswer, validateCliWorkspace, validateGlobalState, validateScriptState, @@ -941,3 +942,76 @@ describe("validateCliWorkspace", () => { }); }); }); + +describe("validateAskAnswer", () => { + it("flags an empty answer", () => { + const checks = validateAskAnswer({ + actual: { answer: " ", docsTool: "llmstxt", toolsUsed: [], toolCallCount: 0 }, + }); + expect(checks).toContainEqual({ + name: "answer is non-empty", + passed: false, + }); + }); + + it("passes a citation group when any alternative URL appears (case-insensitive)", () => { + const checks = validateAskAnswer({ + actual: { + answer: + "You can schedule scripts with cron. See HTTPS://WWW.WINDMILL.DEV/DOCS/CORE_CONCEPTS/SCHEDULING for details.", + docsTool: "llmstxt", + toolsUsed: ["list_docs_pages", "read_docs_page"], + toolCallCount: 2, + }, + validate: { + answerIncludesAny: [ + [ + "windmill.dev/docs/core_concepts/scheduling", + "windmill.dev/docs/getting_started/triggers", + ], + ], + }, + }); + + const citationCheck = checks.find((entry) => + entry.name.startsWith("answer cites one of:") + ); + expect(citationCheck?.passed).toBe(true); + }); + + it("fails a citation group when none of the alternatives appear", () => { + const checks = validateAskAnswer({ + actual: { + answer: "Use schedules to run scripts on a cron.", + docsTool: "inkeep", + toolsUsed: ["get_documentation"], + toolCallCount: 1, + }, + validate: { + answerIncludesAny: [["windmill.dev/docs/core_concepts/scheduling"]], + }, + }); + + const citationCheck = checks.find((entry) => + entry.name.startsWith("answer cites one of:") + ); + expect(citationCheck?.passed).toBe(false); + }); + + it("enforces answerNotIncludes", () => { + const checks = validateAskAnswer({ + actual: { + answer: "Windmill has a built-in COBOL runtime.", + docsTool: "llmstxt", + toolsUsed: [], + toolCallCount: 0, + }, + validate: { answerNotIncludes: ["built-in cobol runtime"] }, + }); + + const notIncludesCheck = checks.find((entry) => + entry.name.startsWith("answer does not include") + ); + expect(notIncludesCheck?.passed).toBe(false); + }); +}); diff --git a/ai_evals/core/validators.ts b/ai_evals/core/validators.ts index e7a7641c00..4a1374a381 100644 --- a/ai_evals/core/validators.ts +++ b/ai_evals/core/validators.ts @@ -2,6 +2,7 @@ import path from "node:path"; import ts from "typescript"; import type { AppValidationSpec, + AskValidationSpec, BenchmarkCheck, CliTrace, CliValidationSpec, @@ -11,6 +12,13 @@ import type { ToolValidationSpec, } from "./types"; +export interface AskAnswerState { + answer: string; + docsTool: string; + toolsUsed: string[]; + toolCallCount: number; +} + export interface ScriptState { path: string; lang: string; @@ -402,6 +410,51 @@ export function validateGlobalState(input: { return checks; } +export function validateAskAnswer(input: { + actual: AskAnswerState; + validate?: AskValidationSpec; +}): BenchmarkCheck[] { + const checks: BenchmarkCheck[] = []; + const answer = input.actual.answer ?? ""; + const normalizedAnswer = answer.toLowerCase(); + + checks.push(check("answer is non-empty", answer.trim().length > 0)); + + const validate = input.validate; + if (!validate) { + return checks; + } + + for (const group of validate.answerIncludesAny ?? []) { + if (group.length === 0) { + continue; + } + const matched = group.some((needle) => + normalizedAnswer.includes(needle.toLowerCase()) + ); + checks.push( + check( + `answer cites one of: ${group.join(" | ")}`, + matched, + matched ? undefined : `answer: ${truncateForDetails(answer)}` + ) + ); + } + + for (const needle of validate.answerNotIncludes ?? []) { + const present = normalizedAnswer.includes(needle.toLowerCase()); + checks.push( + check( + `answer does not include '${needle}'`, + !present, + present ? `answer: ${truncateForDetails(answer)}` : undefined + ) + ); + } + + return checks; +} + export function validateAppState(input: { actual: AppFilesState; initial?: AppFilesState; diff --git a/ai_evals/modes/ask.ts b/ai_evals/modes/ask.ts new file mode 100644 index 0000000000..f8905958d0 --- /dev/null +++ b/ai_evals/modes/ask.ts @@ -0,0 +1,90 @@ +import { + runAskEval, + resolveDocsToolVariant, +} from "../adapters/frontend/core/ask/askEvalRunner"; +import type { FrontendEvalModelConfig } from "../core/models"; +import type { + AskValidationSpec, + BenchmarkArtifactFile, + ModeRunner, +} from "../core/types"; +import { validateAskAnswer, type AskAnswerState } from "../core/validators"; +import type { WindmillBackendSettings } from "../core/windmillBackendSettings"; +import { getFrontendApiKey } from "./frontendCommon"; + +export function createAskModeRunner( + modelConfig: FrontendEvalModelConfig, + backendSettings: WindmillBackendSettings, +): ModeRunner { + const variant = resolveDocsToolVariant(); + + return { + mode: "ask", + concurrency: 2, + judgeThreshold: 80, + async loadInitial() { + return undefined; + }, + async loadExpected() { + return undefined; + }, + async run(prompt, _initial, context) { + const result = await runAskEval( + prompt, + getFrontendApiKey(modelConfig.provider), + { + variant, + maxIterations: context.evalCase?.runtime?.maxTurns, + provider: modelConfig.provider, + model: modelConfig.model, + backend: backendSettings, + runContext: context, + }, + ); + + return { + success: result.success, + actual: result.state, + error: result.error, + assistantMessageCount: result.assistantMessageCount, + toolCallCount: result.toolCallCount, + toolsUsed: result.toolsUsed, + toolCallDetails: result.toolCallDetails, + skillsInvoked: [], + tokenUsage: result.tokenUsage, + }; + }, + validate({ evalCase, actual }) { + return validateAskAnswer({ + actual, + validate: evalCase.validate as AskValidationSpec | undefined, + }); + }, + // The judge must stay blind to which docs-tool arm produced the answer, so + // it only ever sees the answer text. + prepareJudgeActual(actual) { + return { answer: actual.answer }; + }, + buildArtifacts(actual): BenchmarkArtifactFile[] { + return [ + { + path: "ask-answer.md", + content: `${actual.answer}\n`, + }, + { + path: "ask-meta.json", + content: + JSON.stringify( + { + docsTool: actual.docsTool, + toolsUsed: [...new Set(actual.toolsUsed)], + toolCallCount: actual.toolCallCount, + }, + null, + 2, + ) + "\n", + }, + ]; + }, + }; +}