test(ai-evals): add ask benchmark mode comparing inkeep vs llms.txt docs tools

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

ai_evals/adapters/frontend/benchmarkRunner.ts

@@ -12,7 +12,7 @@ import { resolveWindmillBackendSettings } from "../../core/windmillBackendSettin
 import { emitFrontendBenchmarkProgress } from "./progress";
 import { DEFAULT_JUDGE_MODEL } from "../../core/judge";
 
-export type FrontendBenchmarkMode = "flow" | "app" | "script" | "global";
+export type FrontendBenchmarkMode = "flow" | "app" | "script" | "global" | "ask";
 
 export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult> {
   const mode = parseMode(process.env.WMILL_FRONTEND_AI_EVAL_MODE);
@@ -42,7 +42,11 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
     backendValidation,
     backendSettings,
   );
-  const runModel = formatRunModelLabel(mode, model);
+  const docsTool =
+    mode === "ask" ? process.env.WMILL_AI_EVAL_DOCS_TOOL?.trim() || "llmstxt" : undefined;
+  const runModel = docsTool
+    ? `${formatRunModelLabel(mode, model)} ask:${docsTool}`
+    : formatRunModelLabel(mode, model);
   const caseResults = await runSuite({
     modeRunner,
     cases: selectedCases,
@@ -92,11 +96,21 @@ async function getModeRunner(
       const { createGlobalModeRunner } = await import("../../modes/global");
       return createGlobalModeRunner(model, backendSettings);
     }
+    case "ask": {
+      const { createAskModeRunner } = await import("../../modes/ask");
+      return createAskModeRunner(model, backendSettings);
+    }
   }
 }
 
 function parseMode(value: string | undefined): FrontendBenchmarkMode {
-  if (value === "flow" || value === "app" || value === "script" || value === "global") {
+  if (
+    value === "flow" ||
+    value === "app" ||
+    value === "script" ||
+    value === "global" ||
+    value === "ask"
+  ) {
     return value;
   }
   throw new Error(`Unsupported frontend benchmark mode: ${String(value)}`);

ai_evals/adapters/frontend/core/ask/askEvalRunner.ts

@@ -0,0 +1,175 @@
+import type { ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs";
+import type { AIProvider } from "$lib/gen/types.gen";
+import {
+  getAskTools,
+  prepareAskSystemMessage,
+  prepareAskUserMessage,
+  type DocsToolVariant,
+} from "../../../../../frontend/src/lib/components/copilot/chat/ask/core";
+import type { ModeRunContext } from "../../../../core/types";
+import type { AskAnswerState } from "../../../../core/validators";
+import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
+import { WindmillBackendClient } from "../../windmillBackend";
+import { runEval } from "../shared";
+import type { TokenUsage, ToolCallDetail } from "../shared/types";
+
+export interface AskEvalResult {
+  success: boolean;
+  state: AskAnswerState;
+  error?: string;
+  assistantMessageCount: number;
+  toolCallCount: number;
+  toolsUsed: string[];
+  toolCallDetails: ToolCallDetail[];
+  tokenUsage: TokenUsage;
+}
+
+export interface AskEvalOptions {
+  variant: DocsToolVariant;
+  model?: string;
+  maxIterations?: number;
+  provider?: AIProvider;
+  backend: WindmillBackendSettings;
+  runContext?: ModeRunContext;
+}
+
+const DOCS_TOOL_ENV = "WMILL_AI_EVAL_DOCS_TOOL";
+
+/**
+ * Resolves which docs-tool arm to benchmark. Defaults to the new llms.txt arm.
+ */
+export function resolveDocsToolVariant(
+  value: string | undefined = process.env[DOCS_TOOL_ENV],
+): DocsToolVariant {
+  return value === "inkeep" ? "inkeep" : "llmstxt";
+}
+
+export async function runAskEval(
+  userPrompt: string,
+  apiKey: string,
+  options: AskEvalOptions,
+): Promise<AskEvalResult> {
+  // The production inkeep tool calls fetch('/api/inkeep') with a relative URL,
+  // which has no origin under node. Install a process-wide shim that rewrites
+  // /api/* calls to the eval backend with an auth token. Only needed for the
+  // inkeep arm; the llms.txt arm fetches absolute windmill.dev URLs natively.
+  if (options.variant === "inkeep") {
+    await installInkeepFetchShim(options.backend);
+  }
+
+  const model = options.model ?? "claude-haiku-4-5-20251001";
+
+  const rawResult = await runEval({
+    userPrompt,
+    systemMessage: prepareAskSystemMessage(undefined, options.variant),
+    userMessage: prepareAskUserMessage(userPrompt),
+    tools: getAskTools(options.variant),
+    helpers: {},
+    apiKey,
+    getOutput: () => ({
+      answer: "",
+      docsTool: options.variant,
+      toolsUsed: [],
+      toolCallCount: 0,
+    }),
+    onAssistantMessageStart: options.runContext?.onAssistantMessageStart,
+    onAssistantToken: options.runContext?.onAssistantChunk,
+    onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd,
+    onToolCall: options.runContext?.onToolCall,
+    options: {
+      maxIterations: options.maxIterations,
+      model,
+      provider: options.provider,
+      backend: options.backend,
+      caseId: options.runContext?.caseId,
+      attempt: options.runContext?.attempt,
+    },
+  });
+
+  const answer = extractFinalAnswer(rawResult.messages);
+
+  return {
+    success: rawResult.success,
+    state: {
+      answer,
+      docsTool: options.variant,
+      toolsUsed: rawResult.toolsCalled,
+      toolCallCount: rawResult.toolCallsCount,
+    },
+    error: rawResult.error,
+    assistantMessageCount: rawResult.iterations,
+    toolCallCount: rawResult.toolCallsCount,
+    toolsUsed: rawResult.toolsCalled,
+    toolCallDetails: rawResult.toolCallDetails,
+    tokenUsage: rawResult.tokenUsage,
+  };
+}
+
+/**
+ * The final answer is the last assistant message with non-empty string content.
+ */
+export function extractFinalAnswer(
+  messages: ChatCompletionMessageParam[],
+): string {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const message = messages[i];
+    if (message.role !== "assistant") {
+      continue;
+    }
+    const content = message.content;
+    if (typeof content === "string" && content.trim().length > 0) {
+      return content;
+    }
+    if (Array.isArray(content)) {
+      const text = content
+        .map((part) =>
+          part && typeof part === "object" && "text" in part
+            ? String((part as { text?: unknown }).text ?? "")
+            : "",
+        )
+        .join("")
+        .trim();
+      if (text.length > 0) {
+        return text;
+      }
+    }
+  }
+  return "";
+}
+
+let inkeepFetchShimInstalled = false;
+
+/**
+ * Installs (once) a process-wide fetch wrapper that rewrites relative /api/*
+ * requests to the eval backend, adding a bearer token. All other URLs are
+ * passed through untouched.
+ */
+async function installInkeepFetchShim(
+  backend: WindmillBackendSettings,
+): Promise<void> {
+  if (inkeepFetchShimInstalled) {
+    return;
+  }
+  inkeepFetchShimInstalled = true;
+
+  const client = new WindmillBackendClient(backend);
+  const originalFetch = globalThis.fetch.bind(globalThis);
+
+  globalThis.fetch = (async (
+    input: RequestInfo | URL,
+    init?: RequestInit,
+  ): Promise<Response> => {
+    if (typeof input === "string" && input.startsWith("/api/")) {
+      const token = await client.getToken();
+      const url = `${backend.baseUrl}${input}`;
+      return await originalFetch(url, {
+        ...init,
+        headers: {
+          ...(init?.headers ?? {}),
+          Authorization: `Bearer ${token}`,
+        },
+      });
+    }
+    return await originalFetch(input as any, init);
+  }) as typeof fetch;
+}

ai_evals/adapters/frontend/progress.ts

@@ -1,4 +1,4 @@
-export type FrontendBenchmarkProgressSurface = 'flow' | 'app' | 'script' | 'global'
+export type FrontendBenchmarkProgressSurface = 'flow' | 'app' | 'script' | 'global' | 'ask'
 
 export type FrontendBenchmarkProgressEvent =
 	| {

ai_evals/adapters/frontend/runtime.ts

@@ -16,7 +16,7 @@ const FRONTEND_BENCHMARK_TEST =
 const FRONTEND_BENCHMARK_CONFIG =
   "../ai_evals/adapters/frontend/vitest.config.ts";
 
-export type FrontendMode = "flow" | "app" | "script" | "global";
+export type FrontendMode = "flow" | "app" | "script" | "global" | "ask";
 
 export async function runFrontendBenchmarkAdapter(input: {
   mode: FrontendMode;
@@ -25,6 +25,7 @@ export async function runFrontendBenchmarkAdapter(input: {
   model?: string;
   verbose?: boolean;
   backendValidation?: string;
+  docsTool?: string;
 }): Promise<BenchmarkRunResult> {
   const tempDir = await mkdtemp(
     path.join(tmpdir(), "wmill-frontend-benchmark-"),
@@ -41,6 +42,8 @@ export async function runFrontendBenchmarkAdapter(input: {
     WMILL_FRONTEND_AI_EVAL_PROGRESS: "1",
     WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? "1" : "0",
     WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION: input.backendValidation ?? "",
+    // Only meaningful for ask mode; selects the docs-tool arm to benchmark.
+    WMILL_AI_EVAL_DOCS_TOOL: input.docsTool ?? process.env.WMILL_AI_EVAL_DOCS_TOOL ?? "",
   };
 
   try {

ai_evals/adapters/frontend/vitestAdapter.test.ts

@@ -434,5 +434,6 @@ benchmarkIt(
 			resetBenchmarkMockBackend()
 		}
 	},
-	600_000
+	// Full-suite runs (30+ cases at concurrency 2-3) routinely exceed 10 minutes.
+	7_200_000
 )

ai_evals/cases/ask.yaml

@@ -0,0 +1,419 @@
+# AI-chat documentation Q&A cases. Each case asks a realistic user question and
+# is graded on answer quality (judge) plus a deterministic URL-citation check.
+# Ground-truth URLs are derived from https://www.windmill.dev/llms.txt — all
+# `answerIncludesAny` URLs are real docs pages (without the `.md` suffix).
+
+# --- Tier 1: direct lookups -------------------------------------------------
+
+- id: ask-lookup-cron-schedule
+  prompt: How do I run a script automatically every Monday at 9am?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains that recurring runs use a schedule with a CRON expression
+    - mentions a concrete cron-like expression or the schedule configuration
+    - the answer is grounded in the Windmill scheduling documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/scheduling"]
+
+- id: ask-lookup-secret-in-python
+  prompt: How do I store a secret like an API key and use it inside a Python script?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains storing the value as a secret variable
+    - explains accessing it from a Python script (e.g. wmill.get_variable or get_resource)
+    - the answer is grounded in the variables and secrets documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/variables_and_secrets"]
+
+- id: ask-lookup-webhook-trigger
+  prompt: How can I trigger one of my scripts from an external service by calling a URL?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains that scripts expose webhook URLs
+    - mentions sync vs async webhook behavior or how to find the webhook URL
+    - the answer is grounded in the webhooks documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/webhooks"]
+
+- id: ask-lookup-retry-step
+  prompt: One step in my flow calls a flaky API. How do I make it automatically retry if it fails?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains configuring retries on a flow step
+    - mentions the number of attempts and/or delay between attempts
+    - the answer is grounded in the error handling or flow editor documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/flows/retries",
+          "windmill.dev/docs/core_concepts/error_handling",
+          "windmill.dev/docs/flows/flow_editor",
+        ]
+
+- id: ask-lookup-connect-database
+  prompt: How do I connect to my Postgres database so my scripts can query it?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains creating a resource holding the database connection details
+    - explains referencing that resource from a script
+    - the answer is grounded in the resources documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/core_concepts/resources_and_types",
+          "windmill.dev/docs/getting_started/scripts_quickstart/sql",
+        ]
+
+- id: ask-lookup-python-deps
+  prompt: How do I add a third-party pip package to a Python script in Windmill?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains that imports are auto-detected and dependencies resolved automatically
+    - mentions how to pin a version or the requirements mechanism
+    - the answer is grounded in the Python scripting documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/getting_started/scripts_quickstart/python",
+          "windmill.dev/docs/advanced/dependencies_in_python",
+        ]
+
+- id: ask-lookup-concurrency-limit
+  prompt: How do I stop a script from running too many times at once so I don't hit an API rate limit?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains setting a concurrency limit on the script
+    - mentions the max number of concurrent executions and/or time window
+    - the answer is grounded in the concurrency limits documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/core_concepts/concurrency_limits",
+          "windmill.dev/docs/script_editor/concurrency_limit",
+        ]
+
+- id: ask-lookup-app-table-button
+  prompt: How do I build a simple internal tool with a table and a button using Windmill?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains using the app editor with drag-and-drop components
+    - mentions wiring a component to a backend script/runnable
+    - the answer is grounded in the app editor or apps quickstart documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/apps/app_editor",
+          "windmill.dev/docs/getting_started/apps_quickstart",
+        ]
+
+# --- Tier 2: conceptual / vocabulary-mismatch -------------------------------
+
+- id: ask-concept-flow-wait-for-approval
+  prompt: How can I make a flow pause and wait for a person to approve before it continues?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies this as a suspend / approval step in a flow
+    - explains that the flow resumes once approved (e.g. via a resume link or form)
+    - the answer is grounded in the flow editor / approval documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/flows/flow_editor",
+          "windmill.dev/docs/flows/flow_approval",
+        ]
+
+- id: ask-concept-remember-value-between-runs
+  prompt: Can a script remember a value from its previous run, like the last timestamp it processed?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies states as the mechanism (getState / setState)
+    - explains state persists between runs of the same script
+    - the answer is grounded in the states / within-windmill documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/persistent_storage/within_windmill"]
+
+- id: ask-concept-steps-at-same-time
+  prompt: How do I make several steps in my flow run at the same time instead of one after another?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies running branches in parallel (branchall / parallel branches)
+    - explains the steps execute concurrently
+    - the answer is grounded in the flow editor documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/flows/flow_editor",
+          "windmill.dev/docs/flows/flow_branches",
+        ]
+
+- id: ask-concept-react-to-db-changes
+  prompt: I want something to happen automatically whenever a new row is inserted into my Postgres table. Is that possible?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies Postgres triggers reacting to insert/update/delete
+    - mentions it listens to database change events (logical replication)
+    - the answer is grounded in the Postgres triggers documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/core_concepts/postgres_triggers",
+          "windmill.dev/docs/getting_started/triggers",
+        ]
+
+- id: ask-concept-reuse-config-everywhere
+  prompt: I keep pasting the same base URL and credentials into many scripts. Is there a cleaner way to manage that shared config?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies variables (and/or resources) as the mechanism for reusable config
+    - explains that the config is defined once and referenced from scripts
+    - the answer is grounded in the variables / resources documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/core_concepts/variables_and_secrets",
+          "windmill.dev/docs/core_concepts/resources_and_types",
+        ]
+
+- id: ask-concept-let-llm-call-my-scripts
+  prompt: I want my AI assistant in Claude or Cursor to be able to run my Windmill scripts. How would I set that up?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies MCP (Model Context Protocol) as the mechanism
+    - explains connecting an MCP client to Windmill to trigger scripts/flows
+    - the answer is grounded in the MCP documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/mcp"]
+
+- id: ask-concept-avoid-recompute
+  prompt: My script does an expensive computation with the same inputs a lot. Can Windmill avoid recomputing the same result every time?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies caching of script/flow results
+    - explains that identical inputs reuse the cached result for a configured duration
+    - the answer is grounded in the caching documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/caching"]
+
+- id: ask-concept-agent-step
+  prompt: I want a flow step where an LLM decides which of my scripts to call based on the input. Does Windmill support that?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies AI agent steps in a flow
+    - explains the agent can call tools/scripts based on the input
+    - the answer is grounded in the AI agents documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/ai_agents"]
+
+# --- Tier 3: multi-page synthesis -------------------------------------------
+
+- id: ask-synthesis-variable-vs-resource
+  prompt: What's the difference between a variable and a resource in Windmill, and when should I use each?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains a variable holds a single reusable value (and secrets for sensitive ones)
+    - explains a resource holds a structured connection object (e.g. database/API credentials)
+    - gives reasonable guidance on when to use each
+    - the answer is grounded in the variables and resources documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/variables_and_secrets"]
+      - ["windmill.dev/docs/core_concepts/resources_and_types"]
+
+- id: ask-synthesis-trigger-options
+  prompt: What are all the different ways I can trigger a flow — both on a schedule and from external events?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - covers scheduled/cron triggering
+    - covers event-based triggering (e.g. webhooks, HTTP routes, or message queues)
+    - the answer is grounded in the triggers / scheduling documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/getting_started/triggers",
+          "windmill.dev/docs/core_concepts/scheduling",
+        ]
+
+- id: ask-synthesis-storage-options
+  prompt: Where should I store data in Windmill? I'm confused about all the storage options.
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - distinguishes object storage (S3) from relational/structured storage and lightweight state/KV options
+    - gives reasonable guidance on which to use when
+    - the answer is grounded in the persistent storage documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/persistent_storage"]
+
+- id: ask-synthesis-deploy-to-prod
+  prompt: I'm building in a dev workspace and want a safe way to promote my scripts and flows to production. What's the recommended workflow?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - describes promoting from staging/dev to prod and/or git-based deployment
+    - mentions draft/deploy and/or git sync as part of the workflow
+    - the answer is grounded in the deploy-to-prod / staging-prod documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/advanced/deploy_to_prod",
+          "windmill.dev/docs/advanced/deploy_gh_gl",
+          "windmill.dev/docs/advanced/canonical_deployment_setups",
+          "windmill.dev/docs/core_concepts/staging_prod",
+          "windmill.dev/docs/advanced/git_sync",
+        ]
+
+- id: ask-synthesis-handle-failures
+  prompt: What are my options for dealing with steps that fail in a flow — both retrying and being notified when something breaks?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - covers retries on failing steps
+    - covers error handlers / failure notification
+    - the answer is grounded in the error handling documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/error_handling"]
+
+- id: ask-synthesis-etl-pipeline
+  prompt: I want to build an ETL pipeline that pulls data, transforms it, and writes results to S3. How does Windmill support that?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - describes building a DAG/flow of steps for extract-transform-load
+    - mentions S3 / object storage integration for the data
+    - the answer is grounded in the data pipelines / object storage documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/core_concepts/data_pipelines",
+          "windmill.dev/docs/core_concepts/object_storage_in_windmill",
+        ]
+
+# --- Tier 4: niche precise facts --------------------------------------------
+
+- id: ask-niche-worker-tags-env
+  prompt: In the self-hosted community edition, how do I make a specific worker only pick up certain jobs?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains assigning tags to jobs and to workers
+    - mentions the WORKER_TAGS environment variable used to configure a worker's tags
+    - the answer is grounded in the worker groups documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/worker_groups"]
+
+- id: ask-niche-s3-object-storage
+  prompt: How do I connect my workspace to S3 so scripts can read and write large files there?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains configuring an S3 / object storage connection for the workspace
+    - mentions reading/writing files via the workspace object storage
+    - the answer is grounded in the object storage documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/object_storage_in_windmill"]
+
+- id: ask-niche-data-tables
+  prompt: Does Windmill have a built-in way to store relational data without me setting up my own Postgres database?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies Windmill data tables as the built-in relational storage
+    - explains storing and querying relational data without an external database
+    - the answer is grounded in the data tables documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/core_concepts/persistent_storage/data_tables",
+          "windmill.dev/docs/core_concepts/persistent_storage",
+        ]
+
+- id: ask-niche-key-value-store
+  prompt: Can I use something like Redis or a key-value store from my Windmill scripts?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - confirms key-value / NoSQL stores (e.g. Redis, MongoDB, Upstash) are supported via resources
+    - explains connecting and using them from scripts
+    - the answer is grounded in the key value stores documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/persistent_storage/key_value_stores"]
+
+- id: ask-niche-secret-encryption-key
+  prompt: How are my workspace secrets encrypted at rest, and can I rotate the encryption key?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains workspace secrets are encrypted with a workspace encryption key
+    - explains the key can be updated/rotated
+    - the answer is grounded in the workspace secret encryption documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/workspace_secret_encryption"]
+
+# --- Tier 5: out-of-scope controls ------------------------------------------
+
+- id: ask-nodocs-cobol-runtime
+  prompt: Does Windmill have a built-in COBOL runtime for running COBOL scripts directly?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - does NOT claim Windmill has a built-in COBOL runtime
+    - says it is not sure / not a supported language rather than fabricating one
+    - optionally suggests a workaround (e.g. Docker/Bash) or asking the Windmill team
+  validate:
+    answerNotIncludes:
+      - "Windmill has a built-in COBOL runtime"
+
+- id: ask-nodocs-onchain-payments
+  prompt: Can Windmill settle blockchain cryptocurrency payments on-chain natively as a built-in feature?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - does NOT claim Windmill has a native on-chain crypto payment feature
+    - says it is not sure / not a documented feature rather than fabricating one
+    - optionally suggests doing it in a normal script or asking the Windmill team
+  validate:
+    answerNotIncludes:
+      - "Windmill natively settles"
+
+- id: ask-nodocs-voice-assistant
+  prompt: Does Windmill ship a built-in voice assistant that I can talk to with my microphone to run flows by voice?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - does NOT claim Windmill ships a built-in microphone voice assistant
+    - says it is not sure / not a documented feature rather than fabricating one
+    - optionally points to real trigger mechanisms or asking the Windmill team
+  validate:
+    answerNotIncludes:
+      - "Windmill ships a built-in voice assistant"

ai_evals/cli/index.ts

@@ -54,6 +54,8 @@ async function main() {
         "  bun run cli -- run flow --backend-validation preview",
         "  bun run cli -- run flow flow-test5-simple-modification --runs 3",
         "  bun run cli -- run global global-test1-script-create",
+        "  bun run cli -- run ask --docs-tool llmstxt",
+        "  bun run cli -- run ask ask-lookup-cron-schedule --docs-tool inkeep",
         "  bun run cli -- run cli bun-hello-script",
         "",
         "Models:",
@@ -71,7 +73,7 @@ async function main() {
   program
     .command("cases")
     .description("List available cases")
-    .argument("[mode]", "cli, flow, script, app, or global", parseOptionalMode)
+    .argument("[mode]", "cli, flow, script, app, global, or ask", parseOptionalMode)
     .action(async (mode?: EvalMode) => {
       await handleCases(mode);
     });
@@ -79,7 +81,7 @@ async function main() {
   program
     .command("run")
     .description("Run one benchmark mode")
-    .argument("<mode>", "cli, flow, script, app, or global", parseMode)
+    .argument("<mode>", "cli, flow, script, app, global, or ask", parseMode)
     .argument("[caseIds...]", "specific case ids to run")
     .option(
       "--runs <n>",
@@ -105,6 +107,11 @@ async function main() {
       "--backend-validation <mode>",
       `backend smoke validation (${BACKEND_VALIDATION_MODES.join(", ")})`,
     )
+    .option(
+      "--docs-tool <arm>",
+      "docs-tool arm for ask mode (inkeep, llmstxt)",
+      parseDocsTool,
+    )
     .action(
       async (
         mode: EvalMode,
@@ -117,6 +124,7 @@ async function main() {
           verbose?: boolean;
           record?: boolean;
           backendValidation?: string;
+          docsTool?: string;
         },
       ) => {
         await handleRun({
@@ -129,6 +137,7 @@ async function main() {
           verbose: options.verbose ?? false,
           record: options.record ?? false,
           backendValidation: options.backendValidation,
+          docsTool: options.docsTool,
         });
       },
     );
@@ -153,7 +162,7 @@ function handleModels() {
   process.stdout.write("Available models\n");
   for (const model of EVAL_MODELS) {
     const supports = [
-      ...(model.frontend ? ["flow", "script", "app", "global"] : []),
+      ...(model.frontend ? ["flow", "script", "app", "global", "ask"] : []),
       ...(model.cli ? ["cli"] : []),
     ];
     const aliases = [
@@ -177,6 +186,7 @@ async function handleRun(input: {
   verbose: boolean;
   record: boolean;
   backendValidation?: string;
+  docsTool?: string;
 }) {
   if (input.record && input.caseIds.length > 0) {
     throw new Error(
@@ -187,6 +197,9 @@ async function handleRun(input: {
     throw new Error("Use either --model or --models, not both");
   }
 
+  // The docs-tool arm only applies to ask mode; default to the new llms.txt arm.
+  const docsTool = input.mode === "ask" ? input.docsTool ?? "llmstxt" : undefined;
+
   const selectedCases = await loadSelectedCases(input.mode, input.caseIds);
   const models = resolveRequestedModels(input.mode, input.model, input.models);
   const backendValidation = parseBackendValidationMode(
@@ -215,13 +228,17 @@ async function handleRun(input: {
   }> = [];
 
   for (const [index, model] of models.entries()) {
-    const runModel = formatRunModelLabel(input.mode, model);
+    const baseRunModel = formatRunModelLabel(input.mode, model);
+    // Distinguish saved results / history by the docs-tool arm.
+    const runModel = docsTool ? `${baseRunModel} ask:${docsTool}` : baseRunModel;
     if (models.length > 1) {
       process.stdout.write(
         `${index > 0 ? "\n" : ""}=== ${input.mode} ${model.id} (${runModel}) ===\n`,
       );
     }
-    process.stderr.write(`Starting ${input.mode} benchmark...\n`);
+    process.stderr.write(
+      `Starting ${input.mode} benchmark${docsTool ? ` (docs-tool: ${docsTool})` : ""}...\n`,
+    );
 
     const result =
       input.mode === "cli"
@@ -238,6 +255,7 @@ async function handleRun(input: {
             model: model.id,
             verbose: input.verbose,
             backendValidation,
+            docsTool,
           });
 
     const resolvedOutputPath =
@@ -317,6 +335,13 @@ function parsePositiveInteger(value: string): number {
   return parsed;
 }
 
+function parseDocsTool(value: string): string {
+  if (value === "inkeep" || value === "llmstxt") {
+    return value;
+  }
+  throw new InvalidArgumentError("docs-tool must be one of: inkeep, llmstxt");
+}
+
 function resolveRequestedModels(
   mode: EvalMode,
   singleModel?: string,

ai_evals/core/cases.test.ts

@@ -246,6 +246,35 @@ describe("loadCases", () => {
     });
   });
 
+  it("loads ask docs Q&A cases with citation validation", async () => {
+    const askCases = await loadCases("ask");
+    expect(askCases.length).toBeGreaterThanOrEqual(25);
+
+    const lookupCase = askCases.find((entry) => entry.id === "ask-lookup-cron-schedule");
+    expect(lookupCase?.runtime).toEqual({ maxTurns: 10 });
+    expect(lookupCase?.validate).toEqual({
+      answerIncludesAny: [["windmill.dev/docs/core_concepts/scheduling"]],
+    });
+    expect(lookupCase?.judgeChecklist?.length).toBeGreaterThan(0);
+
+    const noDocsCase = askCases.find((entry) => entry.id === "ask-nodocs-cobol-runtime");
+    expect(noDocsCase?.validate).toEqual({
+      answerNotIncludes: ["Windmill has a built-in COBOL runtime"],
+    });
+
+    // Every case must cap turns and either cite docs or assert a forbidden claim.
+    for (const entry of askCases) {
+      expect(entry.runtime?.maxTurns).toBe(10);
+      const validate = entry.validate as
+        | { answerIncludesAny?: string[][]; answerNotIncludes?: string[] }
+        | undefined;
+      expect(
+        (validate?.answerIncludesAny?.length ?? 0) > 0 ||
+          (validate?.answerNotIncludes?.length ?? 0) > 0,
+      ).toBe(true);
+    }
+  });
+
   it("loads tool expectations for workspace mutation cases", async () => {
     const scriptCases = await loadCases("script");
     const caseEntry = scriptCases.find(

ai_evals/core/models.test.ts

@@ -48,4 +48,11 @@ describe("resolveEvalModel", () => {
       "Model gemini-3-flash-preview is not supported for cli mode",
     );
   });
+
+  it("resolves frontend models for ask mode", () => {
+    expect(resolveEvalModel("ask", "sonnet").frontend).toEqual({
+      provider: "anthropic",
+      model: "claude-sonnet-4-5-20250929",
+    });
+  });
 });

ai_evals/core/models.ts

@@ -161,7 +161,7 @@ export function resolveEvalModel(
 export function getEvalModelHelpText(): string {
   return EVAL_MODELS.map((model) => {
     const modes = [
-      ...(model.frontend ? ["flow", "script", "app", "global"] : []),
+      ...(model.frontend ? ["flow", "script", "app", "global", "ask"] : []),
       ...(model.cli ? ["cli"] : []),
     ];
     return `  ${model.id.padEnd(8)} ${model.label} (${modes.join(", ")})`;

ai_evals/core/runSuite.ts

@@ -225,7 +225,9 @@ async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
           checklist: input.evalCase.judgeChecklist,
           initial,
           expected: input.modeRunner.mode === "cli" ? undefined : expected,
-          actual: run.actual,
+          actual: input.modeRunner.prepareJudgeActual
+            ? input.modeRunner.prepareJudgeActual(run.actual)
+            : run.actual,
           model: input.judgeModel,
         });
 

ai_evals/core/types.ts

@@ -1,4 +1,4 @@
-export const EVAL_MODES = ["cli", "flow", "script", "app", "global"] as const;
+export const EVAL_MODES = ["cli", "flow", "script", "app", "global", "ask"] as const;
 
 export type EvalMode = (typeof EVAL_MODES)[number];
 
@@ -131,6 +131,18 @@ export interface GlobalValidationSpec {
   }>;
 }
 
+export interface AskValidationSpec {
+  /**
+   * URL-citation / required-mention check. A list of groups; each group passes
+   * if ANY of its alternative substrings appears in the answer
+   * (case-insensitive). Used where several documentation URLs are acceptable
+   * answers to the same question.
+   */
+  answerIncludesAny?: string[][];
+  /** Substrings that must NOT appear in the answer (case-insensitive). */
+  answerNotIncludes?: string[];
+}
+
 export interface CliValidationSpec {
   requiredSkills?: string[];
   forbiddenSkills?: string[];
@@ -172,7 +184,11 @@ export interface ToolValidationSpec {
   toolCallArgs?: ToolCallArgumentRule[];
 }
 
-export type EvalValidationSpec = FlowValidationSpec | AppValidationSpec | GlobalValidationSpec;
+export type EvalValidationSpec =
+  | FlowValidationSpec
+  | AppValidationSpec
+  | GlobalValidationSpec
+  | AskValidationSpec;
 
 export interface EvalCase {
   id: string;
@@ -294,6 +310,12 @@ export interface ModeRunner<TInitial, TExpected, TActual> {
     context: ModeRunContext;
   }): Promise<BackendValidationResult | null>;
   buildArtifacts?(actual: TActual): BenchmarkArtifactFile[];
+  /**
+   * Optional transform applied to `actual` before it is handed to the LLM judge.
+   * Use it to strip fields the judge must stay blind to (e.g. which docs-tool
+   * arm produced an answer). When omitted, the judge receives `actual` as-is.
+   */
+  prepareJudgeActual?(actual: TActual): unknown;
 }
 
 export interface BenchmarkAttemptResult {

ai_evals/core/validators.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it } from "bun:test";
 import {
   validateAppState,
+  validateAskAnswer,
   validateCliWorkspace,
   validateGlobalState,
   validateScriptState,
@@ -941,3 +942,76 @@ describe("validateCliWorkspace", () => {
     });
   });
 });
+
+describe("validateAskAnswer", () => {
+  it("flags an empty answer", () => {
+    const checks = validateAskAnswer({
+      actual: { answer: "   ", docsTool: "llmstxt", toolsUsed: [], toolCallCount: 0 },
+    });
+    expect(checks).toContainEqual({
+      name: "answer is non-empty",
+      passed: false,
+    });
+  });
+
+  it("passes a citation group when any alternative URL appears (case-insensitive)", () => {
+    const checks = validateAskAnswer({
+      actual: {
+        answer:
+          "You can schedule scripts with cron. See HTTPS://WWW.WINDMILL.DEV/DOCS/CORE_CONCEPTS/SCHEDULING for details.",
+        docsTool: "llmstxt",
+        toolsUsed: ["list_docs_pages", "read_docs_page"],
+        toolCallCount: 2,
+      },
+      validate: {
+        answerIncludesAny: [
+          [
+            "windmill.dev/docs/core_concepts/scheduling",
+            "windmill.dev/docs/getting_started/triggers",
+          ],
+        ],
+      },
+    });
+
+    const citationCheck = checks.find((entry) =>
+      entry.name.startsWith("answer cites one of:")
+    );
+    expect(citationCheck?.passed).toBe(true);
+  });
+
+  it("fails a citation group when none of the alternatives appear", () => {
+    const checks = validateAskAnswer({
+      actual: {
+        answer: "Use schedules to run scripts on a cron.",
+        docsTool: "inkeep",
+        toolsUsed: ["get_documentation"],
+        toolCallCount: 1,
+      },
+      validate: {
+        answerIncludesAny: [["windmill.dev/docs/core_concepts/scheduling"]],
+      },
+    });
+
+    const citationCheck = checks.find((entry) =>
+      entry.name.startsWith("answer cites one of:")
+    );
+    expect(citationCheck?.passed).toBe(false);
+  });
+
+  it("enforces answerNotIncludes", () => {
+    const checks = validateAskAnswer({
+      actual: {
+        answer: "Windmill has a built-in COBOL runtime.",
+        docsTool: "llmstxt",
+        toolsUsed: [],
+        toolCallCount: 0,
+      },
+      validate: { answerNotIncludes: ["built-in cobol runtime"] },
+    });
+
+    const notIncludesCheck = checks.find((entry) =>
+      entry.name.startsWith("answer does not include")
+    );
+    expect(notIncludesCheck?.passed).toBe(false);
+  });
+});

ai_evals/core/validators.ts

@@ -2,6 +2,7 @@ import path from "node:path";
 import ts from "typescript";
 import type {
   AppValidationSpec,
+  AskValidationSpec,
   BenchmarkCheck,
   CliTrace,
   CliValidationSpec,
@@ -11,6 +12,13 @@ import type {
   ToolValidationSpec,
 } from "./types";
 
+export interface AskAnswerState {
+  answer: string;
+  docsTool: string;
+  toolsUsed: string[];
+  toolCallCount: number;
+}
+
 export interface ScriptState {
   path: string;
   lang: string;
@@ -402,6 +410,51 @@ export function validateGlobalState(input: {
   return checks;
 }
 
+export function validateAskAnswer(input: {
+  actual: AskAnswerState;
+  validate?: AskValidationSpec;
+}): BenchmarkCheck[] {
+  const checks: BenchmarkCheck[] = [];
+  const answer = input.actual.answer ?? "";
+  const normalizedAnswer = answer.toLowerCase();
+
+  checks.push(check("answer is non-empty", answer.trim().length > 0));
+
+  const validate = input.validate;
+  if (!validate) {
+    return checks;
+  }
+
+  for (const group of validate.answerIncludesAny ?? []) {
+    if (group.length === 0) {
+      continue;
+    }
+    const matched = group.some((needle) =>
+      normalizedAnswer.includes(needle.toLowerCase())
+    );
+    checks.push(
+      check(
+        `answer cites one of: ${group.join(" | ")}`,
+        matched,
+        matched ? undefined : `answer: ${truncateForDetails(answer)}`
+      )
+    );
+  }
+
+  for (const needle of validate.answerNotIncludes ?? []) {
+    const present = normalizedAnswer.includes(needle.toLowerCase());
+    checks.push(
+      check(
+        `answer does not include '${needle}'`,
+        !present,
+        present ? `answer: ${truncateForDetails(answer)}` : undefined
+      )
+    );
+  }
+
+  return checks;
+}
+
 export function validateAppState(input: {
   actual: AppFilesState;
   initial?: AppFilesState;

ai_evals/modes/ask.ts

@@ -0,0 +1,90 @@
+import {
+  runAskEval,
+  resolveDocsToolVariant,
+} from "../adapters/frontend/core/ask/askEvalRunner";
+import type { FrontendEvalModelConfig } from "../core/models";
+import type {
+  AskValidationSpec,
+  BenchmarkArtifactFile,
+  ModeRunner,
+} from "../core/types";
+import { validateAskAnswer, type AskAnswerState } from "../core/validators";
+import type { WindmillBackendSettings } from "../core/windmillBackendSettings";
+import { getFrontendApiKey } from "./frontendCommon";
+
+export function createAskModeRunner(
+  modelConfig: FrontendEvalModelConfig,
+  backendSettings: WindmillBackendSettings,
+): ModeRunner<undefined, undefined, AskAnswerState> {
+  const variant = resolveDocsToolVariant();
+
+  return {
+    mode: "ask",
+    concurrency: 2,
+    judgeThreshold: 80,
+    async loadInitial() {
+      return undefined;
+    },
+    async loadExpected() {
+      return undefined;
+    },
+    async run(prompt, _initial, context) {
+      const result = await runAskEval(
+        prompt,
+        getFrontendApiKey(modelConfig.provider),
+        {
+          variant,
+          maxIterations: context.evalCase?.runtime?.maxTurns,
+          provider: modelConfig.provider,
+          model: modelConfig.model,
+          backend: backendSettings,
+          runContext: context,
+        },
+      );
+
+      return {
+        success: result.success,
+        actual: result.state,
+        error: result.error,
+        assistantMessageCount: result.assistantMessageCount,
+        toolCallCount: result.toolCallCount,
+        toolsUsed: result.toolsUsed,
+        toolCallDetails: result.toolCallDetails,
+        skillsInvoked: [],
+        tokenUsage: result.tokenUsage,
+      };
+    },
+    validate({ evalCase, actual }) {
+      return validateAskAnswer({
+        actual,
+        validate: evalCase.validate as AskValidationSpec | undefined,
+      });
+    },
+    // The judge must stay blind to which docs-tool arm produced the answer, so
+    // it only ever sees the answer text.
+    prepareJudgeActual(actual) {
+      return { answer: actual.answer };
+    },
+    buildArtifacts(actual): BenchmarkArtifactFile[] {
+      return [
+        {
+          path: "ask-answer.md",
+          content: `${actual.answer}\n`,
+        },
+        {
+          path: "ask-meta.json",
+          content:
+            JSON.stringify(
+              {
+                docsTool: actual.docsTool,
+                toolsUsed: [...new Set(actual.toolsUsed)],
+                toolCallCount: actual.toolCallCount,
+              },
+              null,
+              2,
+            ) + "\n",
+        },
+      ];
+    },
+  };
+}