feat(ai-chat): wire llms.txt docs lookup tools into global chat mode

Add list_docs_pages/read_docs_page to globalTools and a docs system-prompt section so the workspace assistant can answer product questions from windmill.dev docs and cite canonical URLs.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

ai_evals/adapters/frontend/benchmarkRunner.ts

@@ -12,7 +12,7 @@ import { resolveWindmillBackendSettings } from "../../core/windmillBackendSettin
 import { emitFrontendBenchmarkProgress } from "./progress";
 import { DEFAULT_JUDGE_MODEL } from "../../core/judge";
 
-export type FrontendBenchmarkMode = "flow" | "app" | "script" | "global";
+export type FrontendBenchmarkMode = "flow" | "app" | "script" | "global" | "ask";
 
 export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult> {
   const mode = parseMode(process.env.WMILL_FRONTEND_AI_EVAL_MODE);
@@ -42,7 +42,11 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
     backendValidation,
     backendSettings,
   );
-  const runModel = formatRunModelLabel(mode, model);
+  const docsTool =
+    mode === "ask" ? process.env.WMILL_AI_EVAL_DOCS_TOOL?.trim() || "llmstxt" : undefined;
+  const runModel = docsTool
+    ? `${formatRunModelLabel(mode, model)} ask:${docsTool}`
+    : formatRunModelLabel(mode, model);
   const caseResults = await runSuite({
     modeRunner,
     cases: selectedCases,
@@ -92,11 +96,21 @@ async function getModeRunner(
       const { createGlobalModeRunner } = await import("../../modes/global");
       return createGlobalModeRunner(model, backendSettings);
     }
+    case "ask": {
+      const { createAskModeRunner } = await import("../../modes/ask");
+      return createAskModeRunner(model, backendSettings);
+    }
   }
 }
 
 function parseMode(value: string | undefined): FrontendBenchmarkMode {
-  if (value === "flow" || value === "app" || value === "script" || value === "global") {
+  if (
+    value === "flow" ||
+    value === "app" ||
+    value === "script" ||
+    value === "global" ||
+    value === "ask"
+  ) {
     return value;
   }
   throw new Error(`Unsupported frontend benchmark mode: ${String(value)}`);

ai_evals/adapters/frontend/core/ask/askEvalRunner.ts

@@ -0,0 +1,175 @@
+import type { ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs";
+import type { AIProvider } from "$lib/gen/types.gen";
+import {
+  getAskTools,
+  prepareAskSystemMessage,
+  prepareAskUserMessage,
+  type DocsToolVariant,
+} from "../../../../../frontend/src/lib/components/copilot/chat/ask/core";
+import type { ModeRunContext } from "../../../../core/types";
+import type { AskAnswerState } from "../../../../core/validators";
+import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
+import { WindmillBackendClient } from "../../windmillBackend";
+import { runEval } from "../shared";
+import type { TokenUsage, ToolCallDetail } from "../shared/types";
+
+export interface AskEvalResult {
+  success: boolean;
+  state: AskAnswerState;
+  error?: string;
+  assistantMessageCount: number;
+  toolCallCount: number;
+  toolsUsed: string[];
+  toolCallDetails: ToolCallDetail[];
+  tokenUsage: TokenUsage;
+}
+
+export interface AskEvalOptions {
+  variant: DocsToolVariant;
+  model?: string;
+  maxIterations?: number;
+  provider?: AIProvider;
+  backend: WindmillBackendSettings;
+  runContext?: ModeRunContext;
+}
+
+const DOCS_TOOL_ENV = "WMILL_AI_EVAL_DOCS_TOOL";
+
+/**
+ * Resolves which docs-tool arm to benchmark. Defaults to the new llms.txt arm.
+ */
+export function resolveDocsToolVariant(
+  value: string | undefined = process.env[DOCS_TOOL_ENV],
+): DocsToolVariant {
+  return value === "inkeep" ? "inkeep" : "llmstxt";
+}
+
+export async function runAskEval(
+  userPrompt: string,
+  apiKey: string,
+  options: AskEvalOptions,
+): Promise<AskEvalResult> {
+  // The production inkeep tool calls fetch('/api/inkeep') with a relative URL,
+  // which has no origin under node. Install a process-wide shim that rewrites
+  // /api/* calls to the eval backend with an auth token. Only needed for the
+  // inkeep arm; the llms.txt arm fetches absolute windmill.dev URLs natively.
+  if (options.variant === "inkeep") {
+    await installInkeepFetchShim(options.backend);
+  }
+
+  const model = options.model ?? "claude-haiku-4-5-20251001";
+
+  const rawResult = await runEval({
+    userPrompt,
+    systemMessage: prepareAskSystemMessage(undefined, options.variant),
+    userMessage: prepareAskUserMessage(userPrompt),
+    tools: getAskTools(options.variant),
+    helpers: {},
+    apiKey,
+    getOutput: () => ({
+      answer: "",
+      docsTool: options.variant,
+      toolsUsed: [],
+      toolCallCount: 0,
+    }),
+    onAssistantMessageStart: options.runContext?.onAssistantMessageStart,
+    onAssistantToken: options.runContext?.onAssistantChunk,
+    onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd,
+    onToolCall: options.runContext?.onToolCall,
+    options: {
+      maxIterations: options.maxIterations,
+      model,
+      provider: options.provider,
+      backend: options.backend,
+      caseId: options.runContext?.caseId,
+      attempt: options.runContext?.attempt,
+    },
+  });
+
+  const answer = extractFinalAnswer(rawResult.messages);
+
+  return {
+    success: rawResult.success,
+    state: {
+      answer,
+      docsTool: options.variant,
+      toolsUsed: rawResult.toolsCalled,
+      toolCallCount: rawResult.toolCallsCount,
+    },
+    error: rawResult.error,
+    assistantMessageCount: rawResult.iterations,
+    toolCallCount: rawResult.toolCallsCount,
+    toolsUsed: rawResult.toolsCalled,
+    toolCallDetails: rawResult.toolCallDetails,
+    tokenUsage: rawResult.tokenUsage,
+  };
+}
+
+/**
+ * The final answer is the last assistant message with non-empty string content.
+ */
+export function extractFinalAnswer(
+  messages: ChatCompletionMessageParam[],
+): string {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const message = messages[i];
+    if (message.role !== "assistant") {
+      continue;
+    }
+    const content = message.content;
+    if (typeof content === "string" && content.trim().length > 0) {
+      return content;
+    }
+    if (Array.isArray(content)) {
+      const text = content
+        .map((part) =>
+          part && typeof part === "object" && "text" in part
+            ? String((part as { text?: unknown }).text ?? "")
+            : "",
+        )
+        .join("")
+        .trim();
+      if (text.length > 0) {
+        return text;
+      }
+    }
+  }
+  return "";
+}
+
+let inkeepFetchShimInstalled = false;
+
+/**
+ * Installs (once) a process-wide fetch wrapper that rewrites relative /api/*
+ * requests to the eval backend, adding a bearer token. All other URLs are
+ * passed through untouched.
+ */
+async function installInkeepFetchShim(
+  backend: WindmillBackendSettings,
+): Promise<void> {
+  if (inkeepFetchShimInstalled) {
+    return;
+  }
+  inkeepFetchShimInstalled = true;
+
+  const client = new WindmillBackendClient(backend);
+  const originalFetch = globalThis.fetch.bind(globalThis);
+
+  globalThis.fetch = (async (
+    input: RequestInfo | URL,
+    init?: RequestInit,
+  ): Promise<Response> => {
+    if (typeof input === "string" && input.startsWith("/api/")) {
+      const token = await client.getToken();
+      const url = `${backend.baseUrl}${input}`;
+      return await originalFetch(url, {
+        ...init,
+        headers: {
+          ...(init?.headers ?? {}),
+          Authorization: `Bearer ${token}`,
+        },
+      });
+    }
+    return await originalFetch(input as any, init);
+  }) as typeof fetch;
+}

ai_evals/adapters/frontend/progress.ts

@@ -1,4 +1,4 @@
-export type FrontendBenchmarkProgressSurface = 'flow' | 'app' | 'script' | 'global'
+export type FrontendBenchmarkProgressSurface = 'flow' | 'app' | 'script' | 'global' | 'ask'
 
 export type FrontendBenchmarkProgressEvent =
 	| {

ai_evals/adapters/frontend/runtime.ts

@@ -16,7 +16,7 @@ const FRONTEND_BENCHMARK_TEST =
 const FRONTEND_BENCHMARK_CONFIG =
   "../ai_evals/adapters/frontend/vitest.config.ts";
 
-export type FrontendMode = "flow" | "app" | "script" | "global";
+export type FrontendMode = "flow" | "app" | "script" | "global" | "ask";
 
 export async function runFrontendBenchmarkAdapter(input: {
   mode: FrontendMode;
@@ -25,6 +25,7 @@ export async function runFrontendBenchmarkAdapter(input: {
   model?: string;
   verbose?: boolean;
   backendValidation?: string;
+  docsTool?: string;
 }): Promise<BenchmarkRunResult> {
   const tempDir = await mkdtemp(
     path.join(tmpdir(), "wmill-frontend-benchmark-"),
@@ -41,6 +42,8 @@ export async function runFrontendBenchmarkAdapter(input: {
     WMILL_FRONTEND_AI_EVAL_PROGRESS: "1",
     WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? "1" : "0",
     WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION: input.backendValidation ?? "",
+    // Only meaningful for ask mode; selects the docs-tool arm to benchmark.
+    WMILL_AI_EVAL_DOCS_TOOL: input.docsTool ?? process.env.WMILL_AI_EVAL_DOCS_TOOL ?? "",
   };
 
   try {

ai_evals/adapters/frontend/vitestAdapter.test.ts

@@ -434,5 +434,6 @@ benchmarkIt(
 			resetBenchmarkMockBackend()
 		}
 	},
-	600_000
+	// Full-suite runs (30+ cases at concurrency 2-3) routinely exceed 10 minutes.
+	7_200_000
 )

ai_evals/cases/ask.yaml

@@ -0,0 +1,419 @@
+# AI-chat documentation Q&A cases. Each case asks a realistic user question and
+# is graded on answer quality (judge) plus a deterministic URL-citation check.
+# Ground-truth URLs are derived from https://www.windmill.dev/llms.txt — all
+# `answerIncludesAny` URLs are real docs pages (without the `.md` suffix).
+
+# --- Tier 1: direct lookups -------------------------------------------------
+
+- id: ask-lookup-cron-schedule
+  prompt: How do I run a script automatically every Monday at 9am?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains that recurring runs use a schedule with a CRON expression
+    - mentions a concrete cron-like expression or the schedule configuration
+    - the answer is grounded in the Windmill scheduling documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/scheduling"]
+
+- id: ask-lookup-secret-in-python
+  prompt: How do I store a secret like an API key and use it inside a Python script?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains storing the value as a secret variable
+    - explains accessing it from a Python script (e.g. wmill.get_variable or get_resource)
+    - the answer is grounded in the variables and secrets documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/variables_and_secrets"]
+
+- id: ask-lookup-webhook-trigger
+  prompt: How can I trigger one of my scripts from an external service by calling a URL?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains that scripts expose webhook URLs
+    - mentions sync vs async webhook behavior or how to find the webhook URL
+    - the answer is grounded in the webhooks documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/webhooks"]
+
+- id: ask-lookup-retry-step
+  prompt: One step in my flow calls a flaky API. How do I make it automatically retry if it fails?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains configuring retries on a flow step
+    - mentions the number of attempts and/or delay between attempts
+    - the answer is grounded in the error handling or flow editor documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/flows/retries",
+          "windmill.dev/docs/core_concepts/error_handling",
+          "windmill.dev/docs/flows/flow_editor",
+        ]
+
+- id: ask-lookup-connect-database
+  prompt: How do I connect to my Postgres database so my scripts can query it?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains creating a resource holding the database connection details
+    - explains referencing that resource from a script
+    - the answer is grounded in the resources documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/core_concepts/resources_and_types",
+          "windmill.dev/docs/getting_started/scripts_quickstart/sql",
+        ]
+
+- id: ask-lookup-python-deps
+  prompt: How do I add a third-party pip package to a Python script in Windmill?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains that imports are auto-detected and dependencies resolved automatically
+    - mentions how to pin a version or the requirements mechanism
+    - the answer is grounded in the Python scripting documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/getting_started/scripts_quickstart/python",
+          "windmill.dev/docs/advanced/dependencies_in_python",
+        ]
+
+- id: ask-lookup-concurrency-limit
+  prompt: How do I stop a script from running too many times at once so I don't hit an API rate limit?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains setting a concurrency limit on the script
+    - mentions the max number of concurrent executions and/or time window
+    - the answer is grounded in the concurrency limits documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/core_concepts/concurrency_limits",
+          "windmill.dev/docs/script_editor/concurrency_limit",
+        ]
+
+- id: ask-lookup-app-table-button
+  prompt: How do I build a simple internal tool with a table and a button using Windmill?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains using the app editor with drag-and-drop components
+    - mentions wiring a component to a backend script/runnable
+    - the answer is grounded in the app editor or apps quickstart documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/apps/app_editor",
+          "windmill.dev/docs/getting_started/apps_quickstart",
+        ]
+
+# --- Tier 2: conceptual / vocabulary-mismatch -------------------------------
+
+- id: ask-concept-flow-wait-for-approval
+  prompt: How can I make a flow pause and wait for a person to approve before it continues?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies this as a suspend / approval step in a flow
+    - explains that the flow resumes once approved (e.g. via a resume link or form)
+    - the answer is grounded in the flow editor / approval documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/flows/flow_editor",
+          "windmill.dev/docs/flows/flow_approval",
+        ]
+
+- id: ask-concept-remember-value-between-runs
+  prompt: Can a script remember a value from its previous run, like the last timestamp it processed?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies states as the mechanism (getState / setState)
+    - explains state persists between runs of the same script
+    - the answer is grounded in the states / within-windmill documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/persistent_storage/within_windmill"]
+
+- id: ask-concept-steps-at-same-time
+  prompt: How do I make several steps in my flow run at the same time instead of one after another?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies running branches in parallel (branchall / parallel branches)
+    - explains the steps execute concurrently
+    - the answer is grounded in the flow editor documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/flows/flow_editor",
+          "windmill.dev/docs/flows/flow_branches",
+        ]
+
+- id: ask-concept-react-to-db-changes
+  prompt: I want something to happen automatically whenever a new row is inserted into my Postgres table. Is that possible?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies Postgres triggers reacting to insert/update/delete
+    - mentions it listens to database change events (logical replication)
+    - the answer is grounded in the Postgres triggers documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/core_concepts/postgres_triggers",
+          "windmill.dev/docs/getting_started/triggers",
+        ]
+
+- id: ask-concept-reuse-config-everywhere
+  prompt: I keep pasting the same base URL and credentials into many scripts. Is there a cleaner way to manage that shared config?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies variables (and/or resources) as the mechanism for reusable config
+    - explains that the config is defined once and referenced from scripts
+    - the answer is grounded in the variables / resources documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/core_concepts/variables_and_secrets",
+          "windmill.dev/docs/core_concepts/resources_and_types",
+        ]
+
+- id: ask-concept-let-llm-call-my-scripts
+  prompt: I want my AI assistant in Claude or Cursor to be able to run my Windmill scripts. How would I set that up?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies MCP (Model Context Protocol) as the mechanism
+    - explains connecting an MCP client to Windmill to trigger scripts/flows
+    - the answer is grounded in the MCP documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/mcp"]
+
+- id: ask-concept-avoid-recompute
+  prompt: My script does an expensive computation with the same inputs a lot. Can Windmill avoid recomputing the same result every time?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies caching of script/flow results
+    - explains that identical inputs reuse the cached result for a configured duration
+    - the answer is grounded in the caching documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/caching"]
+
+- id: ask-concept-agent-step
+  prompt: I want a flow step where an LLM decides which of my scripts to call based on the input. Does Windmill support that?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies AI agent steps in a flow
+    - explains the agent can call tools/scripts based on the input
+    - the answer is grounded in the AI agents documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/ai_agents"]
+
+# --- Tier 3: multi-page synthesis -------------------------------------------
+
+- id: ask-synthesis-variable-vs-resource
+  prompt: What's the difference between a variable and a resource in Windmill, and when should I use each?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains a variable holds a single reusable value (and secrets for sensitive ones)
+    - explains a resource holds a structured connection object (e.g. database/API credentials)
+    - gives reasonable guidance on when to use each
+    - the answer is grounded in the variables and resources documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/variables_and_secrets"]
+      - ["windmill.dev/docs/core_concepts/resources_and_types"]
+
+- id: ask-synthesis-trigger-options
+  prompt: What are all the different ways I can trigger a flow — both on a schedule and from external events?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - covers scheduled/cron triggering
+    - covers event-based triggering (e.g. webhooks, HTTP routes, or message queues)
+    - the answer is grounded in the triggers / scheduling documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/getting_started/triggers",
+          "windmill.dev/docs/core_concepts/scheduling",
+        ]
+
+- id: ask-synthesis-storage-options
+  prompt: Where should I store data in Windmill? I'm confused about all the storage options.
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - distinguishes object storage (S3) from relational/structured storage and lightweight state/KV options
+    - gives reasonable guidance on which to use when
+    - the answer is grounded in the persistent storage documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/persistent_storage"]
+
+- id: ask-synthesis-deploy-to-prod
+  prompt: I'm building in a dev workspace and want a safe way to promote my scripts and flows to production. What's the recommended workflow?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - describes promoting from staging/dev to prod and/or git-based deployment
+    - mentions draft/deploy and/or git sync as part of the workflow
+    - the answer is grounded in the deploy-to-prod / staging-prod documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/advanced/deploy_to_prod",
+          "windmill.dev/docs/advanced/deploy_gh_gl",
+          "windmill.dev/docs/advanced/canonical_deployment_setups",
+          "windmill.dev/docs/core_concepts/staging_prod",
+          "windmill.dev/docs/advanced/git_sync",
+        ]
+
+- id: ask-synthesis-handle-failures
+  prompt: What are my options for dealing with steps that fail in a flow — both retrying and being notified when something breaks?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - covers retries on failing steps
+    - covers error handlers / failure notification
+    - the answer is grounded in the error handling documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/error_handling"]
+
+- id: ask-synthesis-etl-pipeline
+  prompt: I want to build an ETL pipeline that pulls data, transforms it, and writes results to S3. How does Windmill support that?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - describes building a DAG/flow of steps for extract-transform-load
+    - mentions S3 / object storage integration for the data
+    - the answer is grounded in the data pipelines / object storage documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/core_concepts/data_pipelines",
+          "windmill.dev/docs/core_concepts/object_storage_in_windmill",
+        ]
+
+# --- Tier 4: niche precise facts --------------------------------------------
+
+- id: ask-niche-worker-tags-env
+  prompt: In the self-hosted community edition, how do I make a specific worker only pick up certain jobs?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains assigning tags to jobs and to workers
+    - mentions the WORKER_TAGS environment variable used to configure a worker's tags
+    - the answer is grounded in the worker groups documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/worker_groups"]
+
+- id: ask-niche-s3-object-storage
+  prompt: How do I connect my workspace to S3 so scripts can read and write large files there?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains configuring an S3 / object storage connection for the workspace
+    - mentions reading/writing files via the workspace object storage
+    - the answer is grounded in the object storage documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/object_storage_in_windmill"]
+
+- id: ask-niche-data-tables
+  prompt: Does Windmill have a built-in way to store relational data without me setting up my own Postgres database?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - identifies Windmill data tables as the built-in relational storage
+    - explains storing and querying relational data without an external database
+    - the answer is grounded in the data tables documentation
+  validate:
+    answerIncludesAny:
+      - [
+          "windmill.dev/docs/core_concepts/persistent_storage/data_tables",
+          "windmill.dev/docs/core_concepts/persistent_storage",
+        ]
+
+- id: ask-niche-key-value-store
+  prompt: Can I use something like Redis or a key-value store from my Windmill scripts?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - confirms key-value / NoSQL stores (e.g. Redis, MongoDB, Upstash) are supported via resources
+    - explains connecting and using them from scripts
+    - the answer is grounded in the key value stores documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/persistent_storage/key_value_stores"]
+
+- id: ask-niche-secret-encryption-key
+  prompt: How are my workspace secrets encrypted at rest, and can I rotate the encryption key?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - explains workspace secrets are encrypted with a workspace encryption key
+    - explains the key can be updated/rotated
+    - the answer is grounded in the workspace secret encryption documentation
+  validate:
+    answerIncludesAny:
+      - ["windmill.dev/docs/core_concepts/workspace_secret_encryption"]
+
+# --- Tier 5: out-of-scope controls ------------------------------------------
+
+- id: ask-nodocs-cobol-runtime
+  prompt: Does Windmill have a built-in COBOL runtime for running COBOL scripts directly?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - does NOT claim Windmill has a built-in COBOL runtime
+    - says it is not sure / not a supported language rather than fabricating one
+    - optionally suggests a workaround (e.g. Docker/Bash) or asking the Windmill team
+  validate:
+    answerNotIncludes:
+      - "Windmill has a built-in COBOL runtime"
+
+- id: ask-nodocs-onchain-payments
+  prompt: Can Windmill settle blockchain cryptocurrency payments on-chain natively as a built-in feature?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - does NOT claim Windmill has a native on-chain crypto payment feature
+    - says it is not sure / not a documented feature rather than fabricating one
+    - optionally suggests doing it in a normal script or asking the Windmill team
+  validate:
+    answerNotIncludes:
+      - "Windmill natively settles"
+
+- id: ask-nodocs-voice-assistant
+  prompt: Does Windmill ship a built-in voice assistant that I can talk to with my microphone to run flows by voice?
+  runtime:
+    maxTurns: 10
+  judgeChecklist:
+    - does NOT claim Windmill ships a built-in microphone voice assistant
+    - says it is not sure / not a documented feature rather than fabricating one
+    - optionally points to real trigger mechanisms or asking the Windmill team
+  validate:
+    answerNotIncludes:
+      - "Windmill ships a built-in voice assistant"

ai_evals/cases/global.yaml

@@ -870,3 +870,27 @@
   judgeChecklist:
     - fetches the logs for the requested job id
     - explains the failure from the returned logs (connection refused to the upstream API)
+
+- id: global-test29-docs-lookup-scheduling
+  prompt: |-
+    How do I schedule a script to run every Monday at 9am in Windmill?
+  runtime:
+    maxTurns: 6
+  validate:
+    draftCountExactly: 0
+  toolExpect:
+    requiredToolsUsed:
+      - list_docs_pages
+      - read_docs_page
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+      - delete_workspace_item
+      - write_script
+  # Answering a product-docs question produces no draft, so the global judge
+  # (which only sees the drafts artifact) would score it empty — validate the
+  # docs-lookup contract via tool use: browse the index first, then read a page.
+  skipJudge: true
+  judgeChecklist:
+    - lists the documentation pages before reading one
+    - reads the scheduling documentation page rather than answering from memory
+    - cites the canonical windmill.dev/docs scheduling URL in the answer

ai_evals/cli/index.ts

@@ -54,6 +54,8 @@ async function main() {
         "  bun run cli -- run flow --backend-validation preview",
         "  bun run cli -- run flow flow-test5-simple-modification --runs 3",
         "  bun run cli -- run global global-test1-script-create",
+        "  bun run cli -- run ask --docs-tool llmstxt",
+        "  bun run cli -- run ask ask-lookup-cron-schedule --docs-tool inkeep",
         "  bun run cli -- run cli bun-hello-script",
         "",
         "Models:",
@@ -71,7 +73,7 @@ async function main() {
   program
     .command("cases")
     .description("List available cases")
-    .argument("[mode]", "cli, flow, script, app, or global", parseOptionalMode)
+    .argument("[mode]", "cli, flow, script, app, global, or ask", parseOptionalMode)
     .action(async (mode?: EvalMode) => {
       await handleCases(mode);
     });
@@ -79,7 +81,7 @@ async function main() {
   program
     .command("run")
     .description("Run one benchmark mode")
-    .argument("<mode>", "cli, flow, script, app, or global", parseMode)
+    .argument("<mode>", "cli, flow, script, app, global, or ask", parseMode)
     .argument("[caseIds...]", "specific case ids to run")
     .option(
       "--runs <n>",
@@ -105,6 +107,11 @@ async function main() {
       "--backend-validation <mode>",
       `backend smoke validation (${BACKEND_VALIDATION_MODES.join(", ")})`,
     )
+    .option(
+      "--docs-tool <arm>",
+      "docs-tool arm for ask mode (inkeep, llmstxt)",
+      parseDocsTool,
+    )
     .action(
       async (
         mode: EvalMode,
@@ -117,6 +124,7 @@ async function main() {
           verbose?: boolean;
           record?: boolean;
           backendValidation?: string;
+          docsTool?: string;
         },
       ) => {
         await handleRun({
@@ -129,6 +137,7 @@ async function main() {
           verbose: options.verbose ?? false,
           record: options.record ?? false,
           backendValidation: options.backendValidation,
+          docsTool: options.docsTool,
         });
       },
     );
@@ -153,7 +162,7 @@ function handleModels() {
   process.stdout.write("Available models\n");
   for (const model of EVAL_MODELS) {
     const supports = [
-      ...(model.frontend ? ["flow", "script", "app", "global"] : []),
+      ...(model.frontend ? ["flow", "script", "app", "global", "ask"] : []),
       ...(model.cli ? ["cli"] : []),
     ];
     const aliases = [
@@ -177,6 +186,7 @@ async function handleRun(input: {
   verbose: boolean;
   record: boolean;
   backendValidation?: string;
+  docsTool?: string;
 }) {
   if (input.record && input.caseIds.length > 0) {
     throw new Error(
@@ -187,6 +197,9 @@ async function handleRun(input: {
     throw new Error("Use either --model or --models, not both");
   }
 
+  // The docs-tool arm only applies to ask mode; default to the new llms.txt arm.
+  const docsTool = input.mode === "ask" ? input.docsTool ?? "llmstxt" : undefined;
+
   const selectedCases = await loadSelectedCases(input.mode, input.caseIds);
   const models = resolveRequestedModels(input.mode, input.model, input.models);
   const backendValidation = parseBackendValidationMode(
@@ -215,13 +228,17 @@ async function handleRun(input: {
   }> = [];
 
   for (const [index, model] of models.entries()) {
-    const runModel = formatRunModelLabel(input.mode, model);
+    const baseRunModel = formatRunModelLabel(input.mode, model);
+    // Distinguish saved results / history by the docs-tool arm.
+    const runModel = docsTool ? `${baseRunModel} ask:${docsTool}` : baseRunModel;
     if (models.length > 1) {
       process.stdout.write(
         `${index > 0 ? "\n" : ""}=== ${input.mode} ${model.id} (${runModel}) ===\n`,
       );
     }
-    process.stderr.write(`Starting ${input.mode} benchmark...\n`);
+    process.stderr.write(
+      `Starting ${input.mode} benchmark${docsTool ? ` (docs-tool: ${docsTool})` : ""}...\n`,
+    );
 
     const result =
       input.mode === "cli"
@@ -238,6 +255,7 @@ async function handleRun(input: {
             model: model.id,
             verbose: input.verbose,
             backendValidation,
+            docsTool,
           });
 
     const resolvedOutputPath =
@@ -317,6 +335,13 @@ function parsePositiveInteger(value: string): number {
   return parsed;
 }
 
+function parseDocsTool(value: string): string {
+  if (value === "inkeep" || value === "llmstxt") {
+    return value;
+  }
+  throw new InvalidArgumentError("docs-tool must be one of: inkeep, llmstxt");
+}
+
 function resolveRequestedModels(
   mode: EvalMode,
   singleModel?: string,

ai_evals/core/cases.test.ts

@@ -246,6 +246,35 @@ describe("loadCases", () => {
     });
   });
 
+  it("loads ask docs Q&A cases with citation validation", async () => {
+    const askCases = await loadCases("ask");
+    expect(askCases.length).toBeGreaterThanOrEqual(25);
+
+    const lookupCase = askCases.find((entry) => entry.id === "ask-lookup-cron-schedule");
+    expect(lookupCase?.runtime).toEqual({ maxTurns: 10 });
+    expect(lookupCase?.validate).toEqual({
+      answerIncludesAny: [["windmill.dev/docs/core_concepts/scheduling"]],
+    });
+    expect(lookupCase?.judgeChecklist?.length).toBeGreaterThan(0);
+
+    const noDocsCase = askCases.find((entry) => entry.id === "ask-nodocs-cobol-runtime");
+    expect(noDocsCase?.validate).toEqual({
+      answerNotIncludes: ["Windmill has a built-in COBOL runtime"],
+    });
+
+    // Every case must cap turns and either cite docs or assert a forbidden claim.
+    for (const entry of askCases) {
+      expect(entry.runtime?.maxTurns).toBe(10);
+      const validate = entry.validate as
+        | { answerIncludesAny?: string[][]; answerNotIncludes?: string[] }
+        | undefined;
+      expect(
+        (validate?.answerIncludesAny?.length ?? 0) > 0 ||
+          (validate?.answerNotIncludes?.length ?? 0) > 0,
+      ).toBe(true);
+    }
+  });
+
   it("loads tool expectations for workspace mutation cases", async () => {
     const scriptCases = await loadCases("script");
     const caseEntry = scriptCases.find(

ai_evals/core/models.test.ts

@@ -48,4 +48,11 @@ describe("resolveEvalModel", () => {
       "Model gemini-3-flash-preview is not supported for cli mode",
     );
   });
+
+  it("resolves frontend models for ask mode", () => {
+    expect(resolveEvalModel("ask", "sonnet").frontend).toEqual({
+      provider: "anthropic",
+      model: "claude-sonnet-4-5-20250929",
+    });
+  });
 });

ai_evals/core/models.ts

@@ -161,7 +161,7 @@ export function resolveEvalModel(
 export function getEvalModelHelpText(): string {
   return EVAL_MODELS.map((model) => {
     const modes = [
-      ...(model.frontend ? ["flow", "script", "app", "global"] : []),
+      ...(model.frontend ? ["flow", "script", "app", "global", "ask"] : []),
       ...(model.cli ? ["cli"] : []),
     ];
     return `  ${model.id.padEnd(8)} ${model.label} (${modes.join(", ")})`;

ai_evals/core/runSuite.ts

@@ -225,7 +225,9 @@ async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
           checklist: input.evalCase.judgeChecklist,
           initial,
           expected: input.modeRunner.mode === "cli" ? undefined : expected,
-          actual: run.actual,
+          actual: input.modeRunner.prepareJudgeActual
+            ? input.modeRunner.prepareJudgeActual(run.actual)
+            : run.actual,
           model: input.judgeModel,
         });
 

ai_evals/core/types.ts

@@ -1,4 +1,4 @@
-export const EVAL_MODES = ["cli", "flow", "script", "app", "global"] as const;
+export const EVAL_MODES = ["cli", "flow", "script", "app", "global", "ask"] as const;
 
 export type EvalMode = (typeof EVAL_MODES)[number];
 
@@ -131,6 +131,18 @@ export interface GlobalValidationSpec {
   }>;
 }
 
+export interface AskValidationSpec {
+  /**
+   * URL-citation / required-mention check. A list of groups; each group passes
+   * if ANY of its alternative substrings appears in the answer
+   * (case-insensitive). Used where several documentation URLs are acceptable
+   * answers to the same question.
+   */
+  answerIncludesAny?: string[][];
+  /** Substrings that must NOT appear in the answer (case-insensitive). */
+  answerNotIncludes?: string[];
+}
+
 export interface CliValidationSpec {
   requiredSkills?: string[];
   forbiddenSkills?: string[];
@@ -172,7 +184,11 @@ export interface ToolValidationSpec {
   toolCallArgs?: ToolCallArgumentRule[];
 }
 
-export type EvalValidationSpec = FlowValidationSpec | AppValidationSpec | GlobalValidationSpec;
+export type EvalValidationSpec =
+  | FlowValidationSpec
+  | AppValidationSpec
+  | GlobalValidationSpec
+  | AskValidationSpec;
 
 export interface EvalCase {
   id: string;
@@ -294,6 +310,12 @@ export interface ModeRunner<TInitial, TExpected, TActual> {
     context: ModeRunContext;
   }): Promise<BackendValidationResult | null>;
   buildArtifacts?(actual: TActual): BenchmarkArtifactFile[];
+  /**
+   * Optional transform applied to `actual` before it is handed to the LLM judge.
+   * Use it to strip fields the judge must stay blind to (e.g. which docs-tool
+   * arm produced an answer). When omitted, the judge receives `actual` as-is.
+   */
+  prepareJudgeActual?(actual: TActual): unknown;
 }
 
 export interface BenchmarkAttemptResult {

ai_evals/core/validators.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it } from "bun:test";
 import {
   validateAppState,
+  validateAskAnswer,
   validateCliWorkspace,
   validateGlobalState,
   validateScriptState,
@@ -941,3 +942,76 @@ describe("validateCliWorkspace", () => {
     });
   });
 });
+
+describe("validateAskAnswer", () => {
+  it("flags an empty answer", () => {
+    const checks = validateAskAnswer({
+      actual: { answer: "   ", docsTool: "llmstxt", toolsUsed: [], toolCallCount: 0 },
+    });
+    expect(checks).toContainEqual({
+      name: "answer is non-empty",
+      passed: false,
+    });
+  });
+
+  it("passes a citation group when any alternative URL appears (case-insensitive)", () => {
+    const checks = validateAskAnswer({
+      actual: {
+        answer:
+          "You can schedule scripts with cron. See HTTPS://WWW.WINDMILL.DEV/DOCS/CORE_CONCEPTS/SCHEDULING for details.",
+        docsTool: "llmstxt",
+        toolsUsed: ["list_docs_pages", "read_docs_page"],
+        toolCallCount: 2,
+      },
+      validate: {
+        answerIncludesAny: [
+          [
+            "windmill.dev/docs/core_concepts/scheduling",
+            "windmill.dev/docs/getting_started/triggers",
+          ],
+        ],
+      },
+    });
+
+    const citationCheck = checks.find((entry) =>
+      entry.name.startsWith("answer cites one of:")
+    );
+    expect(citationCheck?.passed).toBe(true);
+  });
+
+  it("fails a citation group when none of the alternatives appear", () => {
+    const checks = validateAskAnswer({
+      actual: {
+        answer: "Use schedules to run scripts on a cron.",
+        docsTool: "inkeep",
+        toolsUsed: ["get_documentation"],
+        toolCallCount: 1,
+      },
+      validate: {
+        answerIncludesAny: [["windmill.dev/docs/core_concepts/scheduling"]],
+      },
+    });
+
+    const citationCheck = checks.find((entry) =>
+      entry.name.startsWith("answer cites one of:")
+    );
+    expect(citationCheck?.passed).toBe(false);
+  });
+
+  it("enforces answerNotIncludes", () => {
+    const checks = validateAskAnswer({
+      actual: {
+        answer: "Windmill has a built-in COBOL runtime.",
+        docsTool: "llmstxt",
+        toolsUsed: [],
+        toolCallCount: 0,
+      },
+      validate: { answerNotIncludes: ["built-in cobol runtime"] },
+    });
+
+    const notIncludesCheck = checks.find((entry) =>
+      entry.name.startsWith("answer does not include")
+    );
+    expect(notIncludesCheck?.passed).toBe(false);
+  });
+});

ai_evals/core/validators.ts

@@ -2,6 +2,7 @@ import path from "node:path";
 import ts from "typescript";
 import type {
   AppValidationSpec,
+  AskValidationSpec,
   BenchmarkCheck,
   CliTrace,
   CliValidationSpec,
@@ -11,6 +12,13 @@ import type {
   ToolValidationSpec,
 } from "./types";
 
+export interface AskAnswerState {
+  answer: string;
+  docsTool: string;
+  toolsUsed: string[];
+  toolCallCount: number;
+}
+
 export interface ScriptState {
   path: string;
   lang: string;
@@ -402,6 +410,51 @@ export function validateGlobalState(input: {
   return checks;
 }
 
+export function validateAskAnswer(input: {
+  actual: AskAnswerState;
+  validate?: AskValidationSpec;
+}): BenchmarkCheck[] {
+  const checks: BenchmarkCheck[] = [];
+  const answer = input.actual.answer ?? "";
+  const normalizedAnswer = answer.toLowerCase();
+
+  checks.push(check("answer is non-empty", answer.trim().length > 0));
+
+  const validate = input.validate;
+  if (!validate) {
+    return checks;
+  }
+
+  for (const group of validate.answerIncludesAny ?? []) {
+    if (group.length === 0) {
+      continue;
+    }
+    const matched = group.some((needle) =>
+      normalizedAnswer.includes(needle.toLowerCase())
+    );
+    checks.push(
+      check(
+        `answer cites one of: ${group.join(" | ")}`,
+        matched,
+        matched ? undefined : `answer: ${truncateForDetails(answer)}`
+      )
+    );
+  }
+
+  for (const needle of validate.answerNotIncludes ?? []) {
+    const present = normalizedAnswer.includes(needle.toLowerCase());
+    checks.push(
+      check(
+        `answer does not include '${needle}'`,
+        !present,
+        present ? `answer: ${truncateForDetails(answer)}` : undefined
+      )
+    );
+  }
+
+  return checks;
+}
+
 export function validateAppState(input: {
   actual: AppFilesState;
   initial?: AppFilesState;

ai_evals/modes/ask.ts

@@ -0,0 +1,90 @@
+import {
+  runAskEval,
+  resolveDocsToolVariant,
+} from "../adapters/frontend/core/ask/askEvalRunner";
+import type { FrontendEvalModelConfig } from "../core/models";
+import type {
+  AskValidationSpec,
+  BenchmarkArtifactFile,
+  ModeRunner,
+} from "../core/types";
+import { validateAskAnswer, type AskAnswerState } from "../core/validators";
+import type { WindmillBackendSettings } from "../core/windmillBackendSettings";
+import { getFrontendApiKey } from "./frontendCommon";
+
+export function createAskModeRunner(
+  modelConfig: FrontendEvalModelConfig,
+  backendSettings: WindmillBackendSettings,
+): ModeRunner<undefined, undefined, AskAnswerState> {
+  const variant = resolveDocsToolVariant();
+
+  return {
+    mode: "ask",
+    concurrency: 2,
+    judgeThreshold: 80,
+    async loadInitial() {
+      return undefined;
+    },
+    async loadExpected() {
+      return undefined;
+    },
+    async run(prompt, _initial, context) {
+      const result = await runAskEval(
+        prompt,
+        getFrontendApiKey(modelConfig.provider),
+        {
+          variant,
+          maxIterations: context.evalCase?.runtime?.maxTurns,
+          provider: modelConfig.provider,
+          model: modelConfig.model,
+          backend: backendSettings,
+          runContext: context,
+        },
+      );
+
+      return {
+        success: result.success,
+        actual: result.state,
+        error: result.error,
+        assistantMessageCount: result.assistantMessageCount,
+        toolCallCount: result.toolCallCount,
+        toolsUsed: result.toolsUsed,
+        toolCallDetails: result.toolCallDetails,
+        skillsInvoked: [],
+        tokenUsage: result.tokenUsage,
+      };
+    },
+    validate({ evalCase, actual }) {
+      return validateAskAnswer({
+        actual,
+        validate: evalCase.validate as AskValidationSpec | undefined,
+      });
+    },
+    // The judge must stay blind to which docs-tool arm produced the answer, so
+    // it only ever sees the answer text.
+    prepareJudgeActual(actual) {
+      return { answer: actual.answer };
+    },
+    buildArtifacts(actual): BenchmarkArtifactFile[] {
+      return [
+        {
+          path: "ask-answer.md",
+          content: `${actual.answer}\n`,
+        },
+        {
+          path: "ask-meta.json",
+          content:
+            JSON.stringify(
+              {
+                docsTool: actual.docsTool,
+                toolsUsed: [...new Set(actual.toolsUsed)],
+                toolCallCount: actual.toolCallCount,
+              },
+              null,
+              2,
+            ) + "\n",
+        },
+      ];
+    },
+  };
+}

frontend/src/lib/components/copilot/chat/ask/core.ts

@@ -4,29 +4,64 @@ import type {
 } from 'openai/resources/index.mjs'
 import type { Tool } from '../shared'
 import { getDocumentationTool } from '../navigator/core'
+import { listDocsPagesTool, readDocsPageTool } from '../docs/core'
 
-export const CHAT_SYSTEM_PROMPT = `
-You are Windmill's intelligent assistant, designed to answer questions about its functionality. It is your only purpose to help the user in the context of the windmill application.
-Windmill is an open-source developer platform for building internal tools, API integrations, background jobs, workflows, and user interfaces. It offers a unified system where scripts are automatically turned into sharable UIs and can be composed into flows or embedded in custom applications.
+export type DocsToolVariant = 'inkeep' | 'llmstxt'
 
-You have access to these tools:
+const CHAT_SYSTEM_PROMPT_INTRO = `You are Windmill's intelligent assistant, designed to answer questions about its functionality. It is your only purpose to help the user in the context of the windmill application.
+Windmill is an open-source developer platform for building internal tools, API integrations, background jobs, workflows, and user interfaces. It offers a unified system where scripts are automatically turned into sharable UIs and can be composed into flows or embedded in custom applications.`
+
+const CHAT_SYSTEM_PROMPT_PRINCIPLES = `GENERAL PRINCIPLES:
+- Be concise but thorough
+- Maintain a friendly, professional tone
+- If you encounter an error or can't complete a request, explain why and suggest alternatives`
+
+const INKEEP_TOOL_SECTION = `You have access to these tools:
 1. Get documentation for user requests (get_documentation)
 
 INSTRUCTIONS:
 - When user asks about something, use the get_documentation tool to retrieve accurate information about how to fulfill the user's request.
 - Complete your response with precisions about how it works based on the documentation. Also drop a link to the relevant documentation if possible.
-- If the user asks about something that you are unsure about, say that you are not sure about the answer and suggest to ask the question to the windmill team.
+- If the user asks about something that you are unsure about, say that you are not sure about the answer and suggest to ask the question to the windmill team.`
 
-GENERAL PRINCIPLES:
-- Be concise but thorough
-- Maintain a friendly, professional tone
-- If you encounter an error or can't complete a request, explain why and suggest alternatives
+const LLMSTXT_TOOL_SECTION = `You have access to these tools:
+1. List all documentation pages (list_docs_pages)
+2. Read a documentation page (read_docs_page)
+
+INSTRUCTIONS:
+- Always call list_docs_pages FIRST to see the full index of available documentation pages with their titles, URLs and descriptions.
+- Pick the 1 to 3 pages most relevant to the user's question, then call read_docs_page on each one. If read_docs_page returns a list of section headings instead of the full page, call it again with the same path and a \`section\` argument to read the relevant section.
+- Answer based ONLY on what you read from the documentation. Do not invent features, flags, syntax, or behavior that you did not see in the docs.
+- Always include the documentation URL(s) you consulted in your answer so the user can read more. Cite the exact "Source page" URL shown at the top of each page you read — never reconstruct a URL from a link inside the page body.
+- If the documentation does not cover the user's question, say so clearly rather than inventing an answer, and suggest asking the Windmill team.`
+
+export const CHAT_SYSTEM_PROMPT = buildAskSystemPrompt('inkeep')
+
+function buildAskSystemPrompt(variant: DocsToolVariant): string {
+	const toolSection = variant === 'llmstxt' ? LLMSTXT_TOOL_SECTION : INKEEP_TOOL_SECTION
+	return `
+${CHAT_SYSTEM_PROMPT_INTRO}
+
+${toolSection}
+
+${CHAT_SYSTEM_PROMPT_PRINCIPLES}
 `
+}
 
-export const askTools: Tool<{}>[] = [getDocumentationTool]
+export function getAskTools(variant: DocsToolVariant = 'inkeep'): Tool<{}>[] {
+	if (variant === 'llmstxt') {
+		return [listDocsPagesTool, readDocsPageTool]
+	}
+	return [getDocumentationTool]
+}
 
-export function prepareAskSystemMessage(customPrompt?: string): ChatCompletionSystemMessageParam {
-	let content = CHAT_SYSTEM_PROMPT
+export const askTools: Tool<{}>[] = getAskTools('inkeep')
+
+export function prepareAskSystemMessage(
+	customPrompt?: string,
+	variant: DocsToolVariant = 'inkeep'
+): ChatCompletionSystemMessageParam {
+	let content = buildAskSystemPrompt(variant)
 
 	// If there's a custom prompt, append it to the system prompt
 	if (customPrompt?.trim()) {

frontend/src/lib/components/copilot/chat/docs/core.test.ts

@@ -0,0 +1,245 @@
+import { describe, expect, it } from 'vitest'
+import {
+	buildDocsOutline,
+	canonicalDocsPageUrl,
+	extractDocsSection,
+	normalizeDocsUrl,
+	parseDocsHeadings,
+	renderDocsPageResult,
+	sanitizeDocsMarkdownLinks
+} from './core'
+
+const SAMPLE = `# Jobs
+
+Intro text about jobs.
+
+## Job kinds
+
+Some kinds.
+
+## Result
+
+### Result of jobs that failed
+
+\`\`\`
+{ "error": "boom" }
+\`\`\`
+
+### Result streaming
+
+#### Returning a stream directly
+
+\`\`\`python
+# Returning a stream directly is a comment heading that must be ignored
+def main():
+    pass
+\`\`\`
+
+## Retention policy
+
+Final section.
+`
+
+describe('parseDocsHeadings', () => {
+	it('parses headings with their levels and ignores headings inside fenced code blocks', () => {
+		const headings = parseDocsHeadings(SAMPLE)
+		const titles = headings.map((h) => `${h.level}:${h.title}`)
+
+		expect(titles).toEqual([
+			'1:Jobs',
+			'2:Job kinds',
+			'2:Result',
+			'3:Result of jobs that failed',
+			'3:Result streaming',
+			'4:Returning a stream directly',
+			'2:Retention policy'
+		])
+		// The "# Returning a stream directly is a comment..." line inside the
+		// python fence must not be parsed as a heading.
+		expect(titles).not.toContain('1:Returning a stream directly is a comment heading that must be ignored')
+	})
+
+	it('returns startIndex offsets that point at the heading line', () => {
+		const headings = parseDocsHeadings(SAMPLE)
+		for (const heading of headings) {
+			expect(SAMPLE.slice(heading.startIndex)).toMatch(
+				new RegExp(`^#{${heading.level}}\\s+${heading.title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}`)
+			)
+		}
+	})
+
+	it('handles tilde fences', () => {
+		const content = '# Title\n\n~~~\n# not a heading\n~~~\n\n## Real\n'
+		const headings = parseDocsHeadings(content)
+		expect(headings.map((h) => h.title)).toEqual(['Title', 'Real'])
+	})
+})
+
+describe('extractDocsSection', () => {
+	it('extracts a section from its heading up to the next same-or-higher level heading', () => {
+		const section = extractDocsSection(SAMPLE, 'Result')
+		expect(section).toBeDefined()
+		expect(section).toContain('## Result')
+		expect(section).toContain('### Result of jobs that failed')
+		expect(section).toContain('### Result streaming')
+		// Stops before the next level-2 heading.
+		expect(section).not.toContain('## Retention policy')
+	})
+
+	it('matches case-insensitively and tolerates punctuation differences', () => {
+		const section = extractDocsSection(SAMPLE, 'retention-policy!')
+		expect(section).toBeDefined()
+		expect(section).toContain('## Retention policy')
+		expect(section).toContain('Final section.')
+	})
+
+	it('returns the deepest section bounded by the next same-level heading', () => {
+		const section = extractDocsSection(SAMPLE, 'Result streaming')
+		expect(section).toBeDefined()
+		expect(section).toContain('### Result streaming')
+		expect(section).toContain('#### Returning a stream directly')
+		expect(section).not.toContain('## Retention policy')
+	})
+
+	it('returns undefined when no heading matches', () => {
+		expect(extractDocsSection(SAMPLE, 'Nonexistent section')).toBeUndefined()
+	})
+})
+
+describe('buildDocsOutline', () => {
+	it('lists headings with approximate per-section sizes and indentation', () => {
+		const outline = buildDocsOutline(SAMPLE)
+		expect(outline).toContain('- Jobs (~')
+		expect(outline).toContain('  - Job kinds (~')
+		expect(outline).toContain('    - Result of jobs that failed (~')
+	})
+
+	it('handles pages with no headings', () => {
+		expect(buildDocsOutline('just some text\nwith no headings')).toBe(
+			'(no markdown headings found on this page)'
+		)
+	})
+})
+
+describe('normalizeDocsUrl', () => {
+	it('appends .md to a bare path', () => {
+		expect(normalizeDocsUrl('/docs/core_concepts/jobs')).toBe(
+			'https://www.windmill.dev/docs/core_concepts/jobs.md'
+		)
+	})
+
+	it('accepts a path without a leading slash', () => {
+		expect(normalizeDocsUrl('docs/core_concepts/jobs')).toBe(
+			'https://www.windmill.dev/docs/core_concepts/jobs.md'
+		)
+	})
+
+	it('accepts a full URL and strips anchors and query strings', () => {
+		expect(
+			normalizeDocsUrl('https://www.windmill.dev/docs/core_concepts/jobs#result?foo=bar')
+		).toBe('https://www.windmill.dev/docs/core_concepts/jobs.md')
+	})
+
+	it('does not double-append .md', () => {
+		expect(normalizeDocsUrl('/docs/core_concepts/jobs.md')).toBe(
+			'https://www.windmill.dev/docs/core_concepts/jobs.md'
+		)
+	})
+
+	it('strips a trailing slash before appending .md', () => {
+		expect(normalizeDocsUrl('/docs/core_concepts/jobs/')).toBe(
+			'https://www.windmill.dev/docs/core_concepts/jobs.md'
+		)
+	})
+
+	it('strips docusaurus numeric ordering prefixes from path segments', () => {
+		expect(normalizeDocsUrl('/docs/flows/13_flow_branches')).toBe(
+			'https://www.windmill.dev/docs/flows/flow_branches.md'
+		)
+	})
+
+	it('converts a .mdx source suffix to .md', () => {
+		expect(normalizeDocsUrl('/docs/flows/13_flow_branches.mdx')).toBe(
+			'https://www.windmill.dev/docs/flows/flow_branches.md'
+		)
+	})
+})
+
+describe('sanitizeDocsMarkdownLinks', () => {
+	const PAGE = 'https://www.windmill.dev/docs/flows/flow_editor.md'
+
+	it('rewrites a relative .mdx source link to a canonical published URL', () => {
+		expect(sanitizeDocsMarkdownLinks('See [retries](./14_retries.mdx) for more.', PAGE)).toBe(
+			'See [retries](https://www.windmill.dev/docs/flows/retries) for more.'
+		)
+	})
+
+	it('strips numeric prefixes from sibling-directory links', () => {
+		expect(
+			sanitizeDocsMarkdownLinks('[handling](../core_concepts/8_error_handling.mdx)', PAGE)
+		).toBe('[handling](https://www.windmill.dev/docs/core_concepts/error_handling)')
+	})
+
+	it('preserves anchors when rewriting', () => {
+		expect(sanitizeDocsMarkdownLinks('[branch all](./13_flow_branches.mdx#branch-all)', PAGE)).toBe(
+			'[branch all](https://www.windmill.dev/docs/flows/flow_branches#branch-all)'
+		)
+	})
+
+	it('leaves image and external links untouched', () => {
+		const input =
+			'![diagram](./assets/flow_example.png) and [site](https://example.com/page.md)'
+		expect(sanitizeDocsMarkdownLinks(input, PAGE)).toBe(input)
+	})
+
+	it('leaves bare anchor links untouched', () => {
+		expect(sanitizeDocsMarkdownLinks('[top](#introduction)', PAGE)).toBe('[top](#introduction)')
+	})
+
+	it('leaves ambiguous ../ cross-directory links untouched', () => {
+		// `../../flows/14_retries.md` is authored against the source tree; resolving
+		// it against the published URL is unreliable, so it must be left as-is.
+		const input = '[retries](../../flows/14_retries.md)'
+		expect(sanitizeDocsMarkdownLinks(input, PAGE)).toBe(input)
+	})
+})
+
+describe('canonicalDocsPageUrl', () => {
+	it('returns the published URL without the .md suffix', () => {
+		expect(canonicalDocsPageUrl('/docs/flows/flow_editor')).toBe(
+			'https://www.windmill.dev/docs/flows/flow_editor'
+		)
+	})
+
+	it('strips numeric prefixes so a source-style path maps to the published URL', () => {
+		expect(canonicalDocsPageUrl('/docs/flows/14_retries.md')).toBe(
+			'https://www.windmill.dev/docs/flows/retries'
+		)
+	})
+})
+
+describe('renderDocsPageResult', () => {
+	it('returns the whole page when small and no section requested', () => {
+		expect(renderDocsPageResult(SAMPLE)).toBe(SAMPLE)
+	})
+
+	it('returns an outline for large pages with no section requested', () => {
+		const large = `# Big\n\n${'x'.repeat(25_000)}\n\n## Tail\n\nmore`
+		const result = renderDocsPageResult(large)
+		expect(result).toContain('This documentation page is large')
+		expect(result).toContain('- Big (~')
+		expect(result).toContain('- Tail (~')
+	})
+
+	it('returns the requested section content when found', () => {
+		const result = renderDocsPageResult(SAMPLE, 'Job kinds')
+		expect(result).toContain('## Job kinds')
+		expect(result).toContain('Some kinds.')
+	})
+
+	it('returns the outline with a note when the requested section is missing', () => {
+		const result = renderDocsPageResult(SAMPLE, 'Does not exist')
+		expect(result).toContain('No section matching "Does not exist" was found')
+		expect(result).toContain('- Jobs (~')
+	})
+})

frontend/src/lib/components/copilot/chat/docs/core.ts

@@ -0,0 +1,417 @@
+import type { Tool } from '../shared'
+import type { ChatCompletionTool } from 'openai/resources/index.mjs'
+
+const DOCS_ORIGIN = 'https://www.windmill.dev'
+const LLMS_TXT_URL = `${DOCS_ORIGIN}/llms.txt`
+const CACHE_TTL_MS = 15 * 60 * 1000
+// Above this size, return an outline of the page's headings instead of the full
+// content, prompting the model to request a specific section.
+const FULL_PAGE_CHAR_LIMIT = 20_000
+
+interface CacheEntry {
+	expiresAt: number
+	promise: Promise<string>
+}
+
+let llmsTxtCache: CacheEntry | undefined
+const pageCache = new Map<string, CacheEntry>()
+
+/**
+ * Fetches the docs index (llms.txt) listing every documentation page. Cached at
+ * module level with a TTL so repeated tool calls within a session reuse it.
+ */
+export async function fetchDocsIndex(): Promise<string> {
+	const now = Date.now()
+	if (llmsTxtCache && llmsTxtCache.expiresAt > now) {
+		return llmsTxtCache.promise
+	}
+
+	const promise = fetchText(LLMS_TXT_URL).catch((error) => {
+		// Drop the failed promise from the cache so the next call retries.
+		if (llmsTxtCache?.promise === promise) {
+			llmsTxtCache = undefined
+		}
+		throw error
+	})
+	llmsTxtCache = { expiresAt: now + CACHE_TTL_MS, promise }
+	return promise
+}
+
+/**
+ * Fetches a single docs page as raw markdown. `path` may be a full URL or a
+ * /docs/... path; it is normalized to a `.md` URL. Cached per resolved URL.
+ */
+export async function fetchDocsPage(path: string): Promise<string> {
+	const url = normalizeDocsUrl(path)
+	const now = Date.now()
+	const cached = pageCache.get(url)
+	if (cached && cached.expiresAt > now) {
+		return cached.promise
+	}
+
+	const promise = fetchText(url)
+		.then((content) => sanitizeDocsMarkdownLinks(content, url))
+		.catch((error) => {
+			if (pageCache.get(url)?.promise === promise) {
+				pageCache.delete(url)
+			}
+			throw error
+		})
+	pageCache.set(url, { expiresAt: now + CACHE_TTL_MS, promise })
+	return promise
+}
+
+async function fetchText(url: string): Promise<string> {
+	const response = await fetch(url)
+	if (!response.ok) {
+		throw new Error(`Request to ${url} failed with status ${response.status}`)
+	}
+	return await response.text()
+}
+
+/**
+ * Normalizes a user/model-supplied docs reference to a fully-qualified `.md`
+ * URL on the docs origin. Accepts:
+ *   - `https://www.windmill.dev/docs/core_concepts/jobs`
+ *   - `/docs/core_concepts/jobs.md`
+ *   - `docs/core_concepts/jobs`
+ */
+export function normalizeDocsUrl(input: string): string {
+	let value = input.trim()
+
+	if (/^https?:\/\//i.test(value)) {
+		// Strip the origin so we can re-anchor to DOCS_ORIGIN and normalize the path.
+		try {
+			const parsed = new URL(value)
+			value = parsed.pathname
+		} catch {
+			// Fall through and treat as a path.
+		}
+	}
+
+	// Drop any query string or hash fragment.
+	value = value.split('#')[0].split('?')[0]
+
+	if (!value.startsWith('/')) {
+		value = `/${value}`
+	}
+
+	// Strip a trailing slash (but keep the leading one).
+	if (value.length > 1 && value.endsWith('/')) {
+		value = value.slice(0, -1)
+	}
+
+	// Relative links inside the raw markdown reference docusaurus source files
+	// (e.g. `13_flow_branches.mdx`), but the published routes drop the numeric
+	// ordering prefixes and use `.md`.
+	value = stripDocsPathPrefixes(value)
+	if (value.endsWith('.mdx')) {
+		value = value.slice(0, -1)
+	}
+
+	if (!value.endsWith('.md')) {
+		value = `${value}.md`
+	}
+
+	return `${DOCS_ORIGIN}${value}`
+}
+
+/**
+ * The canonical published URL a model should cite for a docs page (the `.md`
+ * fetch URL without the suffix), e.g. `https://www.windmill.dev/docs/flows/retries`.
+ */
+export function canonicalDocsPageUrl(path: string): string {
+	return normalizeDocsUrl(path).replace(/\.md$/i, '')
+}
+
+/**
+ * Strips docusaurus numeric ordering prefixes (`13_`, `8-`) from each segment of
+ * a docs path so it matches the published route. Operates on the path only.
+ */
+function stripDocsPathPrefixes(path: string): string {
+	return path
+		.split('/')
+		.map((segment) => segment.replace(/^\d+[_-]/, ''))
+		.join('/')
+}
+
+/**
+ * Rewrites relative/source-file doc links inside raw page markdown to canonical
+ * published URLs, so the model never echoes a docusaurus source path (e.g.
+ * `./13_flow_branches.mdx`) into its answer as a broken link. Resolves each link
+ * relative to the page it came from, strips numeric ordering prefixes, and drops
+ * the `.md`/`.mdx` extension. Non-doc links (external, images, anchors) are left
+ * untouched.
+ */
+export function sanitizeDocsMarkdownLinks(content: string, pageUrl: string): string {
+	return content.replace(/\]\(([^)\s]+?)(\s+"[^"]*")?\)/g, (match, target: string, title) => {
+		if (!/\.mdx?($|[#?])/i.test(target)) {
+			// Only rewrite links to docusaurus source files (.md/.mdx); leave
+			// images, external URLs and bare anchors untouched.
+			return match
+		}
+		if (/(^|\/)\.\.\//.test(target)) {
+			// `../` cross-directory links are authored against the docusaurus
+			// source tree, whose depth differs from the published URL, so strict
+			// resolution is unreliable. Leave them for the canonical-URL header to
+			// disambiguate rather than risk rewriting to a wrong path.
+			return match
+		}
+		let resolved: URL
+		try {
+			resolved = new URL(target, pageUrl)
+		} catch {
+			return match
+		}
+		if (resolved.origin !== DOCS_ORIGIN || !resolved.pathname.startsWith('/docs/')) {
+			return match
+		}
+		const pathname = stripDocsPathPrefixes(resolved.pathname).replace(/\.mdx?$/i, '')
+		return `](${DOCS_ORIGIN}${pathname}${resolved.hash}${title ?? ''})`
+	})
+}
+
+export interface DocsHeading {
+	level: number
+	title: string
+	/** Character offset of the start of the heading line within the document. */
+	startIndex: number
+}
+
+/**
+ * Parses the markdown headings (`#`–`####`) of a docs page, ignoring any
+ * heading-like lines that appear inside fenced code blocks (``` fences), which
+ * are common in docs pages (e.g. `# comment` inside a python sample).
+ */
+export function parseDocsHeadings(content: string): DocsHeading[] {
+	const headings: DocsHeading[] = []
+	let offset = 0
+	let inFence = false
+	let fenceMarker = ''
+
+	const lines = content.split('\n')
+	for (const line of lines) {
+		const fence = matchFence(line)
+		if (fence) {
+			if (!inFence) {
+				inFence = true
+				fenceMarker = fence
+			} else if (line.trimStart().startsWith(fenceMarker)) {
+				inFence = false
+				fenceMarker = ''
+			}
+			offset += line.length + 1
+			continue
+		}
+
+		if (!inFence) {
+			const match = /^(#{1,4})\s+(.*\S)\s*$/.exec(line)
+			if (match) {
+				headings.push({
+					level: match[1].length,
+					title: match[2].trim(),
+					startIndex: offset
+				})
+			}
+		}
+
+		offset += line.length + 1
+	}
+
+	return headings
+}
+
+function matchFence(line: string): string | undefined {
+	const trimmed = line.trimStart()
+	const match = /^(`{3,}|~{3,})/.exec(trimmed)
+	return match ? match[1] : undefined
+}
+
+/**
+ * Builds a human-readable outline of a page's headings, including an approximate
+ * character size for each section. Used when a page is too large to return whole.
+ */
+export function buildDocsOutline(content: string): string {
+	const headings = parseDocsHeadings(content)
+	if (headings.length === 0) {
+		return '(no markdown headings found on this page)'
+	}
+
+	const lines = headings.map((heading, index) => {
+		const sectionEnd = sectionEndIndex(content, headings, index)
+		const approxChars = sectionEnd - heading.startIndex
+		const indent = '  '.repeat(Math.max(0, heading.level - 1))
+		return `${indent}- ${heading.title} (~${approxChars} chars)`
+	})
+
+	return lines.join('\n')
+}
+
+function sectionEndIndex(content: string, headings: DocsHeading[], index: number): number {
+	const heading = headings[index]
+	// A section ends at the next heading of the same or higher (shallower) level.
+	for (let i = index + 1; i < headings.length; i++) {
+		if (headings[i].level <= heading.level) {
+			return headings[i].startIndex
+		}
+	}
+	return content.length
+}
+
+/** Normalizes a heading title for tolerant, case/punctuation-insensitive matching. */
+function normalizeHeadingTitle(title: string): string {
+	return title
+		.toLowerCase()
+		.replace(/[^a-z0-9]+/g, ' ')
+		.trim()
+}
+
+/**
+ * Extracts the content of the section whose heading matches `section`, from the
+ * matching heading up to the next heading of the same or higher level. Matching
+ * is case-insensitive and tolerant of minor punctuation differences. Returns
+ * `undefined` when no heading matches.
+ */
+export function extractDocsSection(content: string, section: string): string | undefined {
+	const headings = parseDocsHeadings(content)
+	const target = normalizeHeadingTitle(section)
+	if (target.length === 0) {
+		return undefined
+	}
+
+	let matchIndex = headings.findIndex(
+		(heading) => normalizeHeadingTitle(heading.title) === target
+	)
+	if (matchIndex === -1) {
+		// Fall back to a contains match so "Result streaming" matches "Result".
+		matchIndex = headings.findIndex((heading) =>
+			normalizeHeadingTitle(heading.title).includes(target)
+		)
+	}
+	if (matchIndex === -1) {
+		return undefined
+	}
+
+	const start = headings[matchIndex].startIndex
+	const end = sectionEndIndex(content, headings, matchIndex)
+	return content.slice(start, end).trim()
+}
+
+const LIST_DOCS_PAGES_TOOL: ChatCompletionTool = {
+	type: 'function',
+	function: {
+		name: 'list_docs_pages',
+		description:
+			'Return the full Windmill documentation index (llms.txt): a list of every docs page with its title, URL and a short description. Call this FIRST, before read_docs_page, to discover which pages are relevant to the user request.',
+		parameters: {
+			type: 'object',
+			properties: {},
+			required: []
+		}
+	}
+}
+
+const READ_DOCS_PAGE_TOOL: ChatCompletionTool = {
+	type: 'function',
+	function: {
+		name: 'read_docs_page',
+		description:
+			'Fetch the raw markdown of a single Windmill documentation page. Provide the `path` (or full URL) of a page found via list_docs_pages. If the page is large, this returns its list of section headings instead of the full content; call again with the `section` argument set to one of those headings to read that section.',
+		parameters: {
+			type: 'object',
+			properties: {
+				path: {
+					type: 'string',
+					description:
+						'The docs page to read, as a path (e.g. /docs/core_concepts/jobs) or full URL (e.g. https://www.windmill.dev/docs/core_concepts/jobs).'
+				},
+				section: {
+					type: 'string',
+					description:
+						'Optional. A heading title from the page outline to read just that section instead of the full page.'
+				}
+			},
+			required: ['path']
+		}
+	}
+}
+
+export const listDocsPagesTool: Tool<{}> = {
+	def: LIST_DOCS_PAGES_TOOL,
+	fn: async ({ toolId, toolCallbacks }) => {
+		toolCallbacks.setToolStatus(toolId, { content: 'Listing documentation pages...' })
+		try {
+			const index = await fetchDocsIndex()
+			toolCallbacks.setToolStatus(toolId, { content: 'Retrieved documentation index' })
+			return index
+		} catch (error) {
+			toolCallbacks.setToolStatus(toolId, {
+				content: 'Error listing documentation pages',
+				error: 'Error listing documentation pages'
+			})
+			console.error('Error listing documentation pages:', error)
+			const errorMessage =
+				error instanceof Error ? error.message : 'An error occurred while listing documentation pages'
+			return `Failed to list documentation pages: ${errorMessage}, pursuing with the user request...`
+		}
+	}
+}
+
+export const readDocsPageTool: Tool<{}> = {
+	def: READ_DOCS_PAGE_TOOL,
+	fn: async ({ args, toolId, toolCallbacks }) => {
+		const path = typeof args?.path === 'string' ? args.path : ''
+		const section = typeof args?.section === 'string' && args.section.trim() ? args.section : undefined
+		toolCallbacks.setToolStatus(toolId, {
+			content: section ? `Reading docs section "${section}"...` : 'Reading documentation page...'
+		})
+		try {
+			if (!path.trim()) {
+				return 'No documentation page path was provided. Provide a `path` from list_docs_pages.'
+			}
+			const content = await fetchDocsPage(path)
+			toolCallbacks.setToolStatus(toolId, { content: 'Read documentation page' })
+			const canonicalUrl = canonicalDocsPageUrl(path)
+			const header = `Source page — cite this URL when referencing this page: ${canonicalUrl}\n\n`
+			return header + renderDocsPageResult(content, section)
+		} catch (error) {
+			toolCallbacks.setToolStatus(toolId, {
+				content: 'Error reading documentation page',
+				error: 'Error reading documentation page'
+			})
+			console.error('Error reading documentation page:', error)
+			const errorMessage =
+				error instanceof Error ? error.message : 'An error occurred while reading the documentation page'
+			return `Failed to read documentation page: ${errorMessage}, pursuing with the user request...`
+		}
+	}
+}
+
+/**
+ * Decides what to return for read_docs_page: a requested section, the full page,
+ * or an outline asking the model to pick a section.
+ */
+export function renderDocsPageResult(content: string, section?: string): string {
+	if (section) {
+		const extracted = extractDocsSection(content, section)
+		if (extracted !== undefined) {
+			return extracted
+		}
+		return [
+			`No section matching "${section}" was found on this page. Available sections:`,
+			'',
+			buildDocsOutline(content)
+		].join('\n')
+	}
+
+	if (content.length <= FULL_PAGE_CHAR_LIMIT) {
+		return content
+	}
+
+	return [
+		'This documentation page is large. Below is its list of sections with approximate sizes.',
+		'Call read_docs_page again with the same path and a `section` set to one of these headings to read that section.',
+		'',
+		buildDocsOutline(content)
+	].join('\n')
+}

frontend/src/lib/components/copilot/chat/global/core.test.ts

@@ -2020,6 +2020,25 @@ describe('session-only preview tools gating', () => {
 	})
 })
 
+describe('documentation lookup tools', () => {
+	it('exposes list_docs_pages and read_docs_page in both the session and non-session tool sets', () => {
+		// The docs tools are read-only references, not session-preview tools, so
+		// they must be present regardless of `sessionPreview`.
+		for (const sessionPreview of [false, true]) {
+			const names = globalToolsFor({ sessionPreview }).map((t) => t.def.function.name)
+			expect(names).toContain('list_docs_pages')
+			expect(names).toContain('read_docs_page')
+		}
+	})
+
+	it('instructs the model to look up product docs and cite the Source page URL', () => {
+		const content = prepareGlobalSystemMessage().content as string
+		expect(content).toContain('list_docs_pages')
+		expect(content).toContain('read_docs_page')
+		expect(content).toContain('Source page')
+	})
+})
+
 describe('prepareGlobalUserMessage', () => {
 	it('injects the active editor reference without contents', () => {
 		__resetUserDraftForTesting()

frontend/src/lib/components/copilot/chat/global/core.ts

@@ -74,6 +74,7 @@ import {
 } from '../shared'
 import type { ContextElement } from '../context'
 import { getDatatableTools } from '../datatableTools'
+import { listDocsPagesTool, readDocsPageTool } from '../docs/core'
 import { UserDraft, type UserDraftMeta } from '$lib/userDraft.svelte'
 import { emptySchema } from '$lib/utils'
 import { inferArgs } from '$lib/infer'
@@ -639,6 +640,12 @@ Rules:
 		: ''
 	}
 
+Documentation:
+- To answer questions about Windmill product features, concepts, or syntax, look them up in the official docs with list_docs_pages and read_docs_page rather than relying on memory.
+- Call list_docs_pages FIRST to see the full index of documentation pages with their titles, URLs and descriptions. Then call read_docs_page on the 1 to 3 most relevant pages. If read_docs_page returns a list of section headings instead of the full page, call it again with the same path and a \`section\` argument to read the relevant section.
+- Base documentation answers ONLY on what you read. Do not invent features, flags, syntax, or behavior you did not see in the docs.
+- When you cite a docs page, use the exact "Source page" URL printed at the top of each read_docs_page result — never reconstruct a URL from a link inside the page body. If the docs do not cover the question, say so rather than inventing an answer.
+
 Flows:
 - read_workspace_item returns compact flow JSON. Inline script bodies appear as "inline_script.<moduleId>".
 - Use read_flow_module_code and set_flow_module_code for inline script bodies.
@@ -1469,6 +1476,8 @@ export const globalTools: Tool<{}>[] = [
 		}
 	},
 	createSearchHubScriptsTool(false),
+	listDocsPagesTool,
+	readDocsPageTool,
 	{
 		def: createToolDef(
 			askUserQuestionSchema,