diff --git a/ai_evals/adapters/frontend/benchmarkRunner.ts b/ai_evals/adapters/frontend/benchmarkRunner.ts index 1729df7170..32107eadf1 100644 --- a/ai_evals/adapters/frontend/benchmarkRunner.ts +++ b/ai_evals/adapters/frontend/benchmarkRunner.ts @@ -96,7 +96,12 @@ async function getModeRunner( } function parseMode(value: string | undefined): FrontendBenchmarkMode { - if (value === "flow" || value === "app" || value === "script" || value === "global") { + if ( + value === "flow" || + value === "app" || + value === "script" || + value === "global" + ) { return value; } throw new Error(`Unsupported frontend benchmark mode: ${String(value)}`); diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index ebbbac8d11..2739eedce2 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -434,5 +434,6 @@ benchmarkIt( resetBenchmarkMockBackend() } }, - 600_000 + // Full-suite runs (30+ cases at concurrency 2-3) routinely exceed 10 minutes. + 7_200_000 ) diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 28839b0594..766515519b 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -870,3 +870,76 @@ judgeChecklist: - fetches the logs for the requested job id - explains the failure from the returned logs (connection refused to the upstream API) + +# --- Documentation search (search_docs) --- +# Pure product-knowledge questions: the assistant should consult the docs via +# search_docs and answer conversationally, not draft or mutate anything. No +# draft is produced, so the global judge is skipped and we validate tool use. + +- id: global-docs-ai-agent-step + prompt: |- + Does Windmill support a flow step where an LLM decides which of my scripts to call based on the input? + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - search_docs + forbiddenToolsUsed: + - write_script + - write_flow + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + +- id: global-docs-retry-step + prompt: |- + How does automatic retry work for a flow step that calls a flaky API? + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - search_docs + forbiddenToolsUsed: + - write_script + - write_flow + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + +- id: global-docs-key-value-store + prompt: |- + Can I use a Redis-style key-value store from my Windmill scripts, and how? + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - search_docs + forbiddenToolsUsed: + - write_script + - write_flow + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + +- id: global-docs-cron-schedule-format + prompt: |- + How do Windmill's cron schedules work, and what format does the schedule expression use? + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - search_docs + forbiddenToolsUsed: + - write_script + - write_flow + - deploy_workspace_item + - delete_workspace_item + skipJudge: true diff --git a/ai_evals/core/cases.test.ts b/ai_evals/core/cases.test.ts index 05e2f1527b..9955a73fa9 100644 --- a/ai_evals/core/cases.test.ts +++ b/ai_evals/core/cases.test.ts @@ -246,6 +246,21 @@ describe("loadCases", () => { }); }); + it("loads global docs-search cases as tool-use checks", async () => { + const globalCases = await loadCases("global"); + const docsCases = globalCases.filter((entry) => + entry.id.startsWith("global-docs-"), + ); + expect(docsCases.length).toBeGreaterThanOrEqual(3); + + // Each docs case verifies the assistant reaches for search_docs and does not + // draft anything; with no draft, the global judge is skipped. + for (const entry of docsCases) { + expect(entry.skipJudge).toBe(true); + expect(entry.toolExpect?.requiredToolsUsed).toContain("search_docs"); + } + }); + it("loads tool expectations for workspace mutation cases", async () => { const scriptCases = await loadCases("script"); const caseEntry = scriptCases.find( diff --git a/ai_evals/core/runSuite.ts b/ai_evals/core/runSuite.ts index ed82d841cb..bb0f9b99a4 100644 --- a/ai_evals/core/runSuite.ts +++ b/ai_evals/core/runSuite.ts @@ -225,7 +225,9 @@ async function runCaseAttempts(input: { checklist: input.evalCase.judgeChecklist, initial, expected: input.modeRunner.mode === "cli" ? undefined : expected, - actual: run.actual, + actual: input.modeRunner.prepareJudgeActual + ? input.modeRunner.prepareJudgeActual(run.actual) + : run.actual, model: input.judgeModel, }); diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 27c2fcddac..9e2e32d5c3 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -172,7 +172,10 @@ export interface ToolValidationSpec { toolCallArgs?: ToolCallArgumentRule[]; } -export type EvalValidationSpec = FlowValidationSpec | AppValidationSpec | GlobalValidationSpec; +export type EvalValidationSpec = + | FlowValidationSpec + | AppValidationSpec + | GlobalValidationSpec; export interface EvalCase { id: string; @@ -294,6 +297,12 @@ export interface ModeRunner { context: ModeRunContext; }): Promise; buildArtifacts?(actual: TActual): BenchmarkArtifactFile[]; + /** + * Optional transform applied to `actual` before it is handed to the LLM judge. + * Use it to strip fields the judge must stay blind to (e.g. which docs-tool + * arm produced an answer). When omitted, the judge receives `actual` as-is. + */ + prepareJudgeActual?(actual: TActual): unknown; } export interface BenchmarkAttemptResult { diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index ca8ffd483e..cb775a6084 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -49,6 +49,7 @@ import { get } from 'svelte/store' import { BROWSER } from 'esm-env' import { workspaceStore, type DBSchemas } from '$lib/stores' import { askTools, prepareAskSystemMessage, prepareAskUserMessage } from './ask/core' +import { readDocsPageTool, searchDocsTool } from './docs/core' import { chatState, DEFAULT_SIZE, triggerablesByAi } from './sharedChatState.svelte' import { createAppBackendRunnableContextElement, @@ -400,7 +401,7 @@ export class AIChatManager { try { this.apiTools = await loadApiTools() if (this.mode === AIMode.API) { - this.tools = [...this.apiTools] + this.tools = [searchDocsTool, readDocsPageTool, ...this.apiTools] } } catch (err) { console.error('Error loading api tools', err) @@ -666,7 +667,7 @@ export class AIChatManager { } else if (mode === AIMode.API) { const customPrompt = getCombinedCustomPrompt(mode) this.systemMessage = prepareApiSystemMessage(customPrompt) - this.tools = [...this.apiTools] + this.tools = [searchDocsTool, readDocsPageTool, ...this.apiTools] this.helpers = {} } else if (mode === AIMode.GLOBAL) { const customPrompt = getCombinedCustomPrompt(mode) diff --git a/frontend/src/lib/components/copilot/chat/api/core.ts b/frontend/src/lib/components/copilot/chat/api/core.ts index 4e47baea72..e4c2ab07c7 100644 --- a/frontend/src/lib/components/copilot/chat/api/core.ts +++ b/frontend/src/lib/components/copilot/chat/api/core.ts @@ -4,7 +4,6 @@ import type { } from 'openai/resources/index.mjs' import type { Tool } from '../shared' import { loadApiTools } from './apiTools' -import { getDocumentationTool } from '../navigator/core' import { userStore } from '$lib/stores' import { get } from 'svelte/store' @@ -14,13 +13,13 @@ You are Windmill's intelligent assistant, designed to interact with the platform Windmill is an open-source developer platform for building internal tools, API integrations, background jobs, workflows, and user interfaces. It offers a unified system where scripts are automatically turned into sharable UIs and can be composed into flows or embedded in custom applications. You have access to these tools: -1. Get documentation for user requests (get_documentation) +1. Search the documentation (search_docs) and read a documentation page (read_docs_page) 2. A comprehensive list of API endpoints to interact with the Windmill backend INSTRUCTIONS: - You can directly query, list, create, update, and delete various Windmill resources like scripts, flows, jobs, resources, variables, schedules, and workers through the provided API tools. - When users ask about specific data or want to perform operations, use the appropriate API endpoints to fulfill their requests. -- Use get_documentation to retrieve accurate information about features, concepts, and best practices when needed. +- Use search_docs (then read_docs_page on a returned Source URL) to retrieve accurate information about features, concepts, and best practices when needed. - Always present API results in a clear, readable format for the user. - If you need to make multiple related API calls to fulfill a request, do so systematically and explain what you're doing. - When showing lists of items, provide meaningful summaries rather than overwhelming the user with raw data. @@ -55,8 +54,6 @@ export async function getApiTools(): Promise[]> { return apiToolsCache } -export const apiTools: Tool<{}>[] = [getDocumentationTool] - export function prepareApiSystemMessage(customPrompt?: string): ChatCompletionSystemMessageParam { let content = CHAT_SYSTEM_PROMPT(get(userStore)?.username ?? '') diff --git a/frontend/src/lib/components/copilot/chat/ask/core.ts b/frontend/src/lib/components/copilot/chat/ask/core.ts index f9ba219599..b93ace1179 100644 --- a/frontend/src/lib/components/copilot/chat/ask/core.ts +++ b/frontend/src/lib/components/copilot/chat/ask/core.ts @@ -3,19 +3,23 @@ import type { ChatCompletionUserMessageParam } from 'openai/resources/index.mjs' import type { Tool } from '../shared' -import { getDocumentationTool } from '../navigator/core' +import { readDocsPageTool, searchDocsTool } from '../docs/core' export const CHAT_SYSTEM_PROMPT = ` You are Windmill's intelligent assistant, designed to answer questions about its functionality. It is your only purpose to help the user in the context of the windmill application. Windmill is an open-source developer platform for building internal tools, API integrations, background jobs, workflows, and user interfaces. It offers a unified system where scripts are automatically turned into sharable UIs and can be composed into flows or embedded in custom applications. You have access to these tools: -1. Get documentation for user requests (get_documentation) +1. Search the documentation (search_docs) +2. Read a documentation page (read_docs_page) INSTRUCTIONS: -- When user asks about something, use the get_documentation tool to retrieve accurate information about how to fulfill the user's request. -- Complete your response with precisions about how it works based on the documentation. Also drop a link to the relevant documentation if possible. -- If the user asks about something that you are unsure about, say that you are not sure about the answer and suggest to ask the question to the windmill team. +- Call search_docs FIRST with a few distinctive keywords from the user's question to find the most relevant documentation pages and matching snippets. +- If the snippets already answer the question, answer directly. Otherwise call read_docs_page with one of the returned Source URLs to read the full page; if read_docs_page returns a list of section headings, call it again with the same path and a \`section\` argument to read the relevant section. +- If the first search returns nothing useful, retry with different or broader keywords before giving up. +- Answer based ONLY on what you find in the documentation. Do not invent features, flags, syntax, or behavior that you did not see in the docs. +- Always include the documentation URL(s) you consulted in your answer. Cite the exact "Source" URL shown in the search results (or the "Source page" URL at the top of a read page) — never reconstruct a URL from a link inside the page body. +- If the documentation does not cover the user's question, say so clearly rather than inventing an answer, and suggest asking the Windmill team. GENERAL PRINCIPLES: - Be concise but thorough @@ -23,7 +27,7 @@ GENERAL PRINCIPLES: - If you encounter an error or can't complete a request, explain why and suggest alternatives ` -export const askTools: Tool<{}>[] = [getDocumentationTool] +export const askTools: Tool<{}>[] = [searchDocsTool, readDocsPageTool] export function prepareAskSystemMessage(customPrompt?: string): ChatCompletionSystemMessageParam { let content = CHAT_SYSTEM_PROMPT diff --git a/frontend/src/lib/components/copilot/chat/docs/core.test.ts b/frontend/src/lib/components/copilot/chat/docs/core.test.ts new file mode 100644 index 0000000000..da0e7f0880 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/docs/core.test.ts @@ -0,0 +1,464 @@ +import { describe, expect, it } from 'vitest' +import { + buildDocsOutline, + canonicalDocsPageUrl, + extractDocsSection, + formatDocsSearchResults, + makeSnippet, + mergeDocsSearchResults, + normalizeDocsUrl, + parseDocsFullText, + parseDocsHeadings, + parseDocsIndex, + renderDocsPageResult, + sanitizeDocsMarkdownLinks, + searchDocsIndex, + searchDocsPages +} from './core' + +const SAMPLE = `# Jobs + +Intro text about jobs. + +## Job kinds + +Some kinds. + +## Result + +### Result of jobs that failed + +\`\`\` +{ "error": "boom" } +\`\`\` + +### Result streaming + +#### Returning a stream directly + +\`\`\`python +# Returning a stream directly is a comment heading that must be ignored +def main(): + pass +\`\`\` + +## Retention policy + +Final section. +` + +describe('parseDocsHeadings', () => { + it('parses headings with their levels and ignores headings inside fenced code blocks', () => { + const headings = parseDocsHeadings(SAMPLE) + const titles = headings.map((h) => `${h.level}:${h.title}`) + + expect(titles).toEqual([ + '1:Jobs', + '2:Job kinds', + '2:Result', + '3:Result of jobs that failed', + '3:Result streaming', + '4:Returning a stream directly', + '2:Retention policy' + ]) + // The "# Returning a stream directly is a comment..." line inside the + // python fence must not be parsed as a heading. + expect(titles).not.toContain('1:Returning a stream directly is a comment heading that must be ignored') + }) + + it('returns startIndex offsets that point at the heading line', () => { + const headings = parseDocsHeadings(SAMPLE) + for (const heading of headings) { + expect(SAMPLE.slice(heading.startIndex)).toMatch( + new RegExp(`^#{${heading.level}}\\s+${heading.title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}`) + ) + } + }) + + it('handles tilde fences', () => { + const content = '# Title\n\n~~~\n# not a heading\n~~~\n\n## Real\n' + const headings = parseDocsHeadings(content) + expect(headings.map((h) => h.title)).toEqual(['Title', 'Real']) + }) +}) + +describe('extractDocsSection', () => { + it('extracts a section from its heading up to the next same-or-higher level heading', () => { + const section = extractDocsSection(SAMPLE, 'Result') + expect(section).toBeDefined() + expect(section).toContain('## Result') + expect(section).toContain('### Result of jobs that failed') + expect(section).toContain('### Result streaming') + // Stops before the next level-2 heading. + expect(section).not.toContain('## Retention policy') + }) + + it('matches case-insensitively and tolerates punctuation differences', () => { + const section = extractDocsSection(SAMPLE, 'retention-policy!') + expect(section).toBeDefined() + expect(section).toContain('## Retention policy') + expect(section).toContain('Final section.') + }) + + it('returns the deepest section bounded by the next same-level heading', () => { + const section = extractDocsSection(SAMPLE, 'Result streaming') + expect(section).toBeDefined() + expect(section).toContain('### Result streaming') + expect(section).toContain('#### Returning a stream directly') + expect(section).not.toContain('## Retention policy') + }) + + it('returns undefined when no heading matches', () => { + expect(extractDocsSection(SAMPLE, 'Nonexistent section')).toBeUndefined() + }) +}) + +describe('buildDocsOutline', () => { + it('lists headings with approximate per-section sizes and indentation', () => { + const outline = buildDocsOutline(SAMPLE) + expect(outline).toContain('- Jobs (~') + expect(outline).toContain(' - Job kinds (~') + expect(outline).toContain(' - Result of jobs that failed (~') + }) + + it('handles pages with no headings', () => { + expect(buildDocsOutline('just some text\nwith no headings')).toBe( + '(no markdown headings found on this page)' + ) + }) +}) + +describe('normalizeDocsUrl', () => { + it('appends .md to a bare path', () => { + expect(normalizeDocsUrl('/docs/core_concepts/jobs')).toBe( + 'https://www.windmill.dev/docs/core_concepts/jobs.md' + ) + }) + + it('accepts a path without a leading slash', () => { + expect(normalizeDocsUrl('docs/core_concepts/jobs')).toBe( + 'https://www.windmill.dev/docs/core_concepts/jobs.md' + ) + }) + + it('accepts a full URL and strips anchors and query strings', () => { + expect( + normalizeDocsUrl('https://www.windmill.dev/docs/core_concepts/jobs#result?foo=bar') + ).toBe('https://www.windmill.dev/docs/core_concepts/jobs.md') + }) + + it('does not double-append .md', () => { + expect(normalizeDocsUrl('/docs/core_concepts/jobs.md')).toBe( + 'https://www.windmill.dev/docs/core_concepts/jobs.md' + ) + }) + + it('strips a trailing slash before appending .md', () => { + expect(normalizeDocsUrl('/docs/core_concepts/jobs/')).toBe( + 'https://www.windmill.dev/docs/core_concepts/jobs.md' + ) + }) + + it('strips docusaurus numeric ordering prefixes from path segments', () => { + expect(normalizeDocsUrl('/docs/flows/13_flow_branches')).toBe( + 'https://www.windmill.dev/docs/flows/flow_branches.md' + ) + }) + + it('converts a .mdx source suffix to .md', () => { + expect(normalizeDocsUrl('/docs/flows/13_flow_branches.mdx')).toBe( + 'https://www.windmill.dev/docs/flows/flow_branches.md' + ) + }) +}) + +describe('sanitizeDocsMarkdownLinks', () => { + const PAGE = 'https://www.windmill.dev/docs/flows/flow_editor.md' + + it('rewrites a relative .mdx source link to a canonical published URL', () => { + expect(sanitizeDocsMarkdownLinks('See [retries](./14_retries.mdx) for more.', PAGE)).toBe( + 'See [retries](https://www.windmill.dev/docs/flows/retries) for more.' + ) + }) + + it('strips numeric prefixes from same-directory links', () => { + expect(sanitizeDocsMarkdownLinks('[handling](./8_error_handling.mdx)', PAGE)).toBe( + '[handling](https://www.windmill.dev/docs/flows/error_handling)' + ) + }) + + it('preserves anchors when rewriting', () => { + expect(sanitizeDocsMarkdownLinks('[branch all](./13_flow_branches.mdx#branch-all)', PAGE)).toBe( + '[branch all](https://www.windmill.dev/docs/flows/flow_branches#branch-all)' + ) + }) + + it('leaves image and external links untouched', () => { + const input = + '![diagram](./assets/flow_example.png) and [site](https://example.com/page.md)' + expect(sanitizeDocsMarkdownLinks(input, PAGE)).toBe(input) + }) + + it('leaves bare anchor links untouched', () => { + expect(sanitizeDocsMarkdownLinks('[top](#introduction)', PAGE)).toBe('[top](#introduction)') + }) + + // `../` links are authored against the docusaurus source tree, whose directory + // depth differs from the published URL on slug-flattened pages, so resolving + // them against the page URL is unreliable (a single `../` can over-escape just + // as a double one does). All `../` links are left untouched and disambiguated + // by the canonical "Source page" header instead. + it('leaves single ../ cross-directory links untouched', () => { + const input = '[handling](../core_concepts/8_error_handling.mdx)' + expect(sanitizeDocsMarkdownLinks(input, PAGE)).toBe(input) + }) + + it('leaves double ../../ cross-directory links untouched', () => { + const input = '[retries](../../flows/14_retries.md)' + expect(sanitizeDocsMarkdownLinks(input, PAGE)).toBe(input) + }) +}) + +describe('canonicalDocsPageUrl', () => { + it('returns the published URL without the .md suffix', () => { + expect(canonicalDocsPageUrl('/docs/flows/flow_editor')).toBe( + 'https://www.windmill.dev/docs/flows/flow_editor' + ) + }) + + it('strips numeric prefixes so a source-style path maps to the published URL', () => { + expect(canonicalDocsPageUrl('/docs/flows/14_retries.md')).toBe( + 'https://www.windmill.dev/docs/flows/retries' + ) + }) +}) + +describe('renderDocsPageResult', () => { + it('returns the whole page when small and no section requested', () => { + expect(renderDocsPageResult(SAMPLE)).toBe(SAMPLE) + }) + + it('returns an outline for large pages with no section requested', () => { + const large = `# Big\n\n${'x'.repeat(25_000)}\n\n## Tail\n\nmore` + const result = renderDocsPageResult(large) + expect(result).toContain('This documentation page is large') + expect(result).toContain('- Big (~') + expect(result).toContain('- Tail (~') + }) + + it('returns the requested section content when found', () => { + const result = renderDocsPageResult(SAMPLE, 'Job kinds') + expect(result).toContain('## Job kinds') + expect(result).toContain('Some kinds.') + }) + + it('returns the outline with a note when the requested section is missing', () => { + const result = renderDocsPageResult(SAMPLE, 'Does not exist') + expect(result).toContain('No section matching "Does not exist" was found') + expect(result).toContain('- Jobs (~') + }) +}) + +// Mirrors the llms-full.txt layout: a corpus preamble, then per-page blocks each +// introduced by a `---` + `## ` lead-in followed by a `Source:` line. +const SAMPLE_FULL = `# Windmill + +> Preamble blurb that precedes the first Source line and must be ignored. + +## Browser automation + +Source: https://www.windmill.dev/docs/advanced/browser_automation + +# Browser automation + +By default, a worker group named \`reports\` handles jobs with the \`chromium\` tag. +The chromium binary will be available on these workers at /usr/bin/chromium. +You can disable the sandbox by passing the --no-sandbox flag. + +--- + +## Worker groups + +Source: https://www.windmill.dev/docs/core_concepts/worker_groups + +# Worker groups + +Worker groups let you assign tags to workers. +Set the chromium tag on a worker so it can run browser jobs. + +--- + +## Scheduling + +Source: https://www.windmill.dev/docs/core_concepts/scheduling + +# Scheduling + +Use cron expressions to schedule scripts and flows. +` + +describe('parseDocsFullText', () => { + it('splits the corpus into pages keyed by Source URL, dropping the preamble', () => { + const pages = parseDocsFullText(SAMPLE_FULL) + expect(pages.map((p) => p.url)).toEqual([ + 'https://www.windmill.dev/docs/advanced/browser_automation', + 'https://www.windmill.dev/docs/core_concepts/worker_groups', + 'https://www.windmill.dev/docs/core_concepts/scheduling' + ]) + }) + + it('uses each page first heading as its title', () => { + const pages = parseDocsFullText(SAMPLE_FULL) + expect(pages.map((p) => p.title)).toEqual([ + 'Browser automation', + 'Worker groups', + 'Scheduling' + ]) + }) + + it('strips the trailing category lead-in so it is not mis-attributed to the previous page', () => { + const pages = parseDocsFullText(SAMPLE_FULL) + const browser = pages.find((p) => p.url.endsWith('/browser_automation')) + // "## Worker groups" introduces the *next* page and must not leak into this body. + expect(browser?.body).not.toContain('Worker groups') + expect(browser?.body).not.toContain('---') + }) +}) + +describe('searchDocsPages', () => { + const pages = parseDocsFullText(SAMPLE_FULL) + + it('ranks the page with more occurrences of the term first', () => { + const results = searchDocsPages(pages, 'chromium') + expect(results.map((r) => r.url)).toEqual([ + 'https://www.windmill.dev/docs/advanced/browser_automation', + 'https://www.windmill.dev/docs/core_concepts/worker_groups' + ]) + expect(results[0].snippets.length).toBeGreaterThan(0) + expect(results[0].snippets.join('\n')).toContain('chromium') + }) + + it('prefers pages that cover every query term over partial matches', () => { + // Only browser_automation mentions both "chromium" and "sandbox". + const results = searchDocsPages(pages, 'chromium sandbox') + expect(results.map((r) => r.url)).toEqual([ + 'https://www.windmill.dev/docs/advanced/browser_automation' + ]) + }) + + it('returns nothing when no term matches', () => { + expect(searchDocsPages(pages, 'kubernetes helm chart')).toEqual([]) + }) + + it('respects the maxPages cap', () => { + const results = searchDocsPages(pages, 'worker', { maxPages: 1 }) + expect(results.length).toBe(1) + }) +}) + +describe('makeSnippet', () => { + it('returns short lines unchanged after collapsing whitespace', () => { + expect(makeSnippet(' hello world ', ['world'], 200)).toBe('hello world') + }) + + it('windows a long line around the first matched term with ellipses', () => { + const line = `${'a '.repeat(200)}NEEDLE${' b'.repeat(200)}` + const snippet = makeSnippet(line, ['needle'], 60) + expect(snippet.length).toBeLessThanOrEqual(62) // 60 + two ellipsis chars + expect(snippet.toLowerCase()).toContain('needle') + expect(snippet.startsWith('…')).toBe(true) + expect(snippet.endsWith('…')).toBe(true) + }) +}) + +describe('formatDocsSearchResults', () => { + it('renders Source URLs, snippet bullets and a citation instruction', () => { + const results = searchDocsPages(parseDocsFullText(SAMPLE_FULL), 'chromium') + const rendered = formatDocsSearchResults('chromium', results) + expect(rendered).toContain('Source: https://www.windmill.dev/docs/advanced/browser_automation') + expect(rendered).toContain(' - ') + expect(rendered).toContain('Cite the exact "Source" URL') + }) + + it('returns a no-match message when there are no results', () => { + expect(formatDocsSearchResults('zzz', [])).toContain('No documentation pages matched "zzz"') + }) +}) + +const SAMPLE_INDEX = `# Windmill + +> Blurb. + +## Documentation structure + +### Core concepts +- [AI agents](https://www.windmill.dev/docs/core_concepts/ai_agents.md): How do I build AI agents in Windmill? Add agent steps to flows. Connect to OpenAI, Anthropic and more. +- [Retries](https://www.windmill.dev/docs/flows/retries.md): How do I retry a failing flow step automatically with exponential backoff? +- [Persistent storage](https://www.windmill.dev/docs/core_concepts/persistent_storage/within_windmill.md): How do I persist state between runs in Windmill? +` + +describe('parseDocsIndex', () => { + it('parses index entries into title, url and description', () => { + const entries = parseDocsIndex(SAMPLE_INDEX) + expect(entries).toHaveLength(3) + expect(entries[0]).toEqual({ + title: 'AI agents', + url: 'https://www.windmill.dev/docs/core_concepts/ai_agents.md', + description: + 'How do I build AI agents in Windmill? Add agent steps to flows. Connect to OpenAI, Anthropic and more.' + }) + }) + + it('ignores lines that are not docs links', () => { + expect(parseDocsIndex('## Heading\n> blurb\nplain text')).toEqual([]) + }) +}) + +describe('searchDocsIndex', () => { + const entries = parseDocsIndex(SAMPLE_INDEX) + + it('surfaces a named feature from its title/description when body grep would miss it', () => { + // The branch-centric phrasing a model used that failed body search; the + // index entry still matches on "agent"/"LLM"-adjacent terms. + const results = searchDocsIndex(entries, 'AI agent step decide') + expect(results[0].url).toBe('https://www.windmill.dev/docs/core_concepts/ai_agents.md') + expect(results[0].snippets[0]).toContain('agent steps') + }) + + it('ranks title matches above description-only matches', () => { + const results = searchDocsIndex(entries, 'retries') + expect(results[0].url).toBe('https://www.windmill.dev/docs/flows/retries.md') + }) + + it('returns nothing when no term matches', () => { + expect(searchDocsIndex(entries, 'kubernetes helm')).toEqual([]) + }) +}) + +describe('mergeDocsSearchResults', () => { + const body: ReturnType = [ + { url: 'https://www.windmill.dev/docs/openflow', title: 'OpenFlow', score: 10, snippets: ['x'] } + ] + const index: ReturnType = [ + // Same page as a body hit but as the index `.md` URL — must dedupe. + { + url: 'https://www.windmill.dev/docs/openflow.md', + title: 'OpenFlow', + score: 5, + snippets: ['desc'] + }, + { url: 'https://www.windmill.dev/docs/flows/retries.md', title: 'Retries', score: 4, snippets: ['desc'] } + ] + + it('keeps body results first and appends index-only matches, deduping by canonical URL', () => { + const merged = mergeDocsSearchResults(body, index) + expect(merged.map((r) => r.url)).toEqual([ + 'https://www.windmill.dev/docs/openflow', + 'https://www.windmill.dev/docs/flows/retries.md' + ]) + }) + + it('respects the maxPages cap', () => { + expect(mergeDocsSearchResults(body, index, 1)).toHaveLength(1) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/docs/core.ts b/frontend/src/lib/components/copilot/chat/docs/core.ts new file mode 100644 index 0000000000..0329d2025d --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/docs/core.ts @@ -0,0 +1,863 @@ +import type { Tool } from '../shared' +import type { ChatCompletionTool } from 'openai/resources/index.mjs' + +const DOCS_ORIGIN = 'https://www.windmill.dev' +const LLMS_TXT_URL = `${DOCS_ORIGIN}/llms.txt` +const LLMS_FULL_TXT_URL = `${DOCS_ORIGIN}/llms-full.txt` +const CACHE_TTL_MS = 15 * 60 * 1000 +// Above this size, return an outline of the page's headings instead of the full +// content, prompting the model to request a specific section. +const FULL_PAGE_CHAR_LIMIT = 20_000 + +// search_docs result caps — keep the returned payload small (the whole point of +// search vs. dumping the index or full pages is token economy). +const SEARCH_MAX_PAGES = 8 +const SEARCH_MAX_SNIPPETS_PER_PAGE = 3 +const SEARCH_MAX_SNIPPET_CHARS = 200 + +interface CacheEntry { + expiresAt: number + promise: Promise +} + +let llmsTxtCache: CacheEntry | undefined +let llmsFullTxtCache: CacheEntry | undefined +const pageCache = new Map() + +/** + * Fetches the docs index (llms.txt) listing every documentation page. Cached at + * module level with a TTL so repeated tool calls within a session reuse it. + */ +export async function fetchDocsIndex(): Promise { + const now = Date.now() + if (llmsTxtCache && llmsTxtCache.expiresAt > now) { + return llmsTxtCache.promise + } + + const promise = fetchText(LLMS_TXT_URL).catch((error) => { + // Drop the failed promise from the cache so the next call retries. + if (llmsTxtCache?.promise === promise) { + llmsTxtCache = undefined + } + throw error + }) + llmsTxtCache = { expiresAt: now + CACHE_TTL_MS, promise } + return promise +} + +/** + * Fetches the full documentation corpus (llms-full.txt): every page concatenated + * into one document, each delimited by a `Source: ` line. ~2 MB. Cached at + * module level with a TTL. Mirrors fetchDocsIndex; used by search_docs to grep + * the whole corpus in a single fetch. + */ +export async function fetchDocsFullText(): Promise { + const now = Date.now() + if (llmsFullTxtCache && llmsFullTxtCache.expiresAt > now) { + return llmsFullTxtCache.promise + } + + const promise = fetchText(LLMS_FULL_TXT_URL).catch((error) => { + if (llmsFullTxtCache?.promise === promise) { + llmsFullTxtCache = undefined + } + throw error + }) + llmsFullTxtCache = { expiresAt: now + CACHE_TTL_MS, promise } + return promise +} + +/** + * Fetches a single docs page as raw markdown. `path` may be a full URL or a + * /docs/... path; it is normalized to a `.md` URL. Cached per resolved URL. + */ +export async function fetchDocsPage(path: string): Promise { + const url = normalizeDocsUrl(path) + const now = Date.now() + const cached = pageCache.get(url) + if (cached && cached.expiresAt > now) { + return cached.promise + } + + const promise = fetchText(url) + .then((content) => sanitizeDocsMarkdownLinks(content, url)) + .catch((error) => { + if (pageCache.get(url)?.promise === promise) { + pageCache.delete(url) + } + throw error + }) + pageCache.set(url, { expiresAt: now + CACHE_TTL_MS, promise }) + return promise +} + +async function fetchText(url: string): Promise { + const response = await fetch(url) + if (!response.ok) { + throw new Error(`Request to ${url} failed with status ${response.status}`) + } + return await response.text() +} + +/** + * Normalizes a user/model-supplied docs reference to a fully-qualified `.md` + * URL on the docs origin. Accepts: + * - `https://www.windmill.dev/docs/core_concepts/jobs` + * - `/docs/core_concepts/jobs.md` + * - `docs/core_concepts/jobs` + */ +export function normalizeDocsUrl(input: string): string { + let value = input.trim() + + if (/^https?:\/\//i.test(value)) { + // Strip the origin so we can re-anchor to DOCS_ORIGIN and normalize the path. + try { + const parsed = new URL(value) + value = parsed.pathname + } catch { + // Fall through and treat as a path. + } + } + + // Drop any query string or hash fragment. + value = value.split('#')[0].split('?')[0] + + if (!value.startsWith('/')) { + value = `/${value}` + } + + // Strip a trailing slash (but keep the leading one). + if (value.length > 1 && value.endsWith('/')) { + value = value.slice(0, -1) + } + + // Relative links inside the raw markdown reference docusaurus source files + // (e.g. `13_flow_branches.mdx`), but the published routes drop the numeric + // ordering prefixes and use `.md`. + value = stripDocsPathPrefixes(value) + if (value.endsWith('.mdx')) { + value = value.slice(0, -1) + } + + if (!value.endsWith('.md')) { + value = `${value}.md` + } + + return `${DOCS_ORIGIN}${value}` +} + +/** + * The canonical published URL a model should cite for a docs page (the `.md` + * fetch URL without the suffix), e.g. `https://www.windmill.dev/docs/flows/retries`. + */ +export function canonicalDocsPageUrl(path: string): string { + return normalizeDocsUrl(path).replace(/\.md$/i, '') +} + +/** + * Strips docusaurus numeric ordering prefixes (`13_`, `8-`) from each segment of + * a docs path so it matches the published route. Operates on the path only. + */ +function stripDocsPathPrefixes(path: string): string { + return path + .split('/') + .map((segment) => segment.replace(/^\d+[_-]/, '')) + .join('/') +} + +/** + * Rewrites relative/source-file doc links inside raw page markdown to canonical + * published URLs, so the model never echoes a docusaurus source path (e.g. + * `./13_flow_branches.mdx`) into its answer as a broken link. Resolves each link + * relative to the page it came from, strips numeric ordering prefixes, and drops + * the `.md`/`.mdx` extension. Non-doc links (external, images, anchors) are left + * untouched. + */ +export function sanitizeDocsMarkdownLinks(content: string, pageUrl: string): string { + return content.replace(/\]\(([^)\s]+?)(\s+"[^"]*")?\)/g, (match, target: string, title) => { + if (!/\.mdx?($|[#?])/i.test(target)) { + // Only rewrite links to docusaurus source files (.md/.mdx); leave + // images, external URLs and bare anchors untouched. + return match + } + if (/(^|\/)\.\.\//.test(target)) { + // `../` cross-directory links are authored against the docusaurus + // source tree, whose depth differs from the published URL, so strict + // resolution is unreliable. Leave them for the canonical-URL header to + // disambiguate rather than risk rewriting to a wrong path. + return match + } + let resolved: URL + try { + resolved = new URL(target, pageUrl) + } catch { + return match + } + if (resolved.origin !== DOCS_ORIGIN || !resolved.pathname.startsWith('/docs/')) { + return match + } + const pathname = stripDocsPathPrefixes(resolved.pathname).replace(/\.mdx?$/i, '') + return `](${DOCS_ORIGIN}${pathname}${resolved.hash}${title ?? ''})` + }) +} + +export interface DocsHeading { + level: number + title: string + /** Character offset of the start of the heading line within the document. */ + startIndex: number +} + +/** + * Parses the markdown headings (`#`–`####`) of a docs page, ignoring any + * heading-like lines that appear inside fenced code blocks (``` fences), which + * are common in docs pages (e.g. `# comment` inside a python sample). + */ +export function parseDocsHeadings(content: string): DocsHeading[] { + const headings: DocsHeading[] = [] + let offset = 0 + let inFence = false + let fenceMarker = '' + + const lines = content.split('\n') + for (const line of lines) { + const fence = matchFence(line) + if (fence) { + if (!inFence) { + inFence = true + fenceMarker = fence + } else if (line.trimStart().startsWith(fenceMarker)) { + inFence = false + fenceMarker = '' + } + offset += line.length + 1 + continue + } + + if (!inFence) { + const match = /^(#{1,4})\s+(.*\S)\s*$/.exec(line) + if (match) { + headings.push({ + level: match[1].length, + title: match[2].trim(), + startIndex: offset + }) + } + } + + offset += line.length + 1 + } + + return headings +} + +function matchFence(line: string): string | undefined { + const trimmed = line.trimStart() + const match = /^(`{3,}|~{3,})/.exec(trimmed) + return match ? match[1] : undefined +} + +/** + * Builds a human-readable outline of a page's headings, including an approximate + * character size for each section. Used when a page is too large to return whole. + */ +export function buildDocsOutline(content: string): string { + const headings = parseDocsHeadings(content) + if (headings.length === 0) { + return '(no markdown headings found on this page)' + } + + const lines = headings.map((heading, index) => { + const sectionEnd = sectionEndIndex(content, headings, index) + const approxChars = sectionEnd - heading.startIndex + const indent = ' '.repeat(Math.max(0, heading.level - 1)) + return `${indent}- ${heading.title} (~${approxChars} chars)` + }) + + return lines.join('\n') +} + +function sectionEndIndex(content: string, headings: DocsHeading[], index: number): number { + const heading = headings[index] + // A section ends at the next heading of the same or higher (shallower) level. + for (let i = index + 1; i < headings.length; i++) { + if (headings[i].level <= heading.level) { + return headings[i].startIndex + } + } + return content.length +} + +/** Normalizes a heading title for tolerant, case/punctuation-insensitive matching. */ +function normalizeHeadingTitle(title: string): string { + return title + .toLowerCase() + .replace(/[^a-z0-9]+/g, ' ') + .trim() +} + +/** + * Extracts the content of the section whose heading matches `section`, from the + * matching heading up to the next heading of the same or higher level. Matching + * is case-insensitive and tolerant of minor punctuation differences. Returns + * `undefined` when no heading matches. + */ +export function extractDocsSection(content: string, section: string): string | undefined { + const headings = parseDocsHeadings(content) + const target = normalizeHeadingTitle(section) + if (target.length === 0) { + return undefined + } + + let matchIndex = headings.findIndex( + (heading) => normalizeHeadingTitle(heading.title) === target + ) + if (matchIndex === -1) { + // Fall back to a contains match so "Result streaming" matches "Result". + matchIndex = headings.findIndex((heading) => + normalizeHeadingTitle(heading.title).includes(target) + ) + } + if (matchIndex === -1) { + return undefined + } + + const start = headings[matchIndex].startIndex + const end = sectionEndIndex(content, headings, matchIndex) + return content.slice(start, end).trim() +} + +const READ_DOCS_PAGE_TOOL: ChatCompletionTool = { + type: 'function', + function: { + name: 'read_docs_page', + description: + 'Fetch the raw markdown of a single Windmill documentation page. Provide the `path` (or full URL) of a page found via search_docs. If the page is large, this returns its list of section headings instead of the full content; call again with the `section` argument set to one of those headings to read that section.', + parameters: { + type: 'object', + properties: { + path: { + type: 'string', + description: + 'The docs page to read, as a path (e.g. /docs/core_concepts/jobs) or full URL (e.g. https://www.windmill.dev/docs/core_concepts/jobs).' + }, + section: { + type: 'string', + description: + 'Optional. A heading title from the page outline to read just that section instead of the full page.' + } + }, + required: ['path'] + } + } +} + +export const readDocsPageTool: Tool<{}> = { + def: READ_DOCS_PAGE_TOOL, + fn: async ({ args, toolId, toolCallbacks }) => { + const path = typeof args?.path === 'string' ? args.path : '' + const section = typeof args?.section === 'string' && args.section.trim() ? args.section : undefined + toolCallbacks.setToolStatus(toolId, { + content: section ? `Reading docs section "${section}"...` : 'Reading documentation page...' + }) + try { + if (!path.trim()) { + return 'No documentation page path was provided. Provide a `path` — e.g. a `Source` URL returned by search_docs.' + } + const content = await fetchDocsPage(path) + toolCallbacks.setToolStatus(toolId, { content: 'Read documentation page' }) + const canonicalUrl = canonicalDocsPageUrl(path) + const header = `Source page — cite this URL when referencing this page: ${canonicalUrl}\n\n` + return header + renderDocsPageResult(content, section) + } catch (error) { + toolCallbacks.setToolStatus(toolId, { + content: 'Error reading documentation page', + error: 'Error reading documentation page' + }) + console.error('Error reading documentation page:', error) + const errorMessage = + error instanceof Error ? error.message : 'An error occurred while reading the documentation page' + return `Failed to read documentation page: ${errorMessage}, pursuing with the user request...` + } + } +} + +/** + * Decides what to return for read_docs_page: a requested section, the full page, + * or an outline asking the model to pick a section. + */ +export function renderDocsPageResult(content: string, section?: string): string { + if (section) { + const extracted = extractDocsSection(content, section) + if (extracted !== undefined) { + return extracted + } + return [ + `No section matching "${section}" was found on this page. Available sections:`, + '', + buildDocsOutline(content) + ].join('\n') + } + + // Gate on the page body only; the caller may prepend a short "Source page" + // header, so the returned payload can exceed this limit by that header's + // length. This threshold only decides whole-page vs. outline, so the small + // overshoot is immaterial. + if (content.length <= FULL_PAGE_CHAR_LIMIT) { + return content + } + + return [ + 'This documentation page is large. Below is its list of sections with approximate sizes.', + 'Call read_docs_page again with the same path and a `section` set to one of these headings to read that section.', + '', + buildDocsOutline(content) + ].join('\n') +} + +// --------------------------------------------------------------------------- +// Full-text docs search (search_docs) +// +// Discovery primitive for the `search` ask variant: instead of dumping the whole +// llms.txt index, grep the full corpus (llms-full.txt) for the user's keywords +// and return only small matching snippets plus each page's `Source:` URL. The +// model then cites that URL directly or passes it to read_docs_page for more. +// --------------------------------------------------------------------------- + +const SOURCE_LINE_RE = /^Source:\s*(\S+)\s*$/ +// In llms-full.txt every page's `Source:` line is preceded by a category-header +// lead-in: `...page body...\n\n---\n\n## \n\nSource: `. Splitting +// on `Source:` lines leaves that lead-in on the *previous* page, so strip a +// trailing `---` + level-2-heading block to avoid mis-attributing the next +// page's category title to the previous page. +const TRAILING_LEAD_IN_RE = /\n+-{3,}[ \t]*\n+#{2}[ \t]+.*[ \t]*\n*$/ + +export interface DocsFullPage { + url: string + title: string + body: string +} + +export interface DocsSearchResult { + url: string + title: string + /** Higher = more relevant. Distinct query terms matched dominate raw occurrences. */ + score: number + snippets: string[] +} + +/** + * Splits the llms-full.txt corpus into per-page records keyed by the `Source:` + * URL. Content before the first `Source:` line (the corpus preamble) is dropped. + */ +export function parseDocsFullText(fullText: string): DocsFullPage[] { + const pages: DocsFullPage[] = [] + let url: string | undefined + let buffer: string[] = [] + + const flush = () => { + if (url === undefined) { + return + } + const body = buffer.join('\n').replace(TRAILING_LEAD_IN_RE, '').trim() + if (body.length > 0) { + pages.push({ url, title: firstHeading(body) ?? url, body }) + } + } + + for (const line of fullText.split('\n')) { + const match = SOURCE_LINE_RE.exec(line) + if (match) { + flush() + url = match[1] + buffer = [] + continue + } + if (url !== undefined) { + buffer.push(line) + } + } + flush() + return pages +} + +function firstHeading(body: string): string | undefined { + for (const line of body.split('\n')) { + const match = /^#{1,6}\s+(.*\S)\s*$/.exec(line) + if (match) { + return match[1].trim() + } + } + return undefined +} + +/** + * Ranks docs pages for a keyword query. The query is split into distinct terms; + * a page's score is `distinctTermsMatched` (dominant) then total occurrences. + * Pages covering every term are preferred over partial matches. Each result + * carries up to `maxSnippetsPerPage` of its most term-dense lines. + */ +export function searchDocsPages( + pages: DocsFullPage[], + query: string, + opts: { maxPages?: number; maxSnippetsPerPage?: number; maxSnippetChars?: number } = {} +): DocsSearchResult[] { + const maxPages = opts.maxPages ?? SEARCH_MAX_PAGES + const maxSnippetsPerPage = opts.maxSnippetsPerPage ?? SEARCH_MAX_SNIPPETS_PER_PAGE + const maxSnippetChars = opts.maxSnippetChars ?? SEARCH_MAX_SNIPPET_CHARS + + const terms = tokenizeQuery(query) + if (terms.length === 0) { + return [] + } + + interface Scored extends DocsSearchResult { + distinctTerms: number + order: number + } + const scored: Scored[] = [] + + pages.forEach((page, order) => { + const lowerBody = page.body.toLowerCase() + let distinctTerms = 0 + let occurrences = 0 + for (const term of terms) { + const count = countOccurrences(lowerBody, term) + if (count > 0) { + distinctTerms += 1 + occurrences += count + } + } + if (distinctTerms === 0) { + return + } + scored.push({ + url: page.url, + title: page.title, + // distinctTerms dominates so a page matching all terms always outranks + // one matching fewer, regardless of raw occurrence counts. + score: distinctTerms * 1_000_000 + occurrences, + distinctTerms, + order, + snippets: selectSnippets(page.body, terms, maxSnippetsPerPage, maxSnippetChars) + }) + }) + + // Prefer pages that cover every query term; fall back to partial matches only + // when nothing covers all of them. + const fullCoverage = scored.filter((entry) => entry.distinctTerms === terms.length) + const pool = fullCoverage.length > 0 ? fullCoverage : scored + + pool.sort((a, b) => b.score - a.score || a.order - b.order) + + return pool + .slice(0, maxPages) + .map(({ url, title, score, snippets }) => ({ url, title, score, snippets })) +} + +/** Splits a query into distinct, lowercased, non-empty terms. */ +function tokenizeQuery(query: string): string[] { + return Array.from( + new Set( + query + .toLowerCase() + .split(/\s+/) + .map((term) => term.trim()) + .filter((term) => term.length > 0) + ) + ) +} + +function countOccurrences(haystack: string, needle: string): number { + if (needle.length === 0) { + return 0 + } + let count = 0 + let index = haystack.indexOf(needle) + while (index !== -1) { + count += 1 + index = haystack.indexOf(needle, index + needle.length) + } + return count +} + +/** + * Picks the most term-dense lines of a page body as snippets, in document order, + * deduped, each trimmed to `maxChars` around the first matched term. + */ +function selectSnippets( + body: string, + terms: string[], + maxSnippets: number, + maxChars: number +): string[] { + interface LineHit { + text: string + distinct: number + order: number + } + const hits: LineHit[] = [] + + body.split('\n').forEach((line, order) => { + const lower = line.toLowerCase() + let distinct = 0 + for (const term of terms) { + if (lower.includes(term)) { + distinct += 1 + } + } + if (distinct === 0) { + return + } + const text = makeSnippet(line, terms, maxChars) + if (text.length > 0) { + hits.push({ text, distinct, order }) + } + }) + + hits.sort((a, b) => b.distinct - a.distinct || a.order - b.order) + + const seen = new Set() + const result: string[] = [] + for (const hit of hits) { + if (seen.has(hit.text)) { + continue + } + seen.add(hit.text) + result.push(hit.text) + if (result.length >= maxSnippets) { + break + } + } + return result +} + +/** + * Collapses a matched line to a single-line snippet of at most `maxChars`, + * windowed around the first matched term (with ellipses) when the line is long. + */ +export function makeSnippet(line: string, terms: string[], maxChars: number): string { + const collapsed = line.replace(/\s+/g, ' ').trim() + if (collapsed.length <= maxChars) { + return collapsed + } + + const lower = collapsed.toLowerCase() + let firstIndex = -1 + for (const term of terms) { + const index = lower.indexOf(term) + if (index !== -1 && (firstIndex === -1 || index < firstIndex)) { + firstIndex = index + } + } + if (firstIndex === -1) { + return `${collapsed.slice(0, maxChars).trimEnd()}…` + } + + const start = Math.max(0, firstIndex - Math.floor(maxChars / 3)) + const end = Math.min(collapsed.length, start + maxChars) + const prefix = start > 0 ? '…' : '' + const suffix = end < collapsed.length ? '…' : '' + return `${prefix}${collapsed.slice(start, end).trim()}${suffix}` +} + +export interface DocsIndexEntry { + title: string + url: string + description: string +} + +// A line in llms.txt: `- [Title](https://.../page.md): question-phrased description`. +const INDEX_ENTRY_RE = /^\s*-\s*\[([^\]]+)\]\(([^)\s]+)\)\s*:?\s*(.*)$/ + +/** Parses the llms.txt index into per-page entries (title, URL, description). */ +export function parseDocsIndex(indexText: string): DocsIndexEntry[] { + const entries: DocsIndexEntry[] = [] + for (const line of indexText.split('\n')) { + const match = INDEX_ENTRY_RE.exec(line) + if (!match) { + continue + } + const [, title, url, description] = match + if (!url.includes('/docs/')) { + continue + } + entries.push({ title: title.trim(), url: url.trim(), description: description.trim() }) + } + return entries +} + +/** + * Ranks index entries for a query by matching its terms against each entry's + * title and description. Title matches weigh more than description matches. + * The description becomes the result's single snippet. This recovers the + * "named feature" discovery that full-text grep misses when the model searches + * the wrong keywords (e.g. finding "AI agents" for "LLM decides which script"). + */ +export function searchDocsIndex( + entries: DocsIndexEntry[], + query: string, + opts: { maxPages?: number } = {} +): DocsSearchResult[] { + const maxPages = opts.maxPages ?? SEARCH_MAX_PAGES + const terms = tokenizeQuery(query) + if (terms.length === 0) { + return [] + } + + interface Scored extends DocsSearchResult { + distinctTerms: number + order: number + } + const scored: Scored[] = [] + + entries.forEach((entry, order) => { + const title = entry.title.toLowerCase() + const description = entry.description.toLowerCase() + let distinctTerms = 0 + let score = 0 + for (const term of terms) { + const inTitle = title.includes(term) + const inDescription = description.includes(term) + if (inTitle || inDescription) { + distinctTerms += 1 + score += (inTitle ? 5 : 0) + (inDescription ? 1 : 0) + } + } + if (distinctTerms === 0) { + return + } + scored.push({ + url: entry.url, + title: entry.title, + score: distinctTerms * 1_000_000 + score, + distinctTerms, + order, + snippets: entry.description ? [entry.description] : [] + }) + }) + + const fullCoverage = scored.filter((entry) => entry.distinctTerms === terms.length) + const pool = fullCoverage.length > 0 ? fullCoverage : scored + pool.sort((a, b) => b.score - a.score || a.order - b.order) + + return pool + .slice(0, maxPages) + .map(({ url, title, score, snippets }) => ({ url, title, score, snippets })) +} + +/** Strips the `.md` suffix and trailing slash so index/body URLs dedupe. */ +function canonicalSearchUrl(url: string): string { + return url.replace(/\.md$/i, '').replace(/\/$/, '') +} + +/** + * Merges full-text (body) results with index-description results. Body matches + * come first (concrete content hits), then index-only matches fill remaining + * slots — so a named feature surfaced only by its index entry still appears even + * when body grep landed on the wrong pages. + */ +export function mergeDocsSearchResults( + bodyResults: DocsSearchResult[], + indexResults: DocsSearchResult[], + maxPages = SEARCH_MAX_PAGES +): DocsSearchResult[] { + const seen = new Set(bodyResults.map((result) => canonicalSearchUrl(result.url))) + const merged = [...bodyResults] + for (const entry of indexResults) { + const key = canonicalSearchUrl(entry.url) + if (seen.has(key)) { + continue + } + seen.add(key) + merged.push(entry) + } + return merged.slice(0, maxPages) +} + +/** Renders search results as the string returned to the model. */ +export function formatDocsSearchResults(query: string, results: DocsSearchResult[]): string { + if (results.length === 0) { + return `No documentation pages matched "${query}". Try fewer or more general keywords (a single distinctive term often works best).` + } + + const blocks = results.map((result) => { + const lines = [`## ${result.title}`, `Source: ${result.url}`] + for (const snippet of result.snippets) { + lines.push(` - ${snippet}`) + } + return lines.join('\n') + }) + + return [ + `Found ${results.length} documentation page(s) matching "${query}", most relevant first:`, + '', + blocks.join('\n\n'), + '', + 'Cite the exact "Source" URL when referencing a page. If these snippets are not enough, call read_docs_page with a Source URL to read the full page or a section.' + ].join('\n') +} + +const SEARCH_DOCS_TOOL: ChatCompletionTool = { + type: 'function', + function: { + name: 'search_docs', + description: + 'Full-text search across the entire Windmill documentation. Provide one or more keywords; returns the most relevant docs pages, each with its Source URL and short matching snippets. Use this FIRST to find relevant pages by their content (a flag, function, error message, config key or concept). If the snippets answer the question, answer directly; otherwise call read_docs_page with a returned Source URL to read more.', + parameters: { + type: 'object', + properties: { + query: { + type: 'string', + description: + 'Keywords to search for in the documentation body, e.g. "chromium worker tag" or "retry exponential backoff". Fewer, more distinctive words match better.' + } + }, + required: ['query'] + } + } +} + +export const searchDocsTool: Tool<{}> = { + def: SEARCH_DOCS_TOOL, + fn: async ({ args, toolId, toolCallbacks }) => { + const query = typeof args?.query === 'string' ? args.query.trim() : '' + toolCallbacks.setToolStatus(toolId, { + content: query ? `Searching documentation for "${query}"...` : 'Searching documentation...' + }) + try { + if (!query) { + return 'No search query was provided. Provide a `query` of one or more keywords.' + } + const bodyResults = searchDocsPages(parseDocsFullText(await fetchDocsFullText()), query, { + maxPages: 5 + }) + // Also match the (small) index titles/descriptions to surface named + // features that body grep misses. Best-effort: a failed index fetch + // still leaves full-text results. + let indexResults: DocsSearchResult[] = [] + try { + indexResults = searchDocsIndex(parseDocsIndex(await fetchDocsIndex()), query, { + maxPages: 4 + }) + } catch (indexError) { + console.error('Error searching documentation index:', indexError) + } + const results = mergeDocsSearchResults(bodyResults, indexResults) + toolCallbacks.setToolStatus(toolId, { + content: + results.length > 0 ? `Found ${results.length} matching page(s)` : 'No matching pages found' + }) + return formatDocsSearchResults(query, results) + } catch (error) { + toolCallbacks.setToolStatus(toolId, { + content: 'Error searching documentation', + error: 'Error searching documentation' + }) + console.error('Error searching documentation:', error) + const errorMessage = + error instanceof Error ? error.message : 'An error occurred while searching the documentation' + return `Failed to search documentation: ${errorMessage}, pursuing with the user request...` + } + } +} diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 724af63e42..284e7aafda 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -72,6 +72,7 @@ import { type ToolCallbacks, type ToolDisplayAction } from '../shared' +import { searchDocsTool, readDocsPageTool } from '../docs/core' import type { ContextElement } from '../context' import { getDatatableTools } from '../datatableTools' import { UserDraft } from '$lib/userDraft.svelte' @@ -677,6 +678,13 @@ Rules: : '' } +Documentation: +- Use search_docs to look up how a Windmill feature works in the official documentation (a flag, concept, function, or "does Windmill support X") instead of guessing about product behavior. It returns matching doc snippets with their Source URL; call read_docs_page with a Source URL to read the full page (or a section, if it returns headings). Cite the Source URL when you rely on it. +- Complete your response with precisions about how it works based on the documentation. Also drop a link to the relevant documentation if possible. +- If the user asks about something that you are unsure about, say that you are not sure about the answer and suggest to ask the question to the windmill team. +- If the first search returns nothing useful, retry with different or broader keywords before giving up. +- If the documentation does not cover the user's question, say so clearly rather than inventing an answer, and suggest asking the Windmill team. + Flows: - read_workspace_item returns compact flow JSON. Inline script bodies appear as "inline_script.". - Use read_flow_module_code and set_flow_module_code for inline script bodies. @@ -1494,6 +1502,8 @@ export const globalTools: Tool<{}>[] = [ } }, createSearchHubScriptsTool(false), + searchDocsTool, + readDocsPageTool, { def: createToolDef( askUserQuestionSchema, diff --git a/frontend/src/lib/components/copilot/chat/navigator/core.ts b/frontend/src/lib/components/copilot/chat/navigator/core.ts index 6da158e6de..945f994862 100644 --- a/frontend/src/lib/components/copilot/chat/navigator/core.ts +++ b/frontend/src/lib/components/copilot/chat/navigator/core.ts @@ -4,6 +4,7 @@ import type { ChatCompletionUserMessageParam } from 'openai/resources/index.mjs' import { createSearchWorkspaceTool, createGetRunnableDetailsTool, type Tool } from '../shared' +import { readDocsPageTool, searchDocsTool } from '../docs/core' import { ResourceService } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { get } from 'svelte/store' @@ -16,13 +17,14 @@ Windmill is an open-source developer platform for building internal tools, API i You have access to these tools: 1. View current buttons and inputs on the page (get_triggerable_components) 2. Execute buttons and inputs (trigger_component) -3. Get documentation for user requests (get_documentation) -4. Change the AI mode to the one specified (change_mode) -5. Search for scripts and flows in the workspace (search_workspace) -6. Get detailed information about a specific script or flow (get_runnable_details) +3. Search the documentation (search_docs) +4. Read a documentation page (read_docs_page) +5. Change the AI mode to the one specified (change_mode) +6. Search for scripts and flows in the workspace (search_workspace) +7. Get detailed information about a specific script or flow (get_runnable_details) INSTRUCTIONS: -- When users ask about application features or concepts, first use get_documentation internally to retrieve accurate information about how to fulfill the user's request. +- When users ask about application features or concepts, first use search_docs (with a few keywords) and, when a snippet is not enough, read_docs_page on a returned Source URL to retrieve accurate information about how to fulfill the user's request. - Then immediately use the available tools to guide the user through the application. Do not wait for the user's confirmation before taking action. - If you detect a confirmation modal that needs user confirmation, stop the navigation and let the user know that the action is pending confirmation. - Use get_triggerable_components to understand available options, and then trigger the components using trigger_component. Then wait a moment before rescanning the current page, and then continue with the next step. Do this 5 times max. @@ -59,30 +61,12 @@ When you complete the user's request, do not say "I created..." or "I updated... Example of good behavior: - User: "How can I set my AI providers?" -- You: +- You: - You: - You: - You: "" ` -const GET_DOCUMENTATION_TOOL: ChatCompletionTool = { - type: 'function', - function: { - name: 'get_documentation', - description: 'Get the documentation for the user request', - parameters: { - type: 'object', - properties: { - request: { - type: 'string', - description: 'The user request' - } - }, - required: ['request'] - } - } -} - // Tool definitions const GET_TRIGGERABLE_COMPONENTS_TOOL: ChatCompletionTool = { type: 'function', @@ -234,47 +218,6 @@ function triggerComponent(args: { id: string; value: string }): string { } } -async function getDocumentation(args: { request: string }): Promise { - const retrieval = await fetch('/api/inkeep', { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - query: args.request - }) - }) - - if (!retrieval.ok) { - const errorText = await retrieval.text() - throw new Error(errorText) - } - - const data = await retrieval.json() - if (!data.choices?.[0]?.message?.content) { - return 'No documentation found for this request' - } - - // Parse the raw response - const raw = data.choices[0].message.content - const parsed = JSON.parse(raw) - - // Clean up the response to include only essential information - if (parsed.content && Array.isArray(parsed.content)) { - const cleanedContent = parsed.content.map((item: any) => ({ - title: item.title, - url: item.url, - content: item.source?.content.map((c: any) => c.text).join('\n') || [] - })) - // Limit the response to 30000 characters max - const stringified = JSON.stringify({ content: cleanedContent }).slice(0, 30000) - - return stringified - } - - return data.choices[0].message.content -} - async function getAvailableResources(args: { resource_type: string }): Promise { const resources = await ResourceService.listResource({ workspace: get(workspaceStore) as string, @@ -318,27 +261,6 @@ const getCurrentPageNameTool: Tool<{}> = { } } -export const getDocumentationTool: Tool<{}> = { - def: GET_DOCUMENTATION_TOOL, - fn: async ({ args, toolId, toolCallbacks }) => { - toolCallbacks.setToolStatus(toolId, { content: 'Getting documentation...' }) - try { - const docResult = await getDocumentation(args) - toolCallbacks.setToolStatus(toolId, { content: 'Retrieved documentation' }) - return docResult - } catch (error) { - toolCallbacks.setToolStatus(toolId, { - content: 'Error getting documentation', - error: 'Error getting documentation' - }) - console.error('Error getting documentation:', error) - const errorMessage = - error instanceof Error ? error.message : 'An error occurred while getting documentation' - return `Failed to get documentation: ${errorMessage}, pursuing with the user request...` - } - } -} - const getAvailableResourcesTool: Tool<{}> = { def: GET_AVAILABLE_RESOURCES_TOOL, fn: async ({ args, toolId, toolCallbacks }) => { @@ -361,7 +283,8 @@ const getAvailableResourcesTool: Tool<{}> = { export const navigatorTools: Tool<{}>[] = [ getTriggerableComponentsTool, triggerComponentTool, - getDocumentationTool, + searchDocsTool, + readDocsPageTool, getCurrentPageNameTool, getAvailableResourcesTool, createSearchWorkspaceTool(),