feat: expose windmill api endpoint catalog to global ai chat (#10199)

* feat: expose windmill api endpoint catalog to global ai chat

* fix: guard variable reads and deletes in ai chat api catalog

* fix: clarify api catalog prompt example for run result access

* fix: block deleteScriptByHash and document resource read boundary
This commit is contained in:
hugocasa
2026-07-20 11:31:51 +02:00
committed by GitHub
parent b5e69ffba6
commit 83a354f831
3 changed files with 604 additions and 1 deletions
@@ -0,0 +1,229 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
const { listMcpToolsMock } = vi.hoisted(() => ({
listMcpToolsMock: vi.fn()
}))
vi.mock('../shared', () => ({
createToolDef: (_schema: unknown, name: string, description: string) => ({
type: 'function',
function: { name, description, parameters: {} }
})
}))
vi.mock('$lib/gen', () => ({
McpService: {
listMcpTools: listMcpToolsMock
}
}))
import { apiCatalogTools, clearApiCatalogCache } from './apiCatalogTools'
const CATALOG = [
{
name: 'listWorkers',
description: 'List workers',
instructions: 'List all workers with their ping status',
path: '/workers/list',
method: 'GET',
query_params_schema: {
type: 'object',
properties: { page: { type: 'integer' }, per_page: { type: 'integer' } }
}
},
{
name: 'getJob',
description: 'Get job details',
instructions: '',
path: '/w/{workspace}/jobs_u/get/{id}',
method: 'GET',
path_params_schema: {
type: 'object',
properties: { workspace: { type: 'string' }, id: { type: 'string' } },
required: ['workspace', 'id']
}
},
{
name: 'deleteSchedule',
description: 'Delete a schedule',
instructions: '',
path: '/w/{workspace}/schedules/delete/{path}',
method: 'DELETE',
path_params_schema: {
type: 'object',
properties: { workspace: { type: 'string' }, path: { type: 'string' } },
required: ['workspace', 'path']
}
},
{
name: 'createFlow',
description: 'Create a flow',
instructions: '',
path: '/w/{workspace}/flows/create',
method: 'POST'
},
{
name: 'getVariable',
description: 'Get variable',
instructions: '',
path: '/w/{workspace}/variables/get/{path}',
method: 'GET'
},
{
name: 'deleteScriptByHash',
description: 'Delete a script by hash',
instructions: '',
path: '/w/{workspace}/scripts/delete/h/{hash}',
method: 'POST',
path_params_schema: {
type: 'object',
properties: { workspace: { type: 'string' }, hash: { type: 'string' } },
required: ['workspace', 'hash']
}
},
{
name: 'runFlowByPath',
description: 'Run flow by path',
instructions: 'Trigger a run of a deployed flow',
path: '/w/{workspace}/jobs/run/f/{path}',
method: 'POST',
path_params_schema: {
type: 'object',
properties: { workspace: { type: 'string' }, path: { type: 'string' } },
required: ['workspace', 'path']
},
body_schema: {
type: 'object',
properties: { args: { type: 'object' } }
}
}
]
function createToolCallbacks() {
return {
setToolStatus: vi.fn(),
removeToolStatus: vi.fn()
} as any
}
function getTool(name: string) {
const tool = apiCatalogTools.find((entry) => entry.def.function.name === name)
if (!tool) throw new Error(`${name} tool not found`)
return tool
}
async function run(name: string, args: Record<string, unknown>) {
const raw = await getTool(name).fn({
args,
workspace: 'test-ws',
helpers: {},
toolCallbacks: createToolCallbacks(),
toolId: 'tool-1'
})
return JSON.parse(raw)
}
beforeEach(() => {
vi.clearAllMocks()
clearApiCatalogCache()
listMcpToolsMock.mockResolvedValue(CATALOG)
vi.unstubAllGlobals()
})
describe('search_api_endpoints', () => {
it('matches on name/path tokens, plural-insensitively, and excludes covered endpoints', async () => {
const result = await run('search_api_endpoints', { query: 'worker' })
expect(result.matches.map((m: any) => m.name)).toEqual(['listWorkers'])
expect(result.matches[0].endpoint).toBe('GET /workers/list')
expect(result.matches[0].params).toEqual(['page', 'per_page'])
expect(result.matches[0].instructions).toContain('ping status')
const flows = await run('search_api_endpoints', { query: 'create flow' })
expect(flows.matches.map((m: any) => m.name)).not.toContain('createFlow')
expect(flows.covered_by_dedicated_tools).toContain('createFlow → use write_flow')
})
it('returns endpoint categories when nothing matches', async () => {
const result = await run('search_api_endpoints', { query: 'kubernetes' })
expect(result.matches).toEqual([])
expect(result.hint).toContain('workers')
expect(result.hint).toContain('jobs')
})
})
describe('call_api_get', () => {
it('rejects covered, unknown, and non-GET endpoints with a pointer', async () => {
const covered = await run('call_api_get', { name: 'createFlow' })
expect(covered.error).toContain('write_flow')
const unknown = await run('call_api_get', { name: 'nope' })
expect(unknown.error).toContain('search_api_endpoints')
const mutating = await run('call_api_get', { name: 'runFlowByPath' })
expect(mutating.error).toContain('call_api_endpoint')
const deleting = await run('call_api_endpoint', { name: 'deleteSchedule' })
expect(deleting.error).toContain('delete_workspace_item')
const byHash = await run('call_api_endpoint', { name: 'deleteScriptByHash' })
expect(byHash.error).toContain('delete_workspace_item')
const search = await run('search_api_endpoints', { query: 'delete script' })
expect(search.matches.map((m: any) => m.name)).not.toContain('deleteScriptByHash')
})
it('refuses variable reads so variable values never reach the model', async () => {
const result = await run('call_api_get', { name: 'getVariable' })
expect(result.success).toBe(false)
expect(result.error).toContain('never readable')
const search = await run('search_api_endpoints', { query: 'variable' })
expect(search.matches.map((m: any) => m.name)).not.toContain('getVariable')
})
it('returns the endpoint schema when a required path param is missing', async () => {
const result = await run('call_api_get', { name: 'getJob' })
expect(result.success).toBe(false)
expect(result.error).toContain('id')
expect(result.schema.path_params_schema.required).toContain('id')
})
it('substitutes path params and sends the rest as query params', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
headers: new Headers({ 'content-type': 'application/json' }),
json: async () => [{ worker: 'w1' }]
})
vi.stubGlobal('fetch', fetchMock)
const result = await run('call_api_get', { name: 'listWorkers', params: { page: 2 } })
expect(fetchMock).toHaveBeenCalledWith('/api/workers/list?page=2', { method: 'GET' })
expect(result).toEqual({ success: true, data: [{ worker: 'w1' }] })
})
})
describe('call_api_endpoint', () => {
it('requires confirmation and executes mutating endpoints with a body', async () => {
expect(getTool('call_api_endpoint').requiresConfirmation).toBe(true)
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
headers: new Headers({ 'content-type': 'text/plain' }),
text: async () => 'job-id-1'
})
vi.stubGlobal('fetch', fetchMock)
const result = await run('call_api_endpoint', {
name: 'runFlowByPath',
params: { path: 'u/me/myflow' },
body: { args: { n: 1 } }
})
expect(fetchMock).toHaveBeenCalledWith('/api/w/test-ws/jobs/run/f/u%2Fme%2Fmyflow', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ args: { n: 1 } })
})
expect(result).toEqual({ success: true, data: 'job-id-1' })
})
it('redirects GET endpoints to call_api_get', async () => {
const result = await run('call_api_endpoint', { name: 'listWorkers' })
expect(result.error).toContain('call_api_get')
})
})
@@ -0,0 +1,369 @@
import { z } from 'zod'
import { McpService, type EndpointTool } from '$lib/gen'
import { createToolDef, type Tool } from '../shared'
/**
* Generic access to the backend's MCP endpoint catalog (the endpoints marked
* `x-mcp-tool` in openapi.yaml) as three small static tools — search, GET
* call, mutating call — instead of one registered tool per endpoint. Search
* results carry parameter names and usage instructions; full parameter schemas
* enter the model's context only when a call fails validation. This keeps the
* per-iteration tool-schema cost constant.
*/
// Endpoints whose job a dedicated global tool already does, plus the variable
// read endpoints. Hidden from search and refused at call time: the authoring
// and delete ones would bypass the draft lifecycle (conflict detection,
// explicit deploy, draft cleanup on delete), the variable reads would expose
// variable values to the model (getVariable even decrypts secrets by default),
// and the rest are exact duplicates that would fragment behavior across two
// code paths. getResource stays available: resource values are readable in
// this chat by design (read_workspace_item returns them too) — secrets belong
// in variables referenced as "$var:path", which a plain get leaves unresolved.
const COVERED_ENDPOINTS: Record<string, string> = {
getVariable: 'read_workspace_item (variable values are never readable in chat)',
listVariable: 'list_workspace_items (variable values are never readable in chat)',
deleteScriptByPath: 'delete_workspace_item',
deleteScriptByHash: 'delete_workspace_item',
deleteFlowByPath: 'delete_workspace_item',
deleteSchedule: 'delete_workspace_item',
deleteVariable: 'delete_workspace_item',
deleteResource: 'delete_workspace_item',
createScript: 'write_script',
createFlow: 'write_flow',
updateFlow: 'patch_flow_json or write_flow',
createApp: 'init_app and the app draft tools',
updateApp: 'write_app_file / write_app_runnable',
createVariable: 'write_variable',
updateVariable: 'write_variable',
createResource: 'write_resource',
updateResource: 'write_resource',
createSchedule: 'write_schedule',
updateSchedule: 'write_schedule',
searchDocs: 'search_docs',
readDocsPage: 'read_docs_page',
listJobs: 'list_runs',
getJobLogs: 'get_job_logs',
runScriptPreviewAndWaitResult: 'test_run_script'
}
const MAX_SEARCH_RESULTS = 10
const MAX_DESCRIPTION_CHARS = 200
const MAX_INSTRUCTIONS_CHARS = 400
const MAX_RESULT_CHARS = 20_000
let catalogCache: { workspace: string; endpoints: EndpointTool[] } | undefined
async function loadCatalog(workspace: string): Promise<EndpointTool[]> {
if (catalogCache?.workspace !== workspace) {
const endpoints = await McpService.listMcpTools({ workspace })
catalogCache = { workspace, endpoints }
}
return catalogCache.endpoints
}
export function clearApiCatalogCache() {
catalogCache = undefined
}
function tokenize(text: string): string[] {
return text
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
.toLowerCase()
.split(/[^a-z0-9]+/)
.filter((t) => t.length > 1)
}
// Cheap plural-insensitive comparison so "workers" matches "worker" and vice versa.
function tokenMatches(token: string, queryToken: string): boolean {
const strip = (t: string) => (t.length > 3 && t.endsWith('s') ? t.slice(0, -1) : t)
return strip(token) === strip(queryToken)
}
function pathSegments(path: string): string[] {
return path.split('/').filter((seg) => seg && seg !== 'w' && !seg.startsWith('{'))
}
// Name and path tokens identify the operation; description tokens only support it.
function scoreEndpoint(endpoint: EndpointTool, queryTokens: string[]): number {
const nameTokens = [...tokenize(endpoint.name), ...pathSegments(endpoint.path).flatMap(tokenize)]
const descTokens = tokenize(`${endpoint.description} ${endpoint.instructions}`)
let score = 0
for (const qt of queryTokens) {
if (nameTokens.some((t) => tokenMatches(t, qt))) score += 3
else if (descTokens.some((t) => tokenMatches(t, qt))) score += 1
}
return score
}
function schemaPropertyNames(schema: unknown): string[] {
const properties = (schema as { properties?: Record<string, unknown> } | null | undefined)
?.properties
return properties ? Object.keys(properties).filter((k) => k !== 'workspace') : []
}
function truncate(text: string, max: number): string {
return text.length > max ? text.slice(0, max) + '…' : text
}
function summarizeEndpoint(endpoint: EndpointTool) {
const params = [
...schemaPropertyNames(endpoint.path_params_schema),
...schemaPropertyNames(endpoint.query_params_schema)
]
const bodyParams = schemaPropertyNames(endpoint.body_schema)
const instructions = endpoint.instructions.trim()
return {
name: endpoint.name,
endpoint: `${endpoint.method.toUpperCase()} ${endpoint.path}`,
description: truncate(endpoint.description, MAX_DESCRIPTION_CHARS),
...(instructions ? { instructions: truncate(instructions, MAX_INSTRUCTIONS_CHARS) } : {}),
...(params.length > 0 ? { params } : {}),
...(bodyParams.length > 0 ? { body_params: bodyParams } : {})
}
}
function endpointSchemaHelp(endpoint: EndpointTool) {
return {
name: endpoint.name,
endpoint: `${endpoint.method.toUpperCase()} ${endpoint.path}`,
path_params_schema: endpoint.path_params_schema ?? undefined,
query_params_schema: endpoint.query_params_schema ?? undefined,
body_schema: endpoint.body_schema ?? undefined
}
}
async function resolveEndpoint(
workspace: string,
name: string
): Promise<{ endpoint: EndpointTool } | { error: string }> {
const covered = COVERED_ENDPOINTS[name]
if (covered) {
return { error: `"${name}" is covered by the dedicated ${covered} tool — use it instead.` }
}
const catalog = await loadCatalog(workspace)
const endpoint = catalog.find((e) => e.name === name)
if (!endpoint) {
return {
error: `Unknown endpoint "${name}". Use search_api_endpoints to find the endpoint name.`
}
}
return { endpoint }
}
async function executeEndpoint(
endpoint: EndpointTool,
workspace: string,
params: Record<string, unknown>,
body?: unknown
): Promise<string> {
let url = `/api${endpoint.path.replace('{workspace}', encodeURIComponent(workspace))}`
const queryParams = new URLSearchParams()
for (const [key, value] of Object.entries(params)) {
if (value === undefined || value === null) continue
if (url.includes(`{${key}}`)) {
url = url.replace(`{${key}}`, encodeURIComponent(String(value)))
} else {
queryParams.append(key, String(value))
}
}
const unresolved = [...url.matchAll(/\{([^}]+)\}/g)].map((m) => m[1])
if (unresolved.length > 0) {
return JSON.stringify({
success: false,
error: `Missing required path parameter(s): ${unresolved.join(', ')}`,
schema: endpointSchemaHelp(endpoint)
})
}
const search = queryParams.toString()
if (search) url += `?${search}`
const method = endpoint.method.toUpperCase()
const fetchOptions: RequestInit = { method }
if (body !== undefined && method !== 'GET') {
fetchOptions.headers = { 'Content-Type': 'application/json' }
fetchOptions.body = JSON.stringify(body)
}
const response = await fetch(url, fetchOptions)
const raw = response.headers.get('content-type')?.includes('application/json')
? await response.json()
: await response.text()
if (!response.ok) {
return JSON.stringify({
success: false,
status: response.status,
error: typeof raw === 'string' ? raw : JSON.stringify(raw),
// 4xx usually means wrong arguments — echo the schema so the model can
// self-correct in the next call without a separate schema-fetch tool.
...(response.status >= 400 && response.status < 500
? { schema: endpointSchemaHelp(endpoint) }
: {})
})
}
const result = JSON.stringify({ success: true, data: raw })
if (result.length <= MAX_RESULT_CHARS) return result
return JSON.stringify({
success: true,
truncated: true,
data: (typeof raw === 'string' ? raw : JSON.stringify(raw)).slice(0, MAX_RESULT_CHARS),
note: `Result truncated to ${MAX_RESULT_CHARS} characters. Use filter or pagination parameters to narrow it.`
})
}
const searchApiEndpointsSchema = z.object({
query: z
.string()
.describe(
'Keywords matched against endpoint names, paths, and descriptions (e.g. "workers", "queue", "run flow"). Jobs are called "runs" in the UI.'
)
})
const callApiGetSchema = z.object({
name: z.string().describe('Endpoint name as returned by search_api_endpoints'),
params: z
.record(z.string(), z.any())
.optional()
.describe(
'Path and query parameter values, keyed by parameter name. The workspace parameter is filled automatically.'
)
})
const callApiEndpointSchema = z.object({
name: z.string().describe('Endpoint name as returned by search_api_endpoints'),
params: z
.record(z.string(), z.any())
.optional()
.describe(
'Path and query parameter values, keyed by parameter name. The workspace parameter is filled automatically.'
),
body: z
.record(z.string(), z.any())
.optional()
.describe('JSON request body, when the endpoint takes one')
})
export const apiCatalogTools: Tool<{}>[] = [
{
def: createToolDef(
searchApiEndpointsSchema,
'search_api_endpoints',
'Search the Windmill REST API endpoint catalog for operations no dedicated tool covers (workers, queue state, job details, running deployed items, deletions, ...). Returns endpoint names to pass to call_api_get or call_api_endpoint.'
),
fn: async ({ args, workspace, toolId, toolCallbacks }) => {
const parsed = searchApiEndpointsSchema.parse(args)
toolCallbacks.setToolStatus(toolId, { content: 'Searching API endpoints...' })
const catalog = await loadCatalog(workspace)
const queryTokens = tokenize(parsed.query)
const available = catalog.filter((e) => !COVERED_ENDPOINTS[e.name])
const scored = available
.map((endpoint) => ({ endpoint, score: scoreEndpoint(endpoint, queryTokens) }))
.filter((s) => s.score > 0)
.sort((a, b) => b.score - a.score || a.endpoint.name.localeCompare(b.endpoint.name))
const coveredHits = catalog
.filter((e) => COVERED_ENDPOINTS[e.name] && scoreEndpoint(e, queryTokens) > 0)
.map((e) => `${e.name} → use ${COVERED_ENDPOINTS[e.name]}`)
if (scored.length === 0) {
const categories = [...new Set(available.map((e) => pathSegments(e.path)[0]))].sort()
const result = JSON.stringify(
{
matches: [],
hint: `No endpoint matched. Available endpoint categories: ${categories.join(', ')}. Retry with different keywords, or use a dedicated tool if one covers the need.`,
...(coveredHits.length > 0 ? { covered_by_dedicated_tools: coveredHits } : {})
},
null,
2
)
toolCallbacks.setToolStatus(toolId, { content: 'No matching API endpoint', result })
return result
}
const top = scored.slice(0, MAX_SEARCH_RESULTS)
const result = JSON.stringify(
{
matches: top.map((s) => summarizeEndpoint(s.endpoint)),
...(scored.length > top.length
? {
note: `${scored.length - top.length} more match(es) — refine the query to see them.`
}
: {}),
...(coveredHits.length > 0 ? { covered_by_dedicated_tools: coveredHits } : {})
},
null,
2
)
toolCallbacks.setToolStatus(toolId, {
content: `Found ${top.length} API endpoint(s) for "${parsed.query}"`,
result
})
return result
}
},
{
def: createToolDef(
callApiGetSchema,
'call_api_get',
'Call a read-only GET endpoint from the API catalog by name. Use search_api_endpoints first to find the endpoint name; a failed call returns the parameter schema.'
),
showDetails: true,
fn: async ({ args, workspace, toolId, toolCallbacks }) => {
const parsed = callApiGetSchema.parse(args)
const resolved = await resolveEndpoint(workspace, parsed.name)
if ('error' in resolved) {
toolCallbacks.setToolStatus(toolId, { content: resolved.error, error: resolved.error })
return JSON.stringify({ success: false, error: resolved.error })
}
if (resolved.endpoint.method.toUpperCase() !== 'GET') {
const error = `"${parsed.name}" is a ${resolved.endpoint.method.toUpperCase()} endpoint — use call_api_endpoint for mutating calls.`
toolCallbacks.setToolStatus(toolId, { content: error, error })
return JSON.stringify({ success: false, error })
}
toolCallbacks.setToolStatus(toolId, { content: `Calling ${parsed.name}...` })
const result = await executeEndpoint(resolved.endpoint, workspace, parsed.params ?? {})
const ok = JSON.parse(result).success === true
toolCallbacks.setToolStatus(toolId, {
content: ok ? `Called ${parsed.name}` : `Call to ${parsed.name} failed`,
result,
...(ok ? {} : { error: `Call to ${parsed.name} failed` })
})
return result
}
},
{
def: createToolDef(
callApiEndpointSchema,
'call_api_endpoint',
'Call a mutating (POST/PUT/PATCH/DELETE) endpoint from the API catalog by name; the user is asked to confirm. Use search_api_endpoints first to find the endpoint name; a failed call returns the parameter schema.'
),
requiresConfirmation: true,
confirmationMessage: (args) => `Call API endpoint ${args?.name ?? ''}`,
showDetails: true,
fn: async ({ args, workspace, toolId, toolCallbacks }) => {
const parsed = callApiEndpointSchema.parse(args)
const resolved = await resolveEndpoint(workspace, parsed.name)
if ('error' in resolved) {
toolCallbacks.setToolStatus(toolId, { content: resolved.error, error: resolved.error })
return JSON.stringify({ success: false, error: resolved.error })
}
if (resolved.endpoint.method.toUpperCase() === 'GET') {
const error = `"${parsed.name}" is a GET endpoint — use call_api_get (no confirmation needed).`
toolCallbacks.setToolStatus(toolId, { content: error, error })
return JSON.stringify({ success: false, error })
}
toolCallbacks.setToolStatus(toolId, { content: `Calling ${parsed.name}...` })
const result = await executeEndpoint(
resolved.endpoint,
workspace,
parsed.params ?? {},
parsed.body
)
const ok = JSON.parse(result).success === true
toolCallbacks.setToolStatus(toolId, {
content: ok ? `Called ${parsed.name}` : `Call to ${parsed.name} failed`,
result,
...(ok ? {} : { error: `Call to ${parsed.name} failed` })
})
return result
}
}
]
@@ -152,6 +152,7 @@ import {
setEphemeralSecretVariableDraftValue,
type DraftPersistResult
} from './userDraftAdapter'
import { apiCatalogTools } from './apiCatalogTools'
import { isSessionPipelinesEnabled, SESSION_PIPELINES_GATED_MESSAGE } from './pipelineGate'
const ITEM_TYPES = [
@@ -936,6 +937,7 @@ ${pipelineBullet}
- After creating or editing a script or flow draft, run test_run_script, test_run_flow, or test_run_step with representative args before reporting that it works. These tools prefer drafts, so testing does not require deployment.
- Use list_runs to find recent runs (optionally filtered by path, creator, label, or status), then get_job_logs with a returned id to inspect a specific run's logs — without starting a new test run.
- Use open_page to show a workspace page with filters applied — Runs, Schedules, Variables, Resources, Assets, Audit logs, or Workspace settings on a specific tab (e.g. "open the failed runs of f/foo/bar", "open the schedule for X", "open the git sync settings"). Only the pages listed for this user in the tool are available; don't offer pages that aren't listed. Don't use it as a substitute for list_runs when you just need the data yourself.
- For a Windmill operation no other tool covers (workers, queue state, a run's result or args, running deployed items, ...), use search_api_endpoints to find a REST endpoint, then call_api_get for reads or call_api_endpoint for mutations (the user is asked to confirm those). Always prefer a dedicated tool when one exists; endpoints for authoring or deleting scripts, flows, apps, schedules, resources, or variables are not available through the API catalog tools — use the draft tools and delete_workspace_item instead.
- When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. Set multiSelect: true only when the answers can genuinely co-apply and the user may pick several (not mutually exclusive).
- When the user asks you to remember a lasting preference, always/never do something, or change/stop a behavior going forward, call update_user_instructions to persist it. It edits only the USER INSTRUCTIONS block (not WORKSPACE INSTRUCTIONS). Keep each instruction concise; do not use it for one-off requests scoped to the current task.
- Keep context targeted.${
@@ -2994,7 +2996,10 @@ export const globalTools: Tool<{}>[] = [
// Workspace-scoped datatable tools (unrestricted: no whitelist, no creation policy)
...getDatatableTools(),
// Read-only tools over files the user attached to the conversation
...fileTools
...fileTools,
// Search + call access to the backend API endpoint catalog, for operations
// no dedicated tool covers
...apiCatalogTools
]
// Tools that only make sense inside an AI session (they drive the session's