From 5feec9b4cd3cdd4eed3edf5be0fc42dabfeb13d2 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 28 Jul 2026 14:35:01 +0200 Subject: [PATCH] fix(ai-chat): make hub script paths readable from the global chat (#10381) * fix(ai-chat): make hub script paths readable from the global chat Co-Authored-By: Claude Opus 5 (1M context) * fix(ai-evals): match hub fixtures on whole words, not substrings Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- ai_evals/adapters/frontend/mockBackend.ts | 162 +++++++++++++++++- ai_evals/cases/global.yaml | 46 +++++ .../copilot/chat/global/core.test.ts | 26 +++ .../components/copilot/chat/global/core.ts | 24 ++- .../components/copilot/chat/shared.test.ts | 67 ++++++++ .../src/lib/components/copilot/chat/shared.ts | 43 +++-- 6 files changed, 352 insertions(+), 16 deletions(-) diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 64cacabd58..40a6dfa4ad 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -809,6 +809,142 @@ export function listBenchmarkMcpTools(): EndpointTool[] { return BENCHMARK_MCP_TOOLS } +/** A stand-in Windmill hub. `search_hub_scripts` and a `hub/` read go out over + * relative `/api/...` fetches, which have no origin here, so without these the + * hub tools throw and no case can exercise hub reuse. Serving fixtures rather + * than the live hub also keeps assertions on script content stable as the real + * hub republishes new versions. */ +const BENCHMARK_HUB_SCRIPTS = [ + { + version_id: 22235, + app: 'holded', + summary: 'Send Document', + terms: 'holded invoice document send email mail', + language: 'bun', + content: `//native +type Holded = { + apiKey: string; +}; +/** + * Send Document + * Send a specific document by email. + */ +export async function main( + auth: Holded, + docType: string, + documentId: string, + body: { + mailTemplateId?: string; + emails: string; + subject?: string; + message?: string; + docIds?: string; + }, +) { + const url = new URL( + \`https://api.holded.com/api/invoicing/v1/documents/\${docType}/\${documentId}/send\`, + ); + + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + key: auth.apiKey, + }, + body: JSON.stringify(body), + }); + if (!response.ok) { + const text = await response.text(); + throw new Error(\`\${response.status} \${text}\`); + } + return await response.json(); +} +`, + schema: { + type: 'object', + required: ['auth', 'docType', 'documentId', 'body'], + properties: { + auth: { type: 'object', format: 'resource-holded' }, + docType: { type: 'string' }, + documentId: { type: 'string' }, + body: { type: 'object' } + } + } + }, + { + version_id: 28294, + app: 'discord', + summary: 'Send a message to Discord using Webhook', + terms: 'discord webhook message send chat channel', + language: 'bunnative', + content: `//native + +type DiscordWebhook = { + webhook_url: string; +}; +export async function main(discord_webhook: DiscordWebhook, message: string) { + const response = await fetch(\`\${discord_webhook.webhook_url}?wait=true\`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ content: message }), + }); + if (!response.ok) { + throw new Error(\`\${response.status} \${await response.text()}\`); + } + return await response.json(); +} +`, + schema: { + type: 'object', + required: ['discord_webhook', 'message'], + properties: { + discord_webhook: { type: 'object', format: 'resource-discord_webhook' }, + message: { type: 'string' } + } + } + } +] + +/** Naive whole-word overlap — enough to rank a handful of fixtures for a natural + * query without pulling an embedding model into the benchmark. Every frontend eval + * shares this handler, so the bar to match is deliberately high: naming the + * integration, or overlapping on three meaningful words. A looser bar answers + * "send a Slack message" with the Discord fixture, handing an unrelated case a + * plausible-looking wrong integration. */ +function searchBenchmarkHubScripts(text: string) { + const tokens = new Set( + text + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((token) => token.length > 2) + ) + return BENCHMARK_HUB_SCRIPTS.map((script) => { + const words = new Set( + `${script.app} ${script.summary} ${script.terms}`.toLowerCase().split(/[^a-z0-9]+/) + ) + const score = [...tokens].filter((token) => words.has(token)).length + return { script, score, namesApp: tokens.has(script.app) } + }) + .filter((entry) => entry.namesApp || entry.score >= 3) + .sort((a, b) => b.score - a.score) + .map(({ script }, index) => ({ + ask_id: script.version_id, + id: script.version_id, + version_id: script.version_id, + summary: script.summary, + app: script.app, + kind: 'script', + score: 1 - index * 0.01 + })) +} + +/** The hub keys a script by its version id; the app and slug segments that + * follow are descriptive, so match on the id exactly as the real hub does. */ +function getBenchmarkHubScript(path: string) { + const versionId = Number(path.replace(/^\/api\/scripts\/hub\/get_full\/hub\//, '').split('/')[0]) + return BENCHMARK_HUB_SCRIPTS.find((script) => script.version_id === versionId) +} + const BENCHMARK_WORKERS = [ { worker: 'wk-benchmark-1', @@ -837,10 +973,16 @@ const BENCHMARK_WORKERS = [ * intercepting it with a synthetic 404 sends the model into retry loops. */ export function hasBenchmarkApiHandler(url: string): boolean { const path = url.split('?')[0] - return path === '/api/workers/list' || /^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path) + return ( + path === '/api/workers/list' || + /^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path) || + path === '/api/embeddings/query_hub_scripts' || + path.startsWith('/api/scripts/hub/get_full/') + ) } -/** Answer a relative `/api/...` fetch issued by the API catalog executor. */ +/** Answer a relative `/api/...` fetch — from the API catalog executor, or from the + * chat's hub tools. */ export function handleBenchmarkApiFetch(url: string): Response { const path = url.split('?')[0] if (path === '/api/workers/list') { @@ -849,5 +991,21 @@ export function handleBenchmarkApiFetch(url: string): Response { if (/^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path)) { return Response.json([]) } + if (path === '/api/embeddings/query_hub_scripts') { + const text = new URLSearchParams(url.split('?')[1] ?? '').get('text') ?? '' + return Response.json(searchBenchmarkHubScripts(text)) + } + if (path.startsWith('/api/scripts/hub/get_full/')) { + const script = getBenchmarkHubScript(path) + if (!script) { + return Response.json({ error: 'hub script not found' }, { status: 404 }) + } + return Response.json({ + content: script.content, + language: script.language, + schema: script.schema, + summary: script.summary + }) + } return Response.json({ error: `no benchmark handler for ${path}` }, { status: 404 }) } diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 72739d2778..005c677e79 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1877,3 +1877,49 @@ stringIncludesAnyOf: - email skipJudge: true + +# --- Windmill Hub reuse (search_hub_scripts + read_workspace_item on a hub/ path) --- +# Holded's API is obscure enough that a model writing from memory cannot reproduce +# its endpoint and `key` auth header — so the draft's fidelity to the published +# script is what proves the hub content was actually fetched, not guessed. +- id: global-hub1-reuse-hub-script + prompt: |- + I want to email one of my Holded invoices to a customer from Windmill. + There is already a script for that on the Windmill hub — reuse it instead of writing + your own, and save it as a draft script at `f/evals/global/holded_send_document`. + Leave it as an AI draft; do not deploy it. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 12 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: script + path: f/evals/global/holded_send_document + language: bun + valueIncludes: + - api.holded.com/api/invoicing/v1/documents + # `mailTemplateId` is an optional field of the published script's body + # that a model writing from memory does not invent, so it is what + # separates reusing the hub script from re-deriving one that merely + # hits the same endpoint. + - mailTemplateId + toolExpect: + requiredToolsUsed: + - search_hub_scripts + - read_workspace_item + - write_script + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: read_workspace_item + field: path + stringIncludesAnyOf: + - hub/ + judgeChecklist: + - the draft sends an existing Holded document by email rather than creating one + - the request targets Holded's document send endpoint, not an invented URL + - authentication uses Holded's own key header rather than a bearer token + - the document type, document id, and recipient emails are inputs to the script + - the result stays an AI draft and is not deployed diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 2ce668d789..9e8cec8b1e 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -69,6 +69,9 @@ vi.mock('$lib/gen', async () => { }), queryHubScripts: vi.fn(async () => []), getHubScriptContentByPath: vi.fn(async () => ''), + getHubScriptByPath: vi.fn(async () => { + throw new Error('getHubScriptByPath mock not configured') + }), listScripts: vi.fn(async () => []) }), JobService: wrapService(actual.JobService, { @@ -534,6 +537,29 @@ describe('global AI tools', () => { ]) }) + it('reads a hub script path through the hub endpoint, not the workspace one', async () => { + vi.mocked(ScriptService.getHubScriptByPath).mockResolvedValueOnce({ + content: 'export async function main() {}', + language: 'bunnative', + summary: 'Send a message to discord using webhook', + schema: { type: 'object', properties: {} } + }) + + const raw = await callGlobalTool('read_workspace_item', { + type: 'script', + path: 'hub/28294/discord/send_a_message_to_discord_using_webhook' + }) + + expect(ScriptService.getHubScriptByPath).toHaveBeenCalledWith({ + path: 'hub/28294/discord/send_a_message_to_discord_using_webhook' + }) + expect(ScriptService.getScriptByPath).not.toHaveBeenCalled() + expect(JSON.parse(raw)).toMatchObject({ + language: 'bunnative', + value: 'export async function main() {}' + }) + }) + it('reads the deployed state, skipping chat and DB drafts, with version: deployed', async () => { await callGlobalTool('write_script', { path: 'f/scripts/greet', diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 80204ba02b..cc91b9810b 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -90,6 +90,7 @@ import { executeFlowStepTestRun, executeTestRun, findAndReplace, + isHubPath, type CreatedResourceTriggerKind, type PreviewCardKind, type Tool, @@ -362,7 +363,11 @@ const listWorkspaceItemsSchema = z.object({ const readWorkspaceItemSchema = z.object({ type: itemTypeSchema, - path: z.string().describe('Workspace path of the item to read.'), + path: z + .string() + .describe( + 'Workspace path of the item to read, or a hub/// path from search_hub_scripts to read a hub script.' + ), trigger_kind: triggerKindSchema .optional() .describe('Required when type is trigger. Identifies which trigger service to call.'), @@ -1170,6 +1175,7 @@ Rules: - Variable values are never readable. For secrets, create a secret variable and reference it from resources as "$var:path/to/variable". - Use search_resource_types before write_resource. - When script or raw app code needs an external npm package you are not fully familiar with, use search_npm_packages to find it and get its documentation and type definitions. Link the package documentation in your answer when you rely on it. +- Hub scripts are prebuilt integrations for third-party services, hosted outside the workspace under \`hub///\` paths. Use search_hub_scripts to find one before hand-writing an integration, then read_workspace_item with type "script" and the returned hub path to get its code, language, and input schema. - Use get_db_schema with a database resource path to fetch its tables and columns before writing SQL (or a script querying that database). - Use get_instructions before writing scripts, flows, resources, or apps. For scripts, pass the target language. ${pipelineBullet} @@ -1778,6 +1784,20 @@ async function readWorkspaceItem( ): Promise { switch (type) { case 'script': { + // Hub scripts are not workspace items: search_hub_scripts hands back + // `hub///` paths, which getScriptByPath cannot resolve. + if (isHubPath(path)) { + const hub = await ScriptService.getHubScriptByPath({ path }) + return { + type: 'script', + path, + summary: hub.summary, + language: hub.language as ScriptLang, + value: hub.content, + schema: hub.schema, + isDraft: false + } + } // Prefer the DB draft (newer than the deployed version) when one exists, // unless the caller explicitly asked for the deployed state. const script = await ScriptService.getScriptByPath({ @@ -2785,7 +2805,7 @@ export const globalTools: Tool<{}>[] = [ return JSON.stringify({ success: false, error: message }) } const draft = - parsed.version === 'deployed' + parsed.version === 'deployed' || isHubPath(parsed.path) ? null : await getGlobalDraft(workspace, parsed.type, parsed.path, parsed.trigger_kind) if (draft) { diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index 473c3d3335..179742763f 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -1124,3 +1124,70 @@ describe('openItemPreviewAction', () => { expect(openItemPreviewAction('raw_app', 'u/me/dash').label).toBe('Open app preview') }) }) + +describe('createSearchHubScriptsTool', () => { + const hit = (version_id: number, app: string, summary: string) => ({ + version_id, + app, + summary, + ask_id: version_id, + id: version_id, + kind: 'script' as const, + score: 1 + }) + + async function runWithContent(getHubScriptByPath: ReturnType) { + const { ScriptService } = await import('$lib/gen') + Object.assign(ScriptService, { + queryHubScripts: vi.fn(async () => [ + hit(1, 'discord', 'Send a message'), + hit(2, 'slack', 'Post a message') + ]), + getHubScriptByPath + }) + const { createSearchHubScriptsTool } = await import('./shared') + const raw = await createSearchHubScriptsTool(true).fn({ + args: { query: 'send a message' }, + toolId: 't1', + toolCallbacks: { setToolStatus: vi.fn() } + } as any) + return JSON.parse(raw) + } + + it('reports each script language alongside its content', async () => { + const results = await runWithContent( + vi.fn(async ({ path }: { path: string }) => ({ + content: `// ${path}`, + language: path.startsWith('hub/1/') ? 'bunnative' : 'python3' + })) + ) + + expect(results).toEqual([ + { + path: 'hub/1/discord/send_a_message', + summary: 'Send a message', + language: 'bunnative', + content: '// hub/1/discord/send_a_message' + }, + { + path: 'hub/2/slack/post_a_message', + summary: 'Post a message', + language: 'python3', + content: '// hub/2/slack/post_a_message' + } + ]) + }) + + it('keeps the other results when one content fetch fails', async () => { + const results = await runWithContent( + vi.fn(async ({ path }: { path: string }) => { + if (path.startsWith('hub/1/')) throw new Error('hub unreachable') + return { content: 'ok', language: 'python3' } + }) + ) + + expect(results[0].error).toContain('hub unreachable') + expect(results[0].content).toBeUndefined() + expect(results[1].content).toBe('ok') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index dd602c640f..dcff212198 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -1130,6 +1130,18 @@ const searchHubScriptsToolDef = createToolDef( 'Search for scripts in the hub' ) +/** The hub resolves a script by its version id alone; the app and summary + * segments are descriptive only. Mirrors the paths the hub pickers build. */ +function hubScriptPath(s: { version_id: number; app: string; summary: string }): string { + return `hub/${s.version_id}/${s.app}/${s.summary.toLowerCase().replaceAll(/\s+/g, '_')}` +} + +/** Hub scripts are hosted outside the workspace: their paths resolve through the + * hub endpoints only, never through workspace lookups or drafts. */ +export function isHubPath(path: string): boolean { + return path.startsWith('hub/') +} + export const createSearchHubScriptsTool = (withContent: boolean = false) => ({ def: searchHubScriptsToolDef, fn: async ({ args, toolId, toolCallbacks }) => { @@ -1141,22 +1153,29 @@ export const createSearchHubScriptsTool = (withContent: boolean = false) => ({ text: parsedArgs.query, kind: 'script' }) + // Each result costs a content fetch, so cap the fan-out when content is wanted. + const matches = withContent ? scripts.slice(0, 3) : scripts toolCallbacks.setToolStatus(toolId, { - content: 'Found ' + scripts.length + ' scripts in the hub related to "' + args.query + '"' + content: `Found ${matches.length} script${matches.length === 1 ? '' : 's'} in the hub related to "${parsedArgs.query}"` }) - // if withContent, fetch scripts with their content, limit to 3 results const results = await Promise.all( - scripts.slice(0, withContent ? 3 : undefined).map(async (s) => { - let content = '' - if (withContent) { - content = await ScriptService.getHubScriptContentByPath({ - path: `hub/${s.version_id}/${s.app}/${s.summary.toLowerCase().replaceAll(/\s+/g, '_')}` - }) + matches.map(async (s) => { + const path = hubScriptPath(s) + if (!withContent) { + return { path, summary: s.summary } } - return { - path: `hub/${s.version_id}/${s.app}/${s.summary.toLowerCase().replaceAll(/\s+/g, '_')}`, - summary: s.summary, - ...(withContent ? { content } : {}) + try { + // get_full, not the raw content endpoint: callers are told to match the + // script's language, which raw content does not carry. + const hub = await ScriptService.getHubScriptByPath({ path }) + return { path, summary: s.summary, language: hub.language, content: hub.content } + } catch (err) { + // One unreachable script must not sink the whole search. + return { + path, + summary: s.summary, + error: `Could not fetch content: ${err instanceof Error ? err.message : String(err)}` + } } }) )