fix(ai-chat): make hub script paths readable from the global chat (#10381)

* fix(ai-chat): make hub script paths readable from the global chat

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix(ai-evals): match hub fixtures on whole words, not substrings

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

ai_evals/adapters/frontend/mockBackend.ts

@@ -809,6 +809,142 @@ export function listBenchmarkMcpTools(): EndpointTool[] {
 	return BENCHMARK_MCP_TOOLS
 }
 
+/** A stand-in Windmill hub. `search_hub_scripts` and a `hub/` read go out over
+ * relative `/api/...` fetches, which have no origin here, so without these the
+ * hub tools throw and no case can exercise hub reuse. Serving fixtures rather
+ * than the live hub also keeps assertions on script content stable as the real
+ * hub republishes new versions. */
+const BENCHMARK_HUB_SCRIPTS = [
+	{
+		version_id: 22235,
+		app: 'holded',
+		summary: 'Send Document',
+		terms: 'holded invoice document send email mail',
+		language: 'bun',
+		content: `//native
+type Holded = {
+  apiKey: string;
+};
+/**
+ * Send Document
+ * Send a specific document by email.
+ */
+export async function main(
+  auth: Holded,
+  docType: string,
+  documentId: string,
+  body: {
+    mailTemplateId?: string;
+    emails: string;
+    subject?: string;
+    message?: string;
+    docIds?: string;
+  },
+) {
+  const url = new URL(
+    \`https://api.holded.com/api/invoicing/v1/documents/\${docType}/\${documentId}/send\`,
+  );
+
+  const response = await fetch(url, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      key: auth.apiKey,
+    },
+    body: JSON.stringify(body),
+  });
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(\`\${response.status} \${text}\`);
+  }
+  return await response.json();
+}
+`,
+		schema: {
+			type: 'object',
+			required: ['auth', 'docType', 'documentId', 'body'],
+			properties: {
+				auth: { type: 'object', format: 'resource-holded' },
+				docType: { type: 'string' },
+				documentId: { type: 'string' },
+				body: { type: 'object' }
+			}
+		}
+	},
+	{
+		version_id: 28294,
+		app: 'discord',
+		summary: 'Send a message to Discord using Webhook',
+		terms: 'discord webhook message send chat channel',
+		language: 'bunnative',
+		content: `//native
+
+type DiscordWebhook = {
+  webhook_url: string;
+};
+export async function main(discord_webhook: DiscordWebhook, message: string) {
+  const response = await fetch(\`\${discord_webhook.webhook_url}?wait=true\`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ content: message }),
+  });
+  if (!response.ok) {
+    throw new Error(\`\${response.status} \${await response.text()}\`);
+  }
+  return await response.json();
+}
+`,
+		schema: {
+			type: 'object',
+			required: ['discord_webhook', 'message'],
+			properties: {
+				discord_webhook: { type: 'object', format: 'resource-discord_webhook' },
+				message: { type: 'string' }
+			}
+		}
+	}
+]
+
+/** Naive whole-word overlap — enough to rank a handful of fixtures for a natural
+ * query without pulling an embedding model into the benchmark. Every frontend eval
+ * shares this handler, so the bar to match is deliberately high: naming the
+ * integration, or overlapping on three meaningful words. A looser bar answers
+ * "send a Slack message" with the Discord fixture, handing an unrelated case a
+ * plausible-looking wrong integration. */
+function searchBenchmarkHubScripts(text: string) {
+	const tokens = new Set(
+		text
+			.toLowerCase()
+			.split(/[^a-z0-9]+/)
+			.filter((token) => token.length > 2)
+	)
+	return BENCHMARK_HUB_SCRIPTS.map((script) => {
+		const words = new Set(
+			`${script.app} ${script.summary} ${script.terms}`.toLowerCase().split(/[^a-z0-9]+/)
+		)
+		const score = [...tokens].filter((token) => words.has(token)).length
+		return { script, score, namesApp: tokens.has(script.app) }
+	})
+		.filter((entry) => entry.namesApp || entry.score >= 3)
+		.sort((a, b) => b.score - a.score)
+		.map(({ script }, index) => ({
+			ask_id: script.version_id,
+			id: script.version_id,
+			version_id: script.version_id,
+			summary: script.summary,
+			app: script.app,
+			kind: 'script',
+			score: 1 - index * 0.01
+		}))
+}
+
+/** The hub keys a script by its version id; the app and slug segments that
+ * follow are descriptive, so match on the id exactly as the real hub does. */
+function getBenchmarkHubScript(path: string) {
+	const versionId = Number(path.replace(/^\/api\/scripts\/hub\/get_full\/hub\//, '').split('/')[0])
+	return BENCHMARK_HUB_SCRIPTS.find((script) => script.version_id === versionId)
+}
+
 const BENCHMARK_WORKERS = [
 	{
 		worker: 'wk-benchmark-1',
@@ -837,10 +973,16 @@ const BENCHMARK_WORKERS = [
  * intercepting it with a synthetic 404 sends the model into retry loops. */
 export function hasBenchmarkApiHandler(url: string): boolean {
 	const path = url.split('?')[0]
-	return path === '/api/workers/list' || /^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path)
+	return (
+		path === '/api/workers/list' ||
+		/^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path) ||
+		path === '/api/embeddings/query_hub_scripts' ||
+		path.startsWith('/api/scripts/hub/get_full/')
+	)
 }
 
-/** Answer a relative `/api/...` fetch issued by the API catalog executor. */
+/** Answer a relative `/api/...` fetch — from the API catalog executor, or from the
+ * chat's hub tools. */
 export function handleBenchmarkApiFetch(url: string): Response {
 	const path = url.split('?')[0]
 	if (path === '/api/workers/list') {
@@ -849,5 +991,21 @@ export function handleBenchmarkApiFetch(url: string): Response {
 	if (/^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path)) {
 		return Response.json([])
 	}
+	if (path === '/api/embeddings/query_hub_scripts') {
+		const text = new URLSearchParams(url.split('?')[1] ?? '').get('text') ?? ''
+		return Response.json(searchBenchmarkHubScripts(text))
+	}
+	if (path.startsWith('/api/scripts/hub/get_full/')) {
+		const script = getBenchmarkHubScript(path)
+		if (!script) {
+			return Response.json({ error: 'hub script not found' }, { status: 404 })
+		}
+		return Response.json({
+			content: script.content,
+			language: script.language,
+			schema: script.schema,
+			summary: script.summary
+		})
+	}
 	return Response.json({ error: `no benchmark handler for ${path}` }, { status: 404 })
 }

ai_evals/cases/global.yaml

@@ -1877,3 +1877,49 @@
         stringIncludesAnyOf:
           - email
   skipJudge: true
+
+# --- Windmill Hub reuse (search_hub_scripts + read_workspace_item on a hub/ path) ---
+# Holded's API is obscure enough that a model writing from memory cannot reproduce
+# its endpoint and `key` auth header — so the draft's fidelity to the published
+# script is what proves the hub content was actually fetched, not guessed.
+- id: global-hub1-reuse-hub-script
+  prompt: |-
+    I want to email one of my Holded invoices to a customer from Windmill.
+    There is already a script for that on the Windmill hub — reuse it instead of writing
+    your own, and save it as a draft script at `f/evals/global/holded_send_document`.
+    Leave it as an AI draft; do not deploy it.
+  initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
+  runtime:
+    maxTurns: 12
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: script
+        path: f/evals/global/holded_send_document
+        language: bun
+        valueIncludes:
+          - api.holded.com/api/invoicing/v1/documents
+          # `mailTemplateId` is an optional field of the published script's body
+          # that a model writing from memory does not invent, so it is what
+          # separates reusing the hub script from re-deriving one that merely
+          # hits the same endpoint.
+          - mailTemplateId
+  toolExpect:
+    requiredToolsUsed:
+      - search_hub_scripts
+      - read_workspace_item
+      - write_script
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+      - delete_workspace_item
+    toolCallArgs:
+      - tool: read_workspace_item
+        field: path
+        stringIncludesAnyOf:
+          - hub/
+  judgeChecklist:
+    - the draft sends an existing Holded document by email rather than creating one
+    - the request targets Holded's document send endpoint, not an invented URL
+    - authentication uses Holded's own key header rather than a bearer token
+    - the document type, document id, and recipient emails are inputs to the script
+    - the result stays an AI draft and is not deployed

frontend/src/lib/components/copilot/chat/global/core.test.ts

@@ -69,6 +69,9 @@ vi.mock('$lib/gen', async () => {
 			}),
 			queryHubScripts: vi.fn(async () => []),
 			getHubScriptContentByPath: vi.fn(async () => ''),
+			getHubScriptByPath: vi.fn(async () => {
+				throw new Error('getHubScriptByPath mock not configured')
+			}),
 			listScripts: vi.fn(async () => [])
 		}),
 		JobService: wrapService(actual.JobService, {
@@ -534,6 +537,29 @@ describe('global AI tools', () => {
 		])
 	})
 
+	it('reads a hub script path through the hub endpoint, not the workspace one', async () => {
+		vi.mocked(ScriptService.getHubScriptByPath).mockResolvedValueOnce({
+			content: 'export async function main() {}',
+			language: 'bunnative',
+			summary: 'Send a message to discord using webhook',
+			schema: { type: 'object', properties: {} }
+		})
+
+		const raw = await callGlobalTool('read_workspace_item', {
+			type: 'script',
+			path: 'hub/28294/discord/send_a_message_to_discord_using_webhook'
+		})
+
+		expect(ScriptService.getHubScriptByPath).toHaveBeenCalledWith({
+			path: 'hub/28294/discord/send_a_message_to_discord_using_webhook'
+		})
+		expect(ScriptService.getScriptByPath).not.toHaveBeenCalled()
+		expect(JSON.parse(raw)).toMatchObject({
+			language: 'bunnative',
+			value: 'export async function main() {}'
+		})
+	})
+
 	it('reads the deployed state, skipping chat and DB drafts, with version: deployed', async () => {
 		await callGlobalTool('write_script', {
 			path: 'f/scripts/greet',

frontend/src/lib/components/copilot/chat/global/core.ts

@@ -90,6 +90,7 @@ import {
 	executeFlowStepTestRun,
 	executeTestRun,
 	findAndReplace,
+	isHubPath,
 	type CreatedResourceTriggerKind,
 	type PreviewCardKind,
 	type Tool,
@@ -362,7 +363,11 @@ const listWorkspaceItemsSchema = z.object({
 
 const readWorkspaceItemSchema = z.object({
 	type: itemTypeSchema,
-	path: z.string().describe('Workspace path of the item to read.'),
+	path: z
+		.string()
+		.describe(
+			'Workspace path of the item to read, or a hub/<version>/<app>/<name> path from search_hub_scripts to read a hub script.'
+		),
 	trigger_kind: triggerKindSchema
 		.optional()
 		.describe('Required when type is trigger. Identifies which trigger service to call.'),
@@ -1170,6 +1175,7 @@ Rules:
 - Variable values are never readable. For secrets, create a secret variable and reference it from resources as "$var:path/to/variable".
 - Use search_resource_types before write_resource.
 - When script or raw app code needs an external npm package you are not fully familiar with, use search_npm_packages to find it and get its documentation and type definitions. Link the package documentation in your answer when you rely on it.
+- Hub scripts are prebuilt integrations for third-party services, hosted outside the workspace under \`hub/<version>/<app>/<name>\` paths. Use search_hub_scripts to find one before hand-writing an integration, then read_workspace_item with type "script" and the returned hub path to get its code, language, and input schema.
 - Use get_db_schema with a database resource path to fetch its tables and columns before writing SQL (or a script querying that database).
 - Use get_instructions before writing scripts, flows, resources, or apps. For scripts, pass the target language.
 ${pipelineBullet}
@@ -1778,6 +1784,20 @@ async function readWorkspaceItem(
 ): Promise<WorkspaceItem> {
 	switch (type) {
 		case 'script': {
+			// Hub scripts are not workspace items: search_hub_scripts hands back
+			// `hub/<version>/<app>/<slug>` paths, which getScriptByPath cannot resolve.
+			if (isHubPath(path)) {
+				const hub = await ScriptService.getHubScriptByPath({ path })
+				return {
+					type: 'script',
+					path,
+					summary: hub.summary,
+					language: hub.language as ScriptLang,
+					value: hub.content,
+					schema: hub.schema,
+					isDraft: false
+				}
+			}
 			// Prefer the DB draft (newer than the deployed version) when one exists,
 			// unless the caller explicitly asked for the deployed state.
 			const script = await ScriptService.getScriptByPath({
@@ -2785,7 +2805,7 @@ export const globalTools: Tool<{}>[] = [
 				return JSON.stringify({ success: false, error: message })
 			}
 			const draft =
-				parsed.version === 'deployed'
+				parsed.version === 'deployed' || isHubPath(parsed.path)
 					? null
 					: await getGlobalDraft(workspace, parsed.type, parsed.path, parsed.trigger_kind)
 			if (draft) {

frontend/src/lib/components/copilot/chat/shared.test.ts

@@ -1124,3 +1124,70 @@ describe('openItemPreviewAction', () => {
 		expect(openItemPreviewAction('raw_app', 'u/me/dash').label).toBe('Open app preview')
 	})
 })
+
+describe('createSearchHubScriptsTool', () => {
+	const hit = (version_id: number, app: string, summary: string) => ({
+		version_id,
+		app,
+		summary,
+		ask_id: version_id,
+		id: version_id,
+		kind: 'script' as const,
+		score: 1
+	})
+
+	async function runWithContent(getHubScriptByPath: ReturnType<typeof vi.fn>) {
+		const { ScriptService } = await import('$lib/gen')
+		Object.assign(ScriptService, {
+			queryHubScripts: vi.fn(async () => [
+				hit(1, 'discord', 'Send a message'),
+				hit(2, 'slack', 'Post a message')
+			]),
+			getHubScriptByPath
+		})
+		const { createSearchHubScriptsTool } = await import('./shared')
+		const raw = await createSearchHubScriptsTool(true).fn({
+			args: { query: 'send a message' },
+			toolId: 't1',
+			toolCallbacks: { setToolStatus: vi.fn() }
+		} as any)
+		return JSON.parse(raw)
+	}
+
+	it('reports each script language alongside its content', async () => {
+		const results = await runWithContent(
+			vi.fn(async ({ path }: { path: string }) => ({
+				content: `// ${path}`,
+				language: path.startsWith('hub/1/') ? 'bunnative' : 'python3'
+			}))
+		)
+
+		expect(results).toEqual([
+			{
+				path: 'hub/1/discord/send_a_message',
+				summary: 'Send a message',
+				language: 'bunnative',
+				content: '// hub/1/discord/send_a_message'
+			},
+			{
+				path: 'hub/2/slack/post_a_message',
+				summary: 'Post a message',
+				language: 'python3',
+				content: '// hub/2/slack/post_a_message'
+			}
+		])
+	})
+
+	it('keeps the other results when one content fetch fails', async () => {
+		const results = await runWithContent(
+			vi.fn(async ({ path }: { path: string }) => {
+				if (path.startsWith('hub/1/')) throw new Error('hub unreachable')
+				return { content: 'ok', language: 'python3' }
+			})
+		)
+
+		expect(results[0].error).toContain('hub unreachable')
+		expect(results[0].content).toBeUndefined()
+		expect(results[1].content).toBe('ok')
+	})
+})

frontend/src/lib/components/copilot/chat/shared.ts

@@ -1130,6 +1130,18 @@ const searchHubScriptsToolDef = createToolDef(
 	'Search for scripts in the hub'
 )
 
+/** The hub resolves a script by its version id alone; the app and summary
+ * segments are descriptive only. Mirrors the paths the hub pickers build. */
+function hubScriptPath(s: { version_id: number; app: string; summary: string }): string {
+	return `hub/${s.version_id}/${s.app}/${s.summary.toLowerCase().replaceAll(/\s+/g, '_')}`
+}
+
+/** Hub scripts are hosted outside the workspace: their paths resolve through the
+ * hub endpoints only, never through workspace lookups or drafts. */
+export function isHubPath(path: string): boolean {
+	return path.startsWith('hub/')
+}
+
 export const createSearchHubScriptsTool = (withContent: boolean = false) => ({
 	def: searchHubScriptsToolDef,
 	fn: async ({ args, toolId, toolCallbacks }) => {
@@ -1141,22 +1153,29 @@ export const createSearchHubScriptsTool = (withContent: boolean = false) => ({
 			text: parsedArgs.query,
 			kind: 'script'
 		})
+		// Each result costs a content fetch, so cap the fan-out when content is wanted.
+		const matches = withContent ? scripts.slice(0, 3) : scripts
 		toolCallbacks.setToolStatus(toolId, {
-			content: 'Found ' + scripts.length + ' scripts in the hub related to "' + args.query + '"'
+			content: `Found ${matches.length} script${matches.length === 1 ? '' : 's'} in the hub related to "${parsedArgs.query}"`
 		})
-		// if withContent, fetch scripts with their content, limit to 3 results
 		const results = await Promise.all(
-			scripts.slice(0, withContent ? 3 : undefined).map(async (s) => {
-				let content = ''
-				if (withContent) {
-					content = await ScriptService.getHubScriptContentByPath({
-						path: `hub/${s.version_id}/${s.app}/${s.summary.toLowerCase().replaceAll(/\s+/g, '_')}`
-					})
+			matches.map(async (s) => {
+				const path = hubScriptPath(s)
+				if (!withContent) {
+					return { path, summary: s.summary }
 				}
-				return {
-					path: `hub/${s.version_id}/${s.app}/${s.summary.toLowerCase().replaceAll(/\s+/g, '_')}`,
-					summary: s.summary,
-					...(withContent ? { content } : {})
+				try {
+					// get_full, not the raw content endpoint: callers are told to match the
+					// script's language, which raw content does not carry.
+					const hub = await ScriptService.getHubScriptByPath({ path })
+					return { path, summary: s.summary, language: hub.language, content: hub.content }
+				} catch (err) {
+					// One unreachable script must not sink the whole search.
+					return {
+						path,
+						summary: s.summary,
+						error: `Could not fetch content: ${err instanceof Error ? err.message : String(err)}`
+					}
 				}
 			})
 		)