From 7e4fb283fa90f84615f08f7841914cc22fe30aff Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 11 Aug 2026 14:59:08 +0200 Subject: [PATCH] feat: give the ai chat hub script descriptions and integration metadata Co-Authored-By: Claude Opus 5 --- backend/windmill-api-embeddings/src/lib.rs | 13 ++ backend/windmill-api/openapi.yaml | 156 ++++++++++++++++++ backend/windmill-api/src/integration.rs | 29 +++- .../components/copilot/chat/global/core.ts | 5 +- .../components/copilot/chat/shared.test.ts | 100 ++++++++++- .../src/lib/components/copilot/chat/shared.ts | 95 ++++++++++- 6 files changed, 388 insertions(+), 10 deletions(-) diff --git a/backend/windmill-api-embeddings/src/lib.rs b/backend/windmill-api-embeddings/src/lib.rs index 35816dfaa8..fc0a51bafd 100644 --- a/backend/windmill-api-embeddings/src/lib.rs +++ b/backend/windmill-api-embeddings/src/lib.rs @@ -78,6 +78,8 @@ pub struct HubScriptResult { id: i64, version_id: i64, summary: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, app: String, kind: String, score: f32, @@ -163,6 +165,9 @@ struct HubScript { id: i64, version_id: i64, summary: String, + // Nearly a fifth of hub scripts carry an explicit `"description": null`; a bare + // String here fails the whole blob and takes hub search down with it. + description: Option, app: String, kind: String, embedding: Vec, @@ -360,6 +365,10 @@ impl EmbeddingsDb { let mut hm = HashMap::new(); hm.insert("ask_id".to_string(), script.ask_id.clone().to_string()); hm.insert("summary".to_string(), script.summary.clone()); + hm.insert( + "description".to_string(), + script.description.clone().unwrap_or_default(), + ); hm.insert("app".to_string(), script.app.clone()); hm.insert("kind".to_string(), script.kind.clone()); hm.insert("id".to_string(), script.id.clone().to_string()); @@ -517,6 +526,10 @@ impl EmbeddingsDb { .get("summary") .ok_or(Error::msg("no summary"))? .to_owned(), + description: metadata + .get("description") + .filter(|d| !d.is_empty()) + .map(|d| d.to_owned()), app: metadata.get("app").ok_or(Error::msg("no app"))?.to_owned(), kind: metadata .get("kind") diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 14672f3a21..ad6b9e44da 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -8354,6 +8354,160 @@ paths: required: - name + /integrations/hub/{app}/meta: + get: + summary: get hub integration metadata + operationId: getHubIntegrationMeta + tags: + - integration + parameters: + - name: app + description: integration slug + in: path + required: true + schema: + type: string + responses: + "200": + description: integration metadata + content: + application/json: + schema: + type: object + properties: + app: + type: string + display_name: + type: string + description: + type: string + nullable: true + docs_url: + type: string + nullable: true + curated: + description: >- + the script set was pruned to idiomatic actions rather than + generated one per endpoint + type: boolean + metadata_source: + description: >- + whether the provider knowledge was authored (curated) or + inferred from the shipped scripts (derived) + type: string + enum: + - curated + - derived + meta: + description: >- + the integration's authored meta.json verbatim; the content + repo owns its schema, so it is passed through unvalidated + type: object + nullable: true + additionalProperties: true + meta_updated_at: + type: string + nullable: true + derived: + description: facts computed from the integration's shipped scripts + type: object + properties: + api_hosts: + type: array + items: + type: object + properties: + host: + type: string + count: + type: number + required: + - host + - count + style: + type: string + enum: + - fetch + - sdk + - mixed + - unknown + languages: + type: object + additionalProperties: + type: number + script_counts: + type: object + properties: + total: + type: number + by_kind: + type: object + additionalProperties: + type: number + required: + - total + - by_kind + top_scripts: + type: array + items: + type: object + properties: + path: + type: string + ask_id: + type: number + version_id: + type: number + summary: + type: string + description: + type: string + nullable: true + kind: + type: string + language: + type: string + nullable: true + views: + type: number + votes: + type: number + required: + - path + - summary + - kind + required: + - api_hosts + - style + - languages + - script_counts + - top_scripts + resource_types: + type: array + items: + type: object + properties: + id: + type: number + name: + type: string + description: + type: string + nullable: true + schema: + type: object + additionalProperties: true + required: + - name + - schema + required: + - app + - display_name + - curated + - metadata_source + - derived + - resource_types + /flows/hub/list: get: summary: list all hub flows @@ -8731,6 +8885,8 @@ paths: type: number summary: type: string + description: + type: string app: type: string kind: diff --git a/backend/windmill-api/src/integration.rs b/backend/windmill-api/src/integration.rs index 2def3e6ed0..8251e8016c 100644 --- a/backend/windmill-api/src/integration.rs +++ b/backend/windmill-api/src/integration.rs @@ -1,9 +1,16 @@ use crate::{db::DB, HTTP_CLIENT}; -use axum::{extract::Query, response::IntoResponse, routing::get, Extension, Router}; +use axum::{ + extract::{Path, Query}, + response::IntoResponse, + routing::get, + Extension, Router, +}; use windmill_common::{error::Error, utils::query_elems_from_hub, HUB_BASE_URL}; pub fn global_service() -> Router { - Router::new().route("/hub/list", get(list_hub_integrations)) + Router::new() + .route("/hub/list", get(list_hub_integrations)) + .route("/hub/{app}/meta", get(get_hub_integration_meta)) } #[derive(serde::Deserialize)] @@ -29,3 +36,21 @@ async fn list_hub_integrations( .await?; Ok::<_, Error>((status_code, headers, response)) } + +/// Everything a caller needs to write code against one integration: its resource +/// types, the provider knowledge the content repo authored, and facts derived from +/// the shipped scripts. A hub older than the endpoint answers 404, which passes +/// through as-is. +async fn get_hub_integration_meta( + Path(app): Path, + Extension(db): Extension, +) -> impl IntoResponse { + let (status_code, headers, response) = query_elems_from_hub( + &HTTP_CLIENT, + &format!("{}/integrations/{}/meta", **HUB_BASE_URL.load(), app), + None, + &db, + ) + .await?; + Ok::<_, Error>((status_code, headers, response)) +} diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index e042809f1e..6d42c4593e 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -94,6 +94,7 @@ import { createToolDef, droppedOptionKeys, createSearchHubScriptsTool, + getHubIntegrationTool, executeFlowStepTestRun, executeTestRun, findAndReplace, @@ -1206,7 +1207,8 @@ Rules: - Variable values are never readable. For secrets, create a secret variable and reference it from resources as "$var:path/to/variable". - Use search_resource_types before write_resource, and get_trigger_schema before write_trigger: the trigger config fields differ per kind and are not listed in the write_trigger definition. - When script or raw app code needs an external npm package you are not fully familiar with, use search_npm_packages to find it and get its documentation and type definitions. Link the package documentation in your answer when you rely on it. -- Hub scripts are prebuilt, vetted integrations for third-party services, hosted outside the workspace under \`hub///\` paths. Check search_hub_scripts before hand-writing code against a third-party API, even when the user never mentions the hub; read a result with read_workspace_item type "script" and its hub path to get its code, language, and input schema. Use what you find in whichever way fits: reference the hub path directly from a flow module or app runnable when a script already does the job, copy it into a workspace draft and adapt it when it is close (record the source hub path in the draft's description), or treat it as a worked example of that integration — which SDK or endpoint it calls, how it authenticates, which resource type it takes — and write your own. A script that does not do what the user asked is still worth reading when it is the only example of that integration: pass its \`integration\` back to search_hub_scripts to list that integration's other scripts with their descriptions, or use the \`suggested_integrations\` a search hands back when it finds nothing.${webSearchBullet} +- Hub scripts are prebuilt, vetted integrations for third-party services, hosted outside the workspace under \`hub///\` paths. Check search_hub_scripts before hand-writing code against a third-party API, even when the user never mentions the hub; read a result with read_workspace_item type "script" and its hub path to get its code, language, and input schema. Use what you find in whichever way fits: reference the hub path directly from a flow module or app runnable when a script already does the job, copy it into a workspace draft and adapt it when it is close (record the source hub path in the draft's description), or take it as a worked example and write your own. A script that does not do what the user asked is still worth reading when it is the only example of that integration: pass its \`integration\` back to search_hub_scripts to list that integration's other scripts with their descriptions, or use the \`suggested_integrations\` a search hands back when it finds nothing. +- Before writing your own code against an integration the hub covers, call get_hub_integration with its slug: it returns the resource type to take, the auth, pagination, enums, error codes and known gotchas in one call, which beats inferring them from script bodies.${webSearchBullet} - Use get_db_schema with a database resource path to fetch its tables and columns before writing SQL (or a script querying that database). - Use get_instructions before writing scripts, flows, resources, or apps. For scripts, pass the target language. ${pipelineBullet} @@ -2625,6 +2627,7 @@ export const globalTools: Tool<{}>[] = [ } }, createSearchHubScriptsTool(false), + getHubIntegrationTool, searchNpmPackagesTool, searchDocsTool, readDocsPageTool, diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index a08bc79629..2aa9712222 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -1406,8 +1406,8 @@ describe('createSearchHubScriptsTool', () => { }) // Browsing by app is what surfaces an integration's other scripts as examples, - // and only the top-scripts endpoint carries the descriptions that make them - // judgeable — the semantic one would both omit those and filter near-misses out. + // and only the top-scripts endpoint takes no query and applies no similarity + // floor, so the near-misses worth reading survive instead of being cut. it('lists an integration through the top-scripts endpoint, with descriptions', async () => { const { ScriptService } = await import('$lib/gen') const getTopHubScripts = vi.fn(async () => ({ @@ -1454,3 +1454,99 @@ describe('createSearchHubScriptsTool', () => { expect(JSON.parse(raw)).toEqual({ results: [], suggested_integrations: ['stripe'] }) }) }) + +describe('getHubIntegrationTool', () => { + const doc = { + app: 'confluence', + display_name: 'Confluence', + curated: true, + metadata_source: 'curated', + meta: { gotchas: ['Auth is Basic with an API token, not the password'] }, + derived: { + api_hosts: [{ host: 'api.atlassian.com', count: 12 }], + style: 'fetch', + languages: { bun: 14 }, + script_counts: { total: 14, by_kind: { script: 13 } }, + top_scripts: [ + { path: 'hub/1/confluence/create_page', summary: 'Create page', language: 'bun' } + ] + }, + resource_types: [{ name: 'confluence', schema: { type: 'object' } }] + } + + // Hand-validated provider knowledge and facts inferred from script bodies must + // stay under separate keys, or the model will report guesses as verified. + it('keeps authored notes apart from what was inferred from the scripts', async () => { + const { IntegrationService } = await import('$lib/gen') + Object.assign(IntegrationService, { getHubIntegrationMeta: vi.fn(async () => doc) }) + + const { getHubIntegrationTool } = await import('./shared') + const parsed = JSON.parse( + await getHubIntegrationTool.fn({ + args: { integration: 'confluence' }, + toolId: 't1', + toolCallbacks: { setToolStatus: vi.fn() } + } as any) + ) + + expect(parsed.verified_provider_notes).toEqual(doc.meta) + expect(parsed.observed_from_scripts).toEqual({ + api_hosts: ['api.atlassian.com'], + style: 'fetch', + languages: ['bun'], + script_counts: { total: 14, by_kind: { script: 13 } } + }) + }) + + // The hub repo owns this payload, so a hub older than this client can send a + // subset. Reading it must degrade to less content, never to a tool error. + it('returns what an older hub sent instead of failing on the missing sections', async () => { + const { IntegrationService } = await import('$lib/gen') + Object.assign(IntegrationService, { + getHubIntegrationMeta: vi.fn(async () => ({ + app: 'stripe', + display_name: 'Stripe', + curated: false + })) + }) + + const { getHubIntegrationTool } = await import('./shared') + const parsed = JSON.parse( + await getHubIntegrationTool.fn({ + args: { integration: 'stripe' }, + toolId: 't1', + toolCallbacks: { setToolStatus: vi.fn() } + } as any) + ) + + expect(parsed.integration).toBe('stripe') + expect(parsed.observed_from_scripts).toBeUndefined() + expect(parsed.resource_types).toEqual([]) + expect(parsed.example_scripts).toEqual([]) + }) + + // A hub with no such integration and one too old to serve the endpoint both 404; + // neither may surface as a tool error, since the model can still read scripts. + it('suggests real slugs instead of failing when the integration is unknown', async () => { + const { IntegrationService } = await import('$lib/gen') + Object.assign(IntegrationService, { + getHubIntegrationMeta: vi.fn(async () => { + throw new Error('Not Found') + }), + listHubIntegrations: vi.fn(async () => [{ name: 'stripe' }, { name: 'slack' }]) + }) + + const { getHubIntegrationTool, clearHubIntegrationsCache } = await import('./shared') + clearHubIntegrationsCache() + const parsed = JSON.parse( + await getHubIntegrationTool.fn({ + args: { integration: 'stripe_billing' }, + toolId: 't1', + toolCallbacks: { setToolStatus: vi.fn() } + } as any) + ) + + expect(parsed.error).toContain('stripe_billing') + expect(parsed.suggested_integrations).toEqual(['stripe']) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index cb1fb294f4..ffc2edb460 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -1247,8 +1247,8 @@ export function isHubPath(path: string): boolean { const MAX_BROWSED_HUB_SCRIPTS = 20 const MAX_SUGGESTED_INTEGRATIONS = 5 -/** Common shape of the two hub listings. Only the top-scripts one carries a - * description; the semantic search response has no field for it. */ +/** Common shape of the two hub listings. Both carry a description, but the hub + * has none for roughly a fifth of its scripts. */ type HubScriptHit = { version_id: number; app: string; summary: string; description?: string } /** The integration slugs are a large but static list, so one fetch per session @@ -1284,6 +1284,91 @@ export const clearHubIntegrationsCache = () => { hubIntegrationsCache = undefined } +const getHubIntegrationSchema = z.object({ + integration: z + .string() + .describe( + 'Integration slug, e.g. "stripe". Take it from a search_hub_scripts result\'s `integration`, or guess the vendor name: a wrong guess comes back with the closest real slugs.' + ) +}) + +const getHubIntegrationToolDef = createToolDef( + getHubIntegrationSchema, + 'get_hub_integration', + 'Read how one integration works before writing code against it: which resource type it takes, its auth, pagination, enums, error codes and gotchas, plus its most-used scripts as examples.' +) + +/** Enough to show the integration's idiom; search_hub_scripts is the way to find + * a specific one. */ +const MAX_INTEGRATION_EXAMPLES = 5 + +export const getHubIntegrationTool = { + def: getHubIntegrationToolDef, + fn: async ({ args, toolId, toolCallbacks }) => { + const { integration } = getHubIntegrationSchema.parse(args) + toolCallbacks.setToolStatus(toolId, { content: `Reading the ${integration} integration...` }) + + let doc: Awaited> + try { + doc = await IntegrationService.getHubIntegrationMeta({ app: integration }) + } catch { + // An unknown slug and a hub predating the endpoint both answer 404, and the + // response is the same either way: hand back real slugs so the model can + // retry or fall back to reading scripts. + toolCallbacks.setToolStatus(toolId, { content: `No hub integration named ${integration}` }) + const suggested = await suggestHubIntegrations(integration) + return JSON.stringify({ + error: `No hub metadata for "${integration}".`, + suggested_integrations: suggested + }) + } + + toolCallbacks.setToolStatus(toolId, { content: `Read the ${doc.display_name} integration` }) + // The hub owns this response's shape and can be older than this client, so + // every section is read as optional: a hub that sends less should return less, + // not fail the call. + const derived = doc.derived + return JSON.stringify({ + integration: doc.app, + display_name: doc.display_name, + ...(doc.description ? { description: doc.description } : {}), + ...(doc.docs_url ? { docs_url: doc.docs_url } : {}), + // Authored provider knowledge and facts inferred from the scripts stay + // separate: only the former was checked against the live API. + ...(doc.meta ? { verified_provider_notes: doc.meta } : {}), + ...(derived + ? { + observed_from_scripts: { + api_hosts: derived.api_hosts?.map((h) => h.host) ?? [], + style: derived.style, + languages: Object.keys(derived.languages ?? {}), + script_counts: derived.script_counts + } + } + : {}), + // Said in the payload rather than the system prompt: it is only true of some + // integrations, and only matters once the model has asked about one. + ...(doc.curated + ? {} + : { + scripts_note: + 'These scripts were generated one per API endpoint from a spec: good for the endpoint shapes, weak as style examples.' + }), + resource_types: (doc.resource_types ?? []).map((rt) => ({ + name: rt.name, + ...(rt.description ? { description: rt.description } : {}), + schema: rt.schema + })), + example_scripts: (derived?.top_scripts ?? []).slice(0, MAX_INTEGRATION_EXAMPLES).map((s) => ({ + path: s.path, + summary: s.summary, + ...(s.description ? { description: s.description } : {}), + ...(s.language ? { language: s.language } : {}) + })) + }) + } +} satisfies Tool<{}> + export const createSearchHubScriptsTool = (withContent: boolean = false) => ({ def: searchHubScriptsToolDef, fn: async ({ args, toolId, toolCallbacks }) => { @@ -1298,9 +1383,9 @@ export const createSearchHubScriptsTool = (withContent: boolean = false) => ({ toolCallbacks.setToolStatus(toolId, { content: `Searching hub scripts for ${subject}...` }) // Listing an integration goes through the hub's top-scripts endpoint rather - // than the semantic one: it is the only one carrying each script's - // description, and it has no similarity floor to drop the near-misses that - // are worth reading as examples of how the integration is used. + // than the semantic one: it takes no query, and it applies no similarity + // floor, so the near-misses worth reading as examples of how the integration + // is used survive instead of being cut. const scripts: HubScriptHit[] = query ? await ScriptService.queryHubScripts({ text: query, kind: 'script', app }) : ((