diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 692a83edeb..1f3d3de3cf 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -1278,6 +1278,9 @@ function listBenchmarkHubScriptsByApp(app: string | null, kind: string | null) { /** What `/integrations/hub//meta` serves. Carries the integration's conventions * and none of the endpoints a case asks the model to write, so the tool shortens the * path to that knowledge without supplying answers. */ +/** The integrations a case writes against, none of which the content repo documents — + * the same majority shape as the live hub, where 18 of ~216 carry a meta.json. Their + * conventions live in their shipped scripts, which is what these cases exercise. */ const BENCHMARK_HUB_INTEGRATION_META: Record< string, { @@ -1293,37 +1296,14 @@ const BENCHMARK_HUB_INTEGRATION_META: Record< description: 'Subscription analytics for recurring-revenue businesses.', docs_url: 'https://developers.baremetrics.com/reference', curated: null, - meta: { - api_docs: 'https://developers.baremetrics.com/reference', - auth: 'Bearer token. The resource carries a single `apiKey` field; send it as `Authorization: Bearer `.', - base_url: 'https://api.baremetrics.com/v1', - pagination: { - pattern: 'page', - request_params: { page: 'page', per_page: 'per_page' } - }, - gotchas: [ - 'Every write is scoped to a source, so the source id is part of the path rather than the body.', - 'The API answers 200 with an empty `{}` body on some writes; treat a 2xx as success rather than parsing a payload.' - ], - errors: { - '401': 'The apiKey is missing or revoked.', - '404': 'Unknown source id, or the key cannot see that source.' - } - } + meta: null }, holded: { display_name: 'Holded', description: 'Invoicing, accounting and CRM for small businesses.', docs_url: 'https://developers.holded.com/reference', curated: null, - meta: { - api_docs: 'https://developers.holded.com/reference', - auth: 'The resource carries an `apiKey`; send it in the `key` header, not as a bearer token.', - base_url: 'https://api.holded.com/api', - gotchas: [ - 'Each product area has its own path segment (invoicing, crm, projects) after the version.' - ] - } + meta: null } } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 332d970319..b54df4f349 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -8647,10 +8647,10 @@ paths: type: string documented: description: >- - whether the integration carries hand-written provider - knowledge, so a caller can tell before spending a call on - /integrations/hub/{app}/meta. Absent on a hub predating the - flag. + whether the integration carries provider knowledge checked + against the live API, on top of the resource type and example + scripts /integrations/hub/{app}/meta returns for any + integration. Absent on a hub predating the flag. type: boolean required: - name diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 7417558400..fb9763c732 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -1306,7 +1306,7 @@ Rules: - Use search_resource_types before write_resource, and get_trigger_schema before write_trigger: the trigger config fields differ per kind and are not listed in the write_trigger definition. - When script or raw app code needs an external npm package you are not fully familiar with, use search_npm_packages to find it and get its documentation and type definitions. Link the package documentation in your answer when you rely on it. - Hub scripts are prebuilt, vetted integrations for third-party services, hosted outside the workspace under \`hub///\` paths. Check search_hub_scripts before hand-writing code against a third-party API, even when the user never mentions the hub; read a result with read_workspace_item type "script" and its hub path to get its code, language, and input schema. Use what you find in whichever way fits: reference the hub path directly from a flow module or app runnable when a script already does the job, copy it into a workspace draft and adapt it when it is close (record the source hub path in the draft's description), or take it as a worked example and write your own. A script that does not do what the user asked is still worth reading when it is the only example of that integration: pass its \`integration\` back to search_hub_scripts to list that integration's other scripts with their descriptions, or use the \`suggested_integrations\` a search hands back when it finds nothing. -- Before writing your own code against an integration the hub covers, call get_hub_integration with its slug: it returns the resource type to take, its auth fields and the integration's most-used scripts, which beats inferring them from script bodies. Call it for the integration you are about to write against whichever it is. A search marks an integration \`documented\` when the hub additionally holds provider knowledge checked against the live API — pagination, enums, error codes and gotchas — so read that closely where it appears rather than trusting your own memory of the API.${webSearchBullet} +- Before writing your own code against an integration the hub covers, call get_hub_integration with its slug: it returns the resource type to take, its auth fields and the integration's most-used scripts, which beats inferring them from script bodies. Call it for the integration you are about to write against, whichever it is. A search marks an integration \`documented\` when the hub additionally holds provider knowledge checked against the live API — pagination, enums, error codes and gotchas — so read that closely where it appears rather than trusting your own memory of the API.${webSearchBullet} - Use get_db_schema with a database resource path to fetch its tables and columns before writing SQL (or a script querying that database). - Use get_instructions before writing scripts, flows, resources, or apps. For scripts, pass the target language. ${pipelineBullet} diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 66711acee3..47e0c6cce1 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -1403,8 +1403,9 @@ async function loadHubIntegrations(): Promise { const integrations = await IntegrationService.listHubIntegrations({ kind: 'script' }) hubIntegrationsCache = integrations.map((i) => ({ name: i.name, - // A hub predating the flag omits it, and so does one that has no authored - // notes at all. Both mean the same thing to a caller: read the scripts. + // A hub predating the flag omits it, and so does one with no authored + // notes. Both mean the same thing: nothing to read beyond what the + // metadata call returns for every integration. documented: i.documented === true })) } catch (err) {