diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index b6dbe042b8..692a83edeb 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -1556,14 +1556,16 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon } if (path === '/api/integrations/hub/list') { const apps = [...new Set(BENCHMARK_HUB_SCRIPTS.map((script) => script.app))].sort() - // Only the integrations lifted whole from the content repo have a meta.json, so - // the flag separates them from the rest here the way it does on the hub, where - // 18 of ~216 qualify. - const documented = new Set( - REAL_HUB_INTEGRATIONS.filter((integration) => integration.meta).map( + // Read off both fixtures the metadata endpoint serves from, or the list would + // call an integration undocumented and then hand back its authored notes. + const documented = new Set([ + ...REAL_HUB_INTEGRATIONS.filter((integration) => integration.meta).map( (integration) => integration.app - ) - ) + ), + ...Object.entries(BENCHMARK_HUB_INTEGRATION_META) + .filter(([, entry]) => entry.meta) + .map(([app]) => app) + ]) return Response.json(apps.map((name) => ({ name, documented: documented.has(name) }))) } if (path.startsWith('/api/scripts/hub/get_full/')) { diff --git a/ai_evals/adapters/frontend/mockBackendApi.test.ts b/ai_evals/adapters/frontend/mockBackendApi.test.ts index d9cfa08bf4..a50a66e1f5 100644 --- a/ai_evals/adapters/frontend/mockBackendApi.test.ts +++ b/ai_evals/adapters/frontend/mockBackendApi.test.ts @@ -81,3 +81,23 @@ describe('benchmark API catalog', () => { expect(handleBenchmarkApiFetch(`/api/w/${WORKSPACE}/jobs_u/get/missing`).status).toBe(404) }) }) + +// A benchmark whose list calls an integration undocumented while its metadata endpoint +// hands back authored notes teaches the model the flag means nothing. +describe('benchmark hub integration list', () => { + beforeEach(() => resetBenchmarkMockBackend()) + + it('flags exactly the integrations whose metadata carries authored notes', async () => { + const listed = (await handleBenchmarkApiFetch('/api/integrations/hub/list').json()) as Array<{ + name: string + documented: boolean + }> + expect(listed.length).toBeGreaterThan(0) + + for (const { name, documented } of listed) { + const res = handleBenchmarkApiFetch(`/api/integrations/hub/${name}/meta`) + const authored = res.status === 200 && !!((await res.json()) as { meta?: unknown }).meta + expect(authored).toBe(documented) + } + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 1c814cbc83..66711acee3 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -1415,12 +1415,9 @@ async function loadHubIntegrations(): Promise { return hubIntegrationsCache } -/** Which integrations carry hand-written provider knowledge, lowercased because a - * hub slug is case-sensitive and both casings of one vendor can exist. The mark says - * get_hub_integration has verified pagination, enums and gotchas to add for this one. - * It must never become a reason to skip the call for the others: that call still - * carries the resource type and the usage-ranked examples, neither of which is - * guessable from a search result. */ +/** Which integrations carry provider knowledge checked against the live API, + * lowercased because a hub slug is case-sensitive and both casings can exist. The + * mark means there is more to read, never that the call is skippable elsewhere. */ async function documentedIntegrations(): Promise> { const available = await loadHubIntegrations() return new Set(available.filter((i) => i.documented).map((i) => i.name.toLowerCase()))