fix: keep the benchmark's documented flag true to what it serves

The eval hub reported Baremetrics and Holded as undocumented while its
metadata endpoint handed back their authored notes, so a case could teach
the model the flag means nothing. Derive it from both fixture sources, and
pin the agreement.

Also trims the documentedIntegrations comment to the four lines AGENTS.md
allows, keeping the case-folding constraint.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

ai_evals/adapters/frontend/mockBackend.ts

@@ -1556,14 +1556,16 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon
 	}
 	if (path === '/api/integrations/hub/list') {
 		const apps = [...new Set(BENCHMARK_HUB_SCRIPTS.map((script) => script.app))].sort()
-		// Only the integrations lifted whole from the content repo have a meta.json, so
-		// the flag separates them from the rest here the way it does on the hub, where
-		// 18 of ~216 qualify.
-		const documented = new Set(
-			REAL_HUB_INTEGRATIONS.filter((integration) => integration.meta).map(
+		// Read off both fixtures the metadata endpoint serves from, or the list would
+		// call an integration undocumented and then hand back its authored notes.
+		const documented = new Set([
+			...REAL_HUB_INTEGRATIONS.filter((integration) => integration.meta).map(
 				(integration) => integration.app
-			)
-		)
+			),
+			...Object.entries(BENCHMARK_HUB_INTEGRATION_META)
+				.filter(([, entry]) => entry.meta)
+				.map(([app]) => app)
+		])
 		return Response.json(apps.map((name) => ({ name, documented: documented.has(name) })))
 	}
 	if (path.startsWith('/api/scripts/hub/get_full/')) {

ai_evals/adapters/frontend/mockBackendApi.test.ts

@@ -81,3 +81,23 @@ describe('benchmark API catalog', () => {
 		expect(handleBenchmarkApiFetch(`/api/w/${WORKSPACE}/jobs_u/get/missing`).status).toBe(404)
 	})
 })
+
+// A benchmark whose list calls an integration undocumented while its metadata endpoint
+// hands back authored notes teaches the model the flag means nothing.
+describe('benchmark hub integration list', () => {
+	beforeEach(() => resetBenchmarkMockBackend())
+
+	it('flags exactly the integrations whose metadata carries authored notes', async () => {
+		const listed = (await handleBenchmarkApiFetch('/api/integrations/hub/list').json()) as Array<{
+			name: string
+			documented: boolean
+		}>
+		expect(listed.length).toBeGreaterThan(0)
+
+		for (const { name, documented } of listed) {
+			const res = handleBenchmarkApiFetch(`/api/integrations/hub/${name}/meta`)
+			const authored = res.status === 200 && !!((await res.json()) as { meta?: unknown }).meta
+			expect(authored).toBe(documented)
+		}
+	})
+})

frontend/src/lib/components/copilot/chat/shared.ts

@@ -1415,12 +1415,9 @@ async function loadHubIntegrations(): Promise<HubIntegration[]> {
 	return hubIntegrationsCache
 }
 
-/** Which integrations carry hand-written provider knowledge, lowercased because a
- * hub slug is case-sensitive and both casings of one vendor can exist. The mark says
- * get_hub_integration has verified pagination, enums and gotchas to add for this one.
- * It must never become a reason to skip the call for the others: that call still
- * carries the resource type and the usage-ranked examples, neither of which is
- * guessable from a search result. */
+/** Which integrations carry provider knowledge checked against the live API,
+ * lowercased because a hub slug is case-sensitive and both casings can exist. The
+ * mark means there is more to read, never that the call is skippable elsewhere. */
 async function documentedIntegrations(): Promise<Set<string>> {
 	const available = await loadHubIntegrations()
 	return new Set(available.filter((i) => i.documented).map((i) => i.name.toLowerCase()))