From f8d8ea58a03817147c442e87196ddca13e1b7999 Mon Sep 17 00:00:00 2001
From: hugocasa <hugo@casademont.ch>
Date: Fri, 28 Aug 2026 17:41:21 +0200
Subject: [PATCH] fix: keep the benchmark's documented flag true to what it
 serves

The eval hub reported Baremetrics and Holded as undocumented while its
metadata endpoint handed back their authored notes, so a case could teach
the model the flag means nothing. Derive it from both fixture sources, and
pin the agreement.

Also trims the documentedIntegrations comment to the four lines AGENTS.md
allows, keeping the case-folding constraint.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 ai_evals/adapters/frontend/mockBackend.ts     | 16 ++++++++-------
 .../adapters/frontend/mockBackendApi.test.ts  | 20 +++++++++++++++++++
 .../src/lib/components/copilot/chat/shared.ts |  9 +++------
 3 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts
index b6dbe042b8..692a83edeb 100644
--- a/ai_evals/adapters/frontend/mockBackend.ts
+++ b/ai_evals/adapters/frontend/mockBackend.ts
@@ -1556,14 +1556,16 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon
 	}
 	if (path === '/api/integrations/hub/list') {
 		const apps = [...new Set(BENCHMARK_HUB_SCRIPTS.map((script) => script.app))].sort()
-		// Only the integrations lifted whole from the content repo have a meta.json, so
-		// the flag separates them from the rest here the way it does on the hub, where
-		// 18 of ~216 qualify.
-		const documented = new Set(
-			REAL_HUB_INTEGRATIONS.filter((integration) => integration.meta).map(
+		// Read off both fixtures the metadata endpoint serves from, or the list would
+		// call an integration undocumented and then hand back its authored notes.
+		const documented = new Set([
+			...REAL_HUB_INTEGRATIONS.filter((integration) => integration.meta).map(
 				(integration) => integration.app
-			)
-		)
+			),
+			...Object.entries(BENCHMARK_HUB_INTEGRATION_META)
+				.filter(([, entry]) => entry.meta)
+				.map(([app]) => app)
+		])
 		return Response.json(apps.map((name) => ({ name, documented: documented.has(name) })))
 	}
 	if (path.startsWith('/api/scripts/hub/get_full/')) {
diff --git a/ai_evals/adapters/frontend/mockBackendApi.test.ts b/ai_evals/adapters/frontend/mockBackendApi.test.ts
index d9cfa08bf4..a50a66e1f5 100644
--- a/ai_evals/adapters/frontend/mockBackendApi.test.ts
+++ b/ai_evals/adapters/frontend/mockBackendApi.test.ts
@@ -81,3 +81,23 @@ describe('benchmark API catalog', () => {
 		expect(handleBenchmarkApiFetch(`/api/w/${WORKSPACE}/jobs_u/get/missing`).status).toBe(404)
 	})
 })
+
+// A benchmark whose list calls an integration undocumented while its metadata endpoint
+// hands back authored notes teaches the model the flag means nothing.
+describe('benchmark hub integration list', () => {
+	beforeEach(() => resetBenchmarkMockBackend())
+
+	it('flags exactly the integrations whose metadata carries authored notes', async () => {
+		const listed = (await handleBenchmarkApiFetch('/api/integrations/hub/list').json()) as Array<{
+			name: string
+			documented: boolean
+		}>
+		expect(listed.length).toBeGreaterThan(0)
+
+		for (const { name, documented } of listed) {
+			const res = handleBenchmarkApiFetch(`/api/integrations/hub/${name}/meta`)
+			const authored = res.status === 200 && !!((await res.json()) as { meta?: unknown }).meta
+			expect(authored).toBe(documented)
+		}
+	})
+})
diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts
index 1c814cbc83..66711acee3 100644
--- a/frontend/src/lib/components/copilot/chat/shared.ts
+++ b/frontend/src/lib/components/copilot/chat/shared.ts
@@ -1415,12 +1415,9 @@ async function loadHubIntegrations(): Promise<HubIntegration[]> {
 	return hubIntegrationsCache
 }
 
-/** Which integrations carry hand-written provider knowledge, lowercased because a
- * hub slug is case-sensitive and both casings of one vendor can exist. The mark says
- * get_hub_integration has verified pagination, enums and gotchas to add for this one.
- * It must never become a reason to skip the call for the others: that call still
- * carries the resource type and the usage-ranked examples, neither of which is
- * guessable from a search result. */
+/** Which integrations carry provider knowledge checked against the live API,
+ * lowercased because a hub slug is case-sensitive and both casings can exist. The
+ * mark means there is more to read, never that the call is skippable elsewhere. */
 async function documentedIntegrations(): Promise<Set<string>> {
 	const available = await loadHubIntegrations()
 	return new Set(available.filter((i) => i.documented).map((i) => i.name.toLowerCase()))