From 2ca2fecce9f927aecea4cbf0e569ba4d59001eb7 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 21 Aug 2026 10:08:45 +0200 Subject: [PATCH] fix: treat hub curated as three-state and speak only for a stated true Co-Authored-By: Claude Opus 5 --- backend/windmill-api/openapi.yaml | 7 ++-- .../components/copilot/chat/shared.test.ts | 33 +++++++++++++++---- .../src/lib/components/copilot/chat/shared.ts | 18 +++++----- 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index ad6b9e44da..3af4425fc4 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -8387,9 +8387,11 @@ paths: nullable: true curated: description: >- - the script set was pruned to idiomatic actions rather than - generated one per endpoint + whether the script set was pruned to idiomatic actions rather + than generated one per endpoint; null when nobody has assessed + it, which is distinct from a stated false type: boolean + nullable: true metadata_source: description: >- whether the provider knowledge was authored (curated) or @@ -8503,7 +8505,6 @@ paths: required: - app - display_name - - curated - metadata_source - derived - resource_types diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index 0e306e0ef2..ddda2c7d76 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -1459,9 +1459,8 @@ describe('getHubIntegrationTool', () => { const doc = { app: 'confluence', display_name: 'Confluence', - // Authored, but curated is opt-in per integration and unset for every - // hand-curated one the hub serves today. - curated: false, + // Authored knowledge, but nobody has asserted whether the script set was pruned. + curated: null, metadata_source: 'curated', meta: { gotchas: ['Auth is Basic with an API token, not the password'] }, derived: { @@ -1492,8 +1491,6 @@ describe('getHubIntegrationTool', () => { ) expect(parsed.verified_provider_notes).toEqual(doc.meta) - // Every hand-curated integration on the hub still reports curated=false, so an - // authored meta.json has to be enough on its own to suppress the note. expect(parsed.scripts_note).toBeUndefined() expect(parsed.observed_from_scripts).toEqual({ api_hosts: ['api.atlassian.com'], @@ -1525,12 +1522,36 @@ describe('getHubIntegrationTool', () => { ) expect(parsed.integration).toBe('stripe') - expect(parsed.scripts_note).toContain('generated one per API endpoint') expect(parsed.observed_from_scripts).toBeUndefined() expect(parsed.resource_types).toEqual([]) expect(parsed.example_scripts).toEqual([]) }) + // `curated` has three states and only `true` is a licence to call the scripts good + // examples. `false` and the unassessed `null` must both stay silent, or the 212 + // integrations nobody has looked at get characterised anyway. + it.each([ + [true, true], + [false, false], + [null, false] + ])('speaks about the script set only when curated is true (%s)', async (curated, expected) => { + const { IntegrationService } = await import('$lib/gen') + Object.assign(IntegrationService, { + getHubIntegrationMeta: vi.fn(async () => ({ ...doc, curated })) + }) + + const { getHubIntegrationTool } = await import('./shared') + const parsed = JSON.parse( + await getHubIntegrationTool.fn({ + args: { integration: 'confluence' }, + toolId: 't1', + toolCallbacks: { setToolStatus: vi.fn() } + } as any) + ) + + expect(!!parsed.scripts_note).toBe(expected) + }) + // A hub with no such integration and one too old to serve the endpoint both 404; // neither may surface as a tool error, since the model can still read scripts. it('suggests real slugs instead of failing when the integration is unknown', async () => { diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 8d4d5e8614..1633c87710 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -1346,16 +1346,16 @@ export const getHubIntegrationTool = { } } : {}), - // Said in the payload rather than the system prompt: it is only true of some - // integrations, and only matters once the model has asked about one. Authored - // knowledge counts as evidence of curation on its own, because `curated` is - // opt-in per integration and unset for every hand-curated one on the hub today. - ...(doc.meta || doc.curated - ? {} - : { + // `curated` is three-state: only an integration's own meta.json asserts it, and + // anything else means nobody has assessed the script set. Speak only for a + // positive assertion, so an unassessed integration is never characterised + // either way; `script_counts` below is the factual signal for the rest. + ...(doc.curated === true + ? { scripts_note: - 'These scripts were generated one per API endpoint from a spec: good for the endpoint shapes, weak as style examples.' - }), + 'These scripts were pruned to idiomatic actions rather than generated one per API endpoint, so they are worth following as style examples.' + } + : {}), resource_types: (doc.resource_types ?? []).map((rt) => ({ name: rt.name, ...(rt.description ? { description: rt.description } : {}),