diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index 291f8633ab..d46fa5e065 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -1405,51 +1405,57 @@ describe('createSearchHubScriptsTool', () => { expect(results[1].content).toBe('ok') }) - // Semantic search ranks a named vendor weakly against the task words, so "create a - // jira ticket" put netlify, zendesk and intercom above every Jira script. - it('narrows the search to an integration the query names outright', async () => { + // Ranking buries an integration the query names when other integrations' scripts + // mention it: none of Salesforce's own scripts come back for "an account in + // salesforce", because Pinterest's summaries say Salesforce too. + it("adds a named integration's own scripts when ranking left them out", async () => { const { ScriptService, IntegrationService } = await import('$lib/gen') - const queryHubScripts = vi.fn(async () => [hit(9, 'jira', 'Create issue')]) + const queryHubScripts = vi.fn(async ({ app }: { app?: string }) => + app === 'salesforce' + ? [hit(9, 'salesforce', 'SOSL Search')] + : [hit(1, 'pinterest', 'Get Salesforce account details')] + ) Object.assign(ScriptService, { queryHubScripts }) Object.assign(IntegrationService, { - listHubIntegrations: vi.fn(async () => [{ name: 'jira' }, { name: 'netlify' }]) + listHubIntegrations: vi.fn(async () => [{ name: 'salesforce' }, { name: 'pinterest' }]) }) const { createSearchHubScriptsTool, clearHubIntegrationsCache } = await import('./shared') clearHubIntegrationsCache() - await createSearchHubScriptsTool().fn({ - args: { query: 'create a jira ticket' }, + const raw = await createSearchHubScriptsTool().fn({ + args: { query: 'look up an account in salesforce' }, toolId: 't1', toolCallbacks: { setToolStatus: vi.fn() } } as any) - expect(queryHubScripts).toHaveBeenCalledWith({ - text: 'create a jira ticket', - kind: 'script', - app: 'jira' - }) + // the broad hits survive; the named integration is added, not substituted + expect(JSON.parse(raw).results.map((r: any) => r.integration)).toEqual([ + 'pinterest', + 'salesforce' + ]) }) - // A fuzzy test would read `send` in "send an invoice" as sendgrid and search the - // wrong integration outright, which is worse than not narrowing at all. - it('leaves the search broad when no integration is named exactly', async () => { + // `monday`, `box` and `linear` are integrations and ordinary words, so a mention + // must never filter the results — semantic search already ranks them when meant. + it('leaves results alone when the named integration is already among them', async () => { const { ScriptService, IntegrationService } = await import('$lib/gen') - const queryHubScripts = vi.fn(async () => [hit(9, 'paypal', 'Create invoice')]) + const queryHubScripts = vi.fn(async () => [hit(1, 'monday', 'Create item')]) Object.assign(ScriptService, { queryHubScripts }) Object.assign(IntegrationService, { - listHubIntegrations: vi.fn(async () => [{ name: 'sendgrid' }, { name: 'paypal' }]) + listHubIntegrations: vi.fn(async () => [{ name: 'monday' }]) }) const { createSearchHubScriptsTool, clearHubIntegrationsCache } = await import('./shared') clearHubIntegrationsCache() await createSearchHubScriptsTool().fn({ - args: { query: 'send an invoice to a client' }, + args: { query: 'run this every monday' }, toolId: 't1', toolCallbacks: { setToolStatus: vi.fn() } } as any) + expect(queryHubScripts).toHaveBeenCalledTimes(1) expect(queryHubScripts).toHaveBeenCalledWith({ - text: 'send an invoice to a client', + text: 'run this every monday', kind: 'script', app: undefined }) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 8d4927e01f..339f33ba96 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -1245,6 +1245,7 @@ export function isHubPath(path: string): boolean { } const MAX_BROWSED_HUB_SCRIPTS = 20 +const MAX_MENTIONED_INTEGRATION_HITS = 3 const MAX_SUGGESTED_INTEGRATIONS = 5 /** Common shape of the two hub listings. Both carry a description, but the hub has @@ -1311,9 +1312,9 @@ function tokenMatchesSlug(token: string, slug: string): boolean { } /** The slug of an integration the query names outright, when it names exactly one. - * Semantic search ranks a named vendor weakly against the task words — "create a jira - * ticket" puts netlify and zendesk above every Jira script — so narrowing to it wins. - * Matching must stay exact: fuzzily, `send` in "send an invoice" is sendgrid. */ + * Exact only: fuzzily, `send` in "send an invoice" reads as sendgrid. Even exact, a + * match is weak evidence of intent — `monday`, `box` and `linear` are ordinary words + * — so callers must treat it as a hint, never as a filter. */ async function integrationNamedIn(query: string): Promise { const list = await loadHubIntegrations() const words = new Set( @@ -1351,6 +1352,18 @@ const getHubIntegrationToolDef = createToolDef( * a specific one. */ const MAX_INTEGRATION_EXAMPLES = 5 +/** `validation` records how the authored notes were checked — the smoke-test method, + * per-surface confidence, the instance used. That is provenance for a human reviewing + * the hub, and up to a third of the document; the model can only act on its verdict. */ +function withoutProvenance(meta: unknown): unknown { + if (typeof meta !== 'object' || meta === null || Array.isArray(meta)) { + return meta + } + const { validation, ...rest } = meta as Record + const status = (validation as { status?: unknown } | undefined)?.status + return status === undefined ? rest : { ...rest, validation: { status } } +} + /** The hub keeps a resource type's schema in a text column and hands it back as a * JSON string, so parse it rather than passing an escaped blob to the model. * Anything already structured goes through untouched. */ @@ -1404,7 +1417,7 @@ export const getHubIntegrationTool = { ...(doc.docs_url ? { docs_url: doc.docs_url } : {}), // Authored provider knowledge and facts inferred from the scripts stay // separate: only the former was checked against the live API. - ...(doc.meta ? { verified_provider_notes: doc.meta } : {}), + ...(doc.meta ? { verified_provider_notes: withoutProvenance(doc.meta) } : {}), ...(derived ? { observed_from_scripts: { @@ -1459,12 +1472,21 @@ export const createSearchHubScriptsTool = (withContent: boolean = false) => ({ // is used survive instead of being cut. let scripts: HubScriptHit[] if (query) { - const narrowTo = app ?? (await integrationNamedIn(query)) - scripts = await ScriptService.queryHubScripts({ text: query, kind: 'script', app: narrowTo }) - // An integration the hub covers thinly can have nothing above the similarity - // floor, and a named vendor must not turn a search into no result at all. - if (scripts.length === 0 && narrowTo !== app) { - scripts = await ScriptService.queryHubScripts({ text: query, kind: 'script', app }) + scripts = await ScriptService.queryHubScripts({ text: query, kind: 'script', app }) + // Ranking can bury an integration the query names: "look up an account in + // salesforce" returns none of Salesforce's scripts, because Pinterest's say + // "Salesforce" too. Add its own hits rather than filtering to it, so a word + // that merely looks like a slug costs a few rows instead of the whole result. + if (!app) { + const mentioned = await integrationNamedIn(query) + if (mentioned && !scripts.some((s) => s.app === mentioned)) { + const own = await ScriptService.queryHubScripts({ + text: query, + kind: 'script', + app: mentioned + }) + scripts = [...scripts, ...own.slice(0, MAX_MENTIONED_INTEGRATION_HITS)] + } } } else { scripts =