diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 4be396c952..cf3ef18086 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -1290,7 +1290,8 @@ function benchmarkDerivedFacts(app: string) { script_counts: { total: scripts.length, by_kind: scripts.reduce>((acc, script) => { - acc[script.kind] = (acc[script.kind] ?? 0) + 1; + const kind = script.kind ?? "script"; + acc[kind] = (acc[kind] ?? 0) + 1; return acc; }, {}), }, @@ -1300,7 +1301,7 @@ function benchmarkDerivedFacts(app: string) { version_id: script.version_id, summary: script.summary, description: script.description ?? null, - kind: script.kind, + kind: script.kind ?? "script", language: script.language, views: 0, votes: 0, diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index dce274991c..4c201d2e44 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -1470,6 +1470,34 @@ describe('createSearchHubScriptsTool', () => { expect(JSON.parse(raw).results.map((r: any) => r.summary)).toEqual(['A', 'Best', 'Second']) }) + // A named integration ranked just below the cap gets no dedicated lookup, so the + // content trim is the only thing standing between it and being dropped entirely. + it('keeps a named integration that ranking placed below the cap', async () => { + const { ScriptService, IntegrationService } = await import('$lib/gen') + Object.assign(ScriptService, { + queryHubScripts: vi.fn(async () => [ + hit(1, 'netlify', 'Create ticket'), + hit(2, 'zendesk', 'Create Ticket'), + hit(3, 'intercom', 'Create a ticket'), + hit(4, 'jira', 'Create issue') + ]), + getHubScriptByPath: vi.fn(async () => ({ content: '// x', language: 'bun' })) + }) + Object.assign(IntegrationService, { + listHubIntegrations: vi.fn(async () => [{ name: 'jira' }, { name: 'netlify' }]) + }) + + const { createSearchHubScriptsTool, clearHubIntegrationsCache } = await import('./shared') + clearHubIntegrationsCache() + const raw = await createSearchHubScriptsTool(true).fn({ + args: { query: 'create a jira ticket' }, + toolId: 't1', + toolCallbacks: { setToolStatus: vi.fn() } + } as any) + + expect(JSON.parse(raw).results.map((r: any) => r.integration)).toContain('jira') + }) + // Ranking buries an integration the query names when other integrations' scripts // mention it: none of Salesforce's own scripts come back for "an account in // salesforce", because Pinterest's summaries say Salesforce too. diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 8f3a020de5..23532f5675 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -1480,6 +1480,7 @@ export const createSearchHubScriptsTool = (withContent: boolean = false) => ({ // Kept apart from the ranked hits rather than counted off the end, so capping // below can keep the best of each instead of whatever the tail happens to hold. let mentioned: HubScriptHit[] = [] + let namedSlug: string | undefined if (query) { ranked = await ScriptService.queryHubScripts({ text: query, kind: 'script', app }) // Ranking can bury an integration the query names: "look up an account in @@ -1487,12 +1488,12 @@ export const createSearchHubScriptsTool = (withContent: boolean = false) => ({ // "Salesforce" too. Add its own hits rather than filtering to it, so a word // that merely looks like a slug costs a few rows instead of the whole result. if (!app) { - const named = await integrationNamedIn(query) - if (named && !ranked.some((s) => s.app === named)) { + namedSlug = await integrationNamedIn(query) + if (namedSlug && !ranked.some((s) => s.app === namedSlug)) { const own = await ScriptService.queryHubScripts({ text: query, kind: 'script', - app: named + app: namedSlug }) mentioned = own.slice(0, MAX_MENTIONED_INTEGRATION_HITS) } @@ -1525,7 +1526,17 @@ export const createSearchHubScriptsTool = (withContent: boolean = false) => ({ let matches = scripts if (withContent && scripts.length > MAX_FETCHED_HUB_SCRIPTS) { const keep = mentioned.slice(0, MAX_FETCHED_HUB_SCRIPTS - 1) - matches = [...ranked.slice(0, MAX_FETCHED_HUB_SCRIPTS - keep.length), ...keep] + let head = ranked.slice(0, MAX_FETCHED_HUB_SCRIPTS - keep.length) + // Ranking can place the named integration just below the cap, in which case + // no hits were fetched for it and trimming would drop it altogether. + const buried = + !keep.length && namedSlug && !head.some((s) => s.app === namedSlug) + ? ranked.find((s) => s.app === namedSlug) + : undefined + if (buried) { + head = [...head.slice(0, head.length - 1), buried] + } + matches = [...head, ...keep] } toolCallbacks.setToolStatus(toolId, { content: `Found ${matches.length} hub script${matches.length === 1 ? '' : 's'} for ${subject}`