From eea13810e448cf6b2a7bc3f97c01e0af03c01aad Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 7 Aug 2026 17:22:20 +0200 Subject: [PATCH] fix: resync web search guidance before the request is built Co-Authored-By: Claude Opus 5 (1M context) --- ai_evals/adapters/frontend/mockBackend.ts | 23 ++++--- .../copilot/chat/AIChatManager.svelte.ts | 23 +++---- .../lib/components/copilot/chat/app/core.ts | 2 +- .../components/copilot/chat/chatLoop.test.ts | 64 ++++++++++++++++++- .../lib/components/copilot/chat/chatLoop.ts | 16 +++-- .../src/lib/components/copilot/chat/shared.ts | 8 ++- 6 files changed, 102 insertions(+), 34 deletions(-) diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 1c331c5167..5015c43882 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -1040,20 +1040,23 @@ export async function main( * integration, or overlapping on three meaningful words. A looser bar answers * "send a Slack message" with the Discord fixture, handing an unrelated case a * plausible-looking wrong integration. */ -function searchBenchmarkHubScripts(text: string) { +function searchBenchmarkHubScripts(text: string, app: string | null) { const tokens = new Set( text .toLowerCase() .split(/[^a-z0-9]+/) .filter((token) => token.length > 2) ) - return BENCHMARK_HUB_SCRIPTS.map((script) => { - const words = new Set( - `${script.app} ${script.summary} ${script.terms}`.toLowerCase().split(/[^a-z0-9]+/) - ) - const score = [...tokens].filter((token) => words.has(token)).length - return { script, score, namesApp: tokens.has(script.app) } - }) + // The real endpoint filters by app before ranking, so honour it here too — + // otherwise a narrowed search silently returns other integrations' scripts. + return BENCHMARK_HUB_SCRIPTS.filter((script) => !app || script.app === app) + .map((script) => { + const words = new Set( + `${script.app} ${script.summary} ${script.terms}`.toLowerCase().split(/[^a-z0-9]+/) + ) + const score = [...tokens].filter((token) => words.has(token)).length + return { script, score, namesApp: tokens.has(script.app) } + }) .filter((entry) => entry.namesApp || entry.score >= 3) .sort((a, b) => b.score - a.score) .map(({ script }, index) => ({ @@ -1193,7 +1196,9 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon } if (path === '/api/embeddings/query_hub_scripts') { const text = new URLSearchParams(url.split('?')[1] ?? '').get('text') ?? '' - return Response.json(searchBenchmarkHubScripts(text)) + return Response.json( + searchBenchmarkHubScripts(text, new URLSearchParams(url.split('?')[1] ?? '').get('app')) + ) } if (path === '/api/scripts/hub/top') { const app = new URLSearchParams(url.split('?')[1] ?? '').get('app') diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 72096c95e4..d2ec4ba326 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -44,7 +44,7 @@ import { loadApiTools } from './api/apiTools' import { prepareScriptUserMessage } from './script/core' import { prepareNavigatorUserMessage } from './navigator/core' import { sendUserToast } from '$lib/toast' -import { workspaceAIClients, getNonStreamingCompletion, providerSupportsWebSearch } from '../lib' +import { workspaceAIClients, getNonStreamingCompletion } from '../lib' import { logFeatureUsage } from '$lib/utils/featureUsage' import { modelSupportsVision } from '../modelConfig' import { getModelContextWindow } from '../modelConfig' @@ -110,7 +110,6 @@ import { setUserCustomPrompts, isWebSearchEnabledForProvider } from '$lib/aiStore' -import type { AIProvider as AIProviderType } from '$lib/gen' import type { WorkspaceMutationTarget } from './workspaceTools' import { globalToolsFor, @@ -1909,12 +1908,12 @@ export class AIChatManager { /** Web-search availability the GLOBAL system message was last built against. */ private globalWebSearchAdvertised: boolean | undefined = undefined - private syncGlobalWebSearchGuidance = (provider: AIProviderType | undefined) => { - if (this.mode !== AIMode.GLOBAL) { - return - } - const available = providerSupportsWebSearch(provider) && isWebSearchEnabledForProvider(provider) - if (available === this.globalWebSearchAdvertised) { + /** Keep the GLOBAL prompt's web-search guidance matching what the loop will + * actually hand the model. `available` is the loop's effective value, so this + * covers a mid-conversation provider switch and the runtime rejection probe + * alike — neither of which the prompt could observe on its own. */ + private syncGlobalWebSearchGuidance = (available: boolean) => { + if (this.mode !== AIMode.GLOBAL || available === this.globalWebSearchAdvertised) { return } this.globalWebSearchAdvertised = available @@ -2350,13 +2349,9 @@ export class AIChatManager { } return undefined }, - onBeforeIteration: async (tools, _helpers, modelProvider) => { + onBeforeIteration: async (tools, _helpers, modelProvider, webSearch) => { this.lastIterationModel = modelProvider - // Web search is provider-hosted, so the prompt is the only thing that can - // advertise it — and the selector stays switchable mid-conversation. Rebuild - // when availability flips, or a switch onto a provider without web search - // leaves the prompt telling the model to use a tool it is no longer handed. - this.syncGlobalWebSearchGuidance(modelProvider.provider) + this.syncGlobalWebSearchGuidance(webSearch) for (const tool of tools) { if (tool.setSchema) { await tool.setSchema(this.helpers) diff --git a/frontend/src/lib/components/copilot/chat/app/core.ts b/frontend/src/lib/components/copilot/chat/app/core.ts index b4fae2649a..c9393fba0e 100644 --- a/frontend/src/lib/components/copilot/chat/app/core.ts +++ b/frontend/src/lib/components/copilot/chat/app/core.ts @@ -977,7 +977,7 @@ Use \`patch_file\` for small, localized edits when you can copy an exact snippet ### Discovery - \`search_workspace(query, type)\`: Search workspace scripts and flows - \`get_runnable_details(path, type)\`: Get details (summary, description, schema, content) of a specific script or flow -- \`search_hub_scripts(query)\`: Search hub scripts +- \`search_hub_scripts(query, integration)\`: Search hub scripts, or list one integration's scripts by slug ### Data Tables - \`list_datatables()\`: List configured datatables with schema and table names only. Does not include columns. Use this directly for table-list or available-tables summaries. diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts index 2c4eb466e9..c9c64c5686 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts @@ -55,19 +55,26 @@ function createConfig({ modelProvider = { provider: 'openai', model: 'gpt-4.1' }, callbacks = createCallbacks(), onWebSearchUnavailable, - onReasoningSummaryUnavailable + onReasoningSummaryUnavailable, + onBeforeIteration, + getSystemMessage = () => ({ role: 'system', content: '' }) }: { workspace: string modelProvider?: ReasoningProviderModel callbacks?: ChatLoopConfig['callbacks'] onWebSearchUnavailable?: ChatLoopConfig['onWebSearchUnavailable'] onReasoningSummaryUnavailable?: ChatLoopConfig['onReasoningSummaryUnavailable'] + onBeforeIteration?: ChatLoopConfig['onBeforeIteration'] + /** Read per iteration, mirroring the getter production passes. */ + getSystemMessage?: () => ChatLoopConfig['systemMessage'] }): ChatLoopConfig { const messages: ChatCompletionMessageParam[] = [{ role: 'user', content: 'search this' }] return { messages, - systemMessage: { role: 'system', content: '' }, + get systemMessage() { + return getSystemMessage() + }, tools: [], helpers: undefined, abortController: new AbortController(), @@ -80,7 +87,8 @@ function createConfig({ workspace, maxIterations: 1, onWebSearchUnavailable, - onReasoningSummaryUnavailable + onReasoningSummaryUnavailable, + onBeforeIteration } } @@ -298,6 +306,56 @@ describe('runChatLoop web search fallback', () => { }) }) +// A caller whose system prompt advertises web search can only keep it honest if it +// learns the effective value before the message is read — otherwise a provider +// switch or a cached rejection advertises a tool that same request never gets. +describe('runChatLoop onBeforeIteration web search sync', () => { + beforeEach(() => { + vi.resetAllMocks() + mocks.providerSupportsWebSearch.mockImplementation( + (provider) => provider === 'openai' || provider === 'anthropic' + ) + mocks.resolveRequestReasoning.mockReturnValue(undefined) + mocks.parseOpenAICompletion.mockResolvedValue({ shouldContinue: false, tokenUsage }) + mocks.parseOpenAIResponsesCompletion.mockResolvedValue({ shouldContinue: false, tokenUsage }) + mocks.getOpenAIResponsesCompletion.mockResolvedValue({}) + }) + + it('reports the effective web search value, false when the provider cannot serve it', async () => { + const seen: boolean[] = [] + const onBeforeIteration = vi.fn(async (_t, _h, _m, webSearch: boolean) => { + seen.push(webSearch) + }) + + await runChatLoop(createConfig({ workspace: `workspace-${randomUUID()}`, onBeforeIteration })) + await runChatLoop( + createConfig({ + workspace: `workspace-${randomUUID()}`, + modelProvider: { provider: 'googleai', model: 'gemini-3-pro' }, + onBeforeIteration + }) + ) + + expect(seen).toEqual([true, false]) + }) + + it('sends the system message the callback rewrote on this iteration, not the next', async () => { + let systemMessage: ChatLoopConfig['systemMessage'] = { role: 'system', content: 'stale' } + const config = createConfig({ + workspace: `workspace-${randomUUID()}`, + getSystemMessage: () => systemMessage, + onBeforeIteration: async () => { + systemMessage = { role: 'system', content: 'resynced' } + } + }) + + await runChatLoop(config) + + const sentMessages = mocks.getOpenAIResponsesCompletion.mock.calls[0][0] + expect(sentMessages[0]).toEqual({ role: 'system', content: 'resynced' }) + }) +}) + describe('runChatLoop reasoning summary fallback', () => { beforeEach(() => { vi.resetAllMocks() diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts index cc963e4b25..ab538f0031 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -71,11 +71,16 @@ export interface ChatLoopConfig { */ addedMessages?: ChatCompletionMessageParam[] /** Called before each iteration (e.g. to refresh tool schemas, or to record - * which model the iteration is about to use). */ + * which model the iteration is about to use). `webSearch` is the effective + * value for this iteration — static gates and the runtime probe both applied — + * so a caller whose system prompt advertises web search can resync here. The + * system message is read after this returns, so such a resync lands on this + * iteration rather than the next. */ onBeforeIteration?: ( tools: Tool[], helpers: any, - modelProvider: ReasoningProviderModel + modelProvider: ReasoningProviderModel, + webSearch: boolean ) => Promise } @@ -349,7 +354,6 @@ export async function runChatLoop(config: ChatLoopConfig): Promise ({ ).asks ?? []) if (scripts.length === 0) { - // A whiffed semantic search still leaves the integration browsable, which - // is what turns "no exact match" into a worked example to follow. - const suggested = query ? await suggestHubIntegrations(query) : [] + // A whiffed search still leaves the integration browsable, which is what + // turns "no exact match" into a worked example to follow. Suggest against + // the slug too, so a browse for a misremembered one lands on the real name + // instead of dead-ending on an empty list. + const suggested = await suggestHubIntegrations(query ?? app ?? '') toolCallbacks.setToolStatus(toolId, { content: `No hub script found for ${subject}` }) return JSON.stringify({ results: [], suggested_integrations: suggested }) }