fix: resync web search guidance before the request is built

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

ai_evals/adapters/frontend/mockBackend.ts

@@ -1040,20 +1040,23 @@ export async function main(
  * integration, or overlapping on three meaningful words. A looser bar answers
  * "send a Slack message" with the Discord fixture, handing an unrelated case a
  * plausible-looking wrong integration. */
-function searchBenchmarkHubScripts(text: string) {
+function searchBenchmarkHubScripts(text: string, app: string | null) {
 	const tokens = new Set(
 		text
 			.toLowerCase()
 			.split(/[^a-z0-9]+/)
 			.filter((token) => token.length > 2)
 	)
-	return BENCHMARK_HUB_SCRIPTS.map((script) => {
-		const words = new Set(
-			`${script.app} ${script.summary} ${script.terms}`.toLowerCase().split(/[^a-z0-9]+/)
-		)
-		const score = [...tokens].filter((token) => words.has(token)).length
-		return { script, score, namesApp: tokens.has(script.app) }
-	})
+	// The real endpoint filters by app before ranking, so honour it here too —
+	// otherwise a narrowed search silently returns other integrations' scripts.
+	return BENCHMARK_HUB_SCRIPTS.filter((script) => !app || script.app === app)
+		.map((script) => {
+			const words = new Set(
+				`${script.app} ${script.summary} ${script.terms}`.toLowerCase().split(/[^a-z0-9]+/)
+			)
+			const score = [...tokens].filter((token) => words.has(token)).length
+			return { script, score, namesApp: tokens.has(script.app) }
+		})
 		.filter((entry) => entry.namesApp || entry.score >= 3)
 		.sort((a, b) => b.score - a.score)
 		.map(({ script }, index) => ({
@@ -1193,7 +1196,9 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon
 	}
 	if (path === '/api/embeddings/query_hub_scripts') {
 		const text = new URLSearchParams(url.split('?')[1] ?? '').get('text') ?? ''
-		return Response.json(searchBenchmarkHubScripts(text))
+		return Response.json(
+			searchBenchmarkHubScripts(text, new URLSearchParams(url.split('?')[1] ?? '').get('app'))
+		)
 	}
 	if (path === '/api/scripts/hub/top') {
 		const app = new URLSearchParams(url.split('?')[1] ?? '').get('app')

frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts

@@ -44,7 +44,7 @@ import { loadApiTools } from './api/apiTools'
 import { prepareScriptUserMessage } from './script/core'
 import { prepareNavigatorUserMessage } from './navigator/core'
 import { sendUserToast } from '$lib/toast'
-import { workspaceAIClients, getNonStreamingCompletion, providerSupportsWebSearch } from '../lib'
+import { workspaceAIClients, getNonStreamingCompletion } from '../lib'
 import { logFeatureUsage } from '$lib/utils/featureUsage'
 import { modelSupportsVision } from '../modelConfig'
 import { getModelContextWindow } from '../modelConfig'
@@ -110,7 +110,6 @@ import {
 	setUserCustomPrompts,
 	isWebSearchEnabledForProvider
 } from '$lib/aiStore'
-import type { AIProvider as AIProviderType } from '$lib/gen'
 import type { WorkspaceMutationTarget } from './workspaceTools'
 import {
 	globalToolsFor,
@@ -1909,12 +1908,12 @@ export class AIChatManager {
 	/** Web-search availability the GLOBAL system message was last built against. */
 	private globalWebSearchAdvertised: boolean | undefined = undefined
 
-	private syncGlobalWebSearchGuidance = (provider: AIProviderType | undefined) => {
-		if (this.mode !== AIMode.GLOBAL) {
-			return
-		}
-		const available = providerSupportsWebSearch(provider) && isWebSearchEnabledForProvider(provider)
-		if (available === this.globalWebSearchAdvertised) {
+	/** Keep the GLOBAL prompt's web-search guidance matching what the loop will
+	 * actually hand the model. `available` is the loop's effective value, so this
+	 * covers a mid-conversation provider switch and the runtime rejection probe
+	 * alike — neither of which the prompt could observe on its own. */
+	private syncGlobalWebSearchGuidance = (available: boolean) => {
+		if (this.mode !== AIMode.GLOBAL || available === this.globalWebSearchAdvertised) {
 			return
 		}
 		this.globalWebSearchAdvertised = available
@@ -2350,13 +2349,9 @@ export class AIChatManager {
 					}
 					return undefined
 				},
-				onBeforeIteration: async (tools, _helpers, modelProvider) => {
+				onBeforeIteration: async (tools, _helpers, modelProvider, webSearch) => {
 					this.lastIterationModel = modelProvider
-					// Web search is provider-hosted, so the prompt is the only thing that can
-					// advertise it — and the selector stays switchable mid-conversation. Rebuild
-					// when availability flips, or a switch onto a provider without web search
-					// leaves the prompt telling the model to use a tool it is no longer handed.
-					this.syncGlobalWebSearchGuidance(modelProvider.provider)
+					this.syncGlobalWebSearchGuidance(webSearch)
 					for (const tool of tools) {
 						if (tool.setSchema) {
 							await tool.setSchema(this.helpers)

frontend/src/lib/components/copilot/chat/app/core.ts

@@ -977,7 +977,7 @@ Use \`patch_file\` for small, localized edits when you can copy an exact snippet
 ### Discovery
 - \`search_workspace(query, type)\`: Search workspace scripts and flows
 - \`get_runnable_details(path, type)\`: Get details (summary, description, schema, content) of a specific script or flow
-- \`search_hub_scripts(query)\`: Search hub scripts
+- \`search_hub_scripts(query, integration)\`: Search hub scripts, or list one integration's scripts by slug
 
 ### Data Tables
 - \`list_datatables()\`: List configured datatables with schema and table names only. Does not include columns. Use this directly for table-list or available-tables summaries.

frontend/src/lib/components/copilot/chat/chatLoop.test.ts

@@ -55,19 +55,26 @@ function createConfig({
 	modelProvider = { provider: 'openai', model: 'gpt-4.1' },
 	callbacks = createCallbacks(),
 	onWebSearchUnavailable,
-	onReasoningSummaryUnavailable
+	onReasoningSummaryUnavailable,
+	onBeforeIteration,
+	getSystemMessage = () => ({ role: 'system', content: '' })
 }: {
 	workspace: string
 	modelProvider?: ReasoningProviderModel
 	callbacks?: ChatLoopConfig['callbacks']
 	onWebSearchUnavailable?: ChatLoopConfig['onWebSearchUnavailable']
 	onReasoningSummaryUnavailable?: ChatLoopConfig['onReasoningSummaryUnavailable']
+	onBeforeIteration?: ChatLoopConfig['onBeforeIteration']
+	/** Read per iteration, mirroring the getter production passes. */
+	getSystemMessage?: () => ChatLoopConfig['systemMessage']
 }): ChatLoopConfig {
 	const messages: ChatCompletionMessageParam[] = [{ role: 'user', content: 'search this' }]
 
 	return {
 		messages,
-		systemMessage: { role: 'system', content: '' },
+		get systemMessage() {
+			return getSystemMessage()
+		},
 		tools: [],
 		helpers: undefined,
 		abortController: new AbortController(),
@@ -80,7 +87,8 @@ function createConfig({
 		workspace,
 		maxIterations: 1,
 		onWebSearchUnavailable,
-		onReasoningSummaryUnavailable
+		onReasoningSummaryUnavailable,
+		onBeforeIteration
 	}
 }
 
@@ -298,6 +306,56 @@ describe('runChatLoop web search fallback', () => {
 	})
 })
 
+// A caller whose system prompt advertises web search can only keep it honest if it
+// learns the effective value before the message is read — otherwise a provider
+// switch or a cached rejection advertises a tool that same request never gets.
+describe('runChatLoop onBeforeIteration web search sync', () => {
+	beforeEach(() => {
+		vi.resetAllMocks()
+		mocks.providerSupportsWebSearch.mockImplementation(
+			(provider) => provider === 'openai' || provider === 'anthropic'
+		)
+		mocks.resolveRequestReasoning.mockReturnValue(undefined)
+		mocks.parseOpenAICompletion.mockResolvedValue({ shouldContinue: false, tokenUsage })
+		mocks.parseOpenAIResponsesCompletion.mockResolvedValue({ shouldContinue: false, tokenUsage })
+		mocks.getOpenAIResponsesCompletion.mockResolvedValue({})
+	})
+
+	it('reports the effective web search value, false when the provider cannot serve it', async () => {
+		const seen: boolean[] = []
+		const onBeforeIteration = vi.fn(async (_t, _h, _m, webSearch: boolean) => {
+			seen.push(webSearch)
+		})
+
+		await runChatLoop(createConfig({ workspace: `workspace-${randomUUID()}`, onBeforeIteration }))
+		await runChatLoop(
+			createConfig({
+				workspace: `workspace-${randomUUID()}`,
+				modelProvider: { provider: 'googleai', model: 'gemini-3-pro' },
+				onBeforeIteration
+			})
+		)
+
+		expect(seen).toEqual([true, false])
+	})
+
+	it('sends the system message the callback rewrote on this iteration, not the next', async () => {
+		let systemMessage: ChatLoopConfig['systemMessage'] = { role: 'system', content: 'stale' }
+		const config = createConfig({
+			workspace: `workspace-${randomUUID()}`,
+			getSystemMessage: () => systemMessage,
+			onBeforeIteration: async () => {
+				systemMessage = { role: 'system', content: 'resynced' }
+			}
+		})
+
+		await runChatLoop(config)
+
+		const sentMessages = mocks.getOpenAIResponsesCompletion.mock.calls[0][0]
+		expect(sentMessages[0]).toEqual({ role: 'system', content: 'resynced' })
+	})
+})
+
 describe('runChatLoop reasoning summary fallback', () => {
 	beforeEach(() => {
 		vi.resetAllMocks()

frontend/src/lib/components/copilot/chat/chatLoop.ts

@@ -71,11 +71,16 @@ export interface ChatLoopConfig {
 	 */
 	addedMessages?: ChatCompletionMessageParam[]
 	/** Called before each iteration (e.g. to refresh tool schemas, or to record
-	 * which model the iteration is about to use). */
+	 * which model the iteration is about to use). `webSearch` is the effective
+	 * value for this iteration — static gates and the runtime probe both applied —
+	 * so a caller whose system prompt advertises web search can resync here. The
+	 * system message is read after this returns, so such a resync lands on this
+	 * iteration rather than the next. */
 	onBeforeIteration?: (
 		tools: Tool<any>[],
 		helpers: any,
-		modelProvider: ReasoningProviderModel
+		modelProvider: ReasoningProviderModel,
+		webSearch: boolean
 	) => Promise<void>
 }
 
@@ -349,7 +354,6 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
 		// Callers can use JS getter properties to provide dynamic values.
 		const tools = config.tools
 		const helpers = config.helpers
-		const systemMessage = config.systemMessage
 		const modelProvider = config.modelProvider
 		const webSearchCacheKey = getWebSearchCacheKey(workspace, modelProvider)
 		const webSearch =
@@ -358,9 +362,13 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
 			!unsupportedWebSearchCache.has(webSearchCacheKey)
 
 		if (onBeforeIteration) {
-			await onBeforeIteration(tools, helpers, modelProvider)
+			await onBeforeIteration(tools, helpers, modelProvider, webSearch)
 		}
 
+		// Read after onBeforeIteration: a caller that resyncs its system prompt to
+		// `webSearch` there must have that land on this request, not the next one.
+		const systemMessage = config.systemMessage
+
 		const pendingUserMessage = getPendingUserMessage?.()
 
 		const isOpenAI =

frontend/src/lib/components/copilot/chat/shared.ts

@@ -1312,9 +1312,11 @@ export const createSearchHubScriptsTool = (withContent: boolean = false) => ({
 				).asks ?? [])
 
 		if (scripts.length === 0) {
-			// A whiffed semantic search still leaves the integration browsable, which
-			// is what turns "no exact match" into a worked example to follow.
-			const suggested = query ? await suggestHubIntegrations(query) : []
+			// A whiffed search still leaves the integration browsable, which is what
+			// turns "no exact match" into a worked example to follow. Suggest against
+			// the slug too, so a browse for a misremembered one lands on the real name
+			// instead of dead-ending on an empty list.
+			const suggested = await suggestHubIntegrations(query ?? app ?? '')
 			toolCallbacks.setToolStatus(toolId, { content: `No hub script found for ${subject}` })
 			return JSON.stringify({ results: [], suggested_integrations: suggested })
 		}