diff --git a/backend/windmill-ai/src/ai_providers.rs b/backend/windmill-ai/src/ai_providers.rs index 52d04910de..5d8e763b69 100644 --- a/backend/windmill-ai/src/ai_providers.rs +++ b/backend/windmill-ai/src/ai_providers.rs @@ -177,6 +177,8 @@ impl TryFrom<&str> for AIProvider { pub struct ProviderConfig { pub resource_path: String, pub models: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub web_search_enabled: Option, } #[derive(Serialize, Deserialize, Debug)] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 6bc58412ea..5e8bd7eaec 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -21635,6 +21635,8 @@ components: type: array items: type: string + web_search_enabled: + type: boolean required: - resource_path - models diff --git a/frontend/src/lib/aiStore.test.ts b/frontend/src/lib/aiStore.test.ts index 1317c3a2d9..fed6147b62 100644 --- a/frontend/src/lib/aiStore.test.ts +++ b/frontend/src/lib/aiStore.test.ts @@ -10,7 +10,7 @@ vi.mock('./gen', () => ({ WorkspaceService: { getCopilotInfo: async () => ({}) } })) -import { setCopilotInfo, copilotInfo } from './aiStore' +import { isWebSearchEnabledForProvider, setCopilotInfo, copilotInfo } from './aiStore' describe('setCopilotInfo legacy /thinking migration', () => { it('strips the /thinking suffix from configured model slots', () => { @@ -34,4 +34,23 @@ describe('setCopilotInfo legacy /thinking migration', () => { // the model list is stripped and deduped expect(info.aiModels.map((m) => m.model)).toEqual(['claude-sonnet-4-6']) }) + + it('defaults provider web search on unless explicitly disabled', () => { + setCopilotInfo({ + providers: { + openai: { + resource_path: 'u/admin/openai', + models: ['gpt-4.1'], + web_search_enabled: false + }, + anthropic: { + resource_path: 'u/admin/anthropic', + models: ['claude-sonnet-4-6'] + } + } + }) + + expect(isWebSearchEnabledForProvider('openai')).toBe(false) + expect(isWebSearchEnabledForProvider('anthropic')).toBe(true) + }) }) diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 5178726056..02fcdce07a 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -37,6 +37,7 @@ export const copilotInfo = writable<{ aiModels: AIProviderModel[] customPrompts?: Record maxTokensPerModel?: Record + webSearchEnabledProviders?: Partial> }>({ enabled: false, codeCompletionModel: undefined, @@ -44,7 +45,8 @@ export const copilotInfo = writable<{ metadataModel: undefined, aiModels: [], customPrompts: {}, - maxTokensPerModel: {} + maxTokensPerModel: {}, + webSearchEnabledProviders: {} }) /** Strip the deprecated /thinking suffix from a configured model slot, if present. */ @@ -87,6 +89,12 @@ export function setCopilotInfo(aiConfig: AIConfig) { })) ) ) + const webSearchEnabledProviders = Object.fromEntries( + Object.entries(aiConfig.providers ?? {}).map(([provider, providerConfig]) => [ + provider, + providerConfig.web_search_enabled !== false + ]) + ) as Partial> copilotSessionModel.update((model) => { if ( @@ -107,7 +115,8 @@ export function setCopilotInfo(aiConfig: AIConfig) { metadataModel: stripModelSuffix(aiConfig.metadata_model), aiModels: aiModels, customPrompts: aiConfig.custom_prompts ?? {}, - maxTokensPerModel: aiConfig.max_tokens_per_model ?? {} + maxTokensPerModel: aiConfig.max_tokens_per_model ?? {}, + webSearchEnabledProviders }) } else { copilotSessionModel.set(undefined) @@ -119,11 +128,19 @@ export function setCopilotInfo(aiConfig: AIConfig) { metadataModel: undefined, aiModels: [], customPrompts: {}, - maxTokensPerModel: {} + maxTokensPerModel: {}, + webSearchEnabledProviders: {} }) } } +export function isWebSearchEnabledForProvider(provider: AIProvider | undefined): boolean { + if (!provider) { + return false + } + return get(copilotInfo).webSearchEnabledProviders?.[provider] ?? true +} + export function getCurrentModel(): ReasoningProviderModel { const model = get(copilotSessionModel) ?? get(copilotInfo).defaultModel ?? get(copilotInfo).aiModels[0] diff --git a/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte b/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte index 9df993e4f3..5f2039e353 100644 --- a/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte +++ b/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte @@ -26,6 +26,11 @@ would be surprising. type Kind = WorkspaceItemKind type ScopeKind = Kind | 'all' + type DrillPickerHandle = { + focus: () => void + handleKeydown: (e: KeyboardEvent) => void + pickHighlighted: () => void + } export type Scope = { kind: ScopeKind; dir?: string } | undefined @@ -51,7 +56,7 @@ would be surprising. flush = false }: Props = $props() - let inner = $state | undefined>(undefined) + let inner = $state(undefined) export function focus() { inner?.focus() diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index ca74c6a956..bc49925688 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -61,7 +61,12 @@ import type AIChatInput from './AIChatInput.svelte' import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core' import { runChatLoop } from './chatLoop' import type { ReviewChangesOpts } from './monaco-adapter' -import { getCurrentModel, tryGetCurrentModel, getCombinedCustomPrompt } from '$lib/aiStore' +import { + getCurrentModel, + tryGetCurrentModel, + getCombinedCustomPrompt, + isWebSearchEnabledForProvider +} from '$lib/aiStore' import type { WorkspaceMutationTarget } from './workspaceTools' import { globalToolsFor, @@ -76,6 +81,8 @@ const MAX_TOKENS_THRESHOLD_PERCENTAGE = 0.05 const MAX_TOKENS_HARD_LIMIT = 5000 const AI_AUTONOMY_MODE_STORAGE_KEY = 'ai-chat-autonomy-mode' const LEGACY_AUTO_ACCEPT_TOOL_CONFIRMATIONS_STORAGE_KEY = 'ai-chat-yolo-mode' +const WEB_SEARCH_ERROR_HINT = + 'Web search is unavailable for this provider/model/key. Disable web search in workspace settings and try again.' export enum AIMode { SCRIPT = 'script', @@ -154,6 +161,20 @@ function persistAutonomyMode(mode: AIAutonomyMode) { localStorage.setItem(AI_AUTONOMY_MODE_STORAGE_KEY, mode) } +function appendWebSearchErrorHint(message: string, shouldAppend: boolean): string { + if (!shouldAppend) { + return message + } + const separator = /[.!?]$/.test(message.trim()) ? ' ' : '. ' + return `${message}${separator}${WEB_SEARCH_ERROR_HINT}` +} + +function getSendRequestErrorMessage(err: unknown, webSearchUnavailable: boolean): string { + const errorMessage = err instanceof Error ? err.message : typeof err === 'string' ? err : undefined + const message = errorMessage ? `Failed to send request: ${errorMessage}` : 'Failed to send request' + return appendWebSearchErrorHint(message, webSearchUnavailable) +} + export class AIChatManager { contextManager = new ContextManager() historyManager = new HistoryManager() @@ -652,7 +673,8 @@ export class AIChatManager { messages, abortController, callbacks, - systemMessage: systemMessageOverride + systemMessage: systemMessageOverride, + onWebSearchUnavailable }: { messages: ChatCompletionMessageParam[] abortController: AbortController @@ -661,6 +683,7 @@ export class AIChatManager { onMessageEnd: () => void } systemMessage?: ChatCompletionSystemMessageParam + onWebSearchUnavailable?: () => void }) => { try { // Use JS getters so runChatLoop re-reads tools/helpers/systemMessage/modelProvider @@ -683,6 +706,9 @@ export class AIChatManager { get modelProvider() { return getCurrentModel() }, + get webSearch() { + return isWebSearchEnabledForProvider(getCurrentModel().provider) + }, clients: { openai: workspaceAIClients.getOpenaiClient(), anthropic: workspaceAIClients.getAnthropicClient() @@ -692,6 +718,7 @@ export class AIChatManager { onSkipResponsesApi: () => { this.skipResponsesApi = true }, + onWebSearchUnavailable, getPendingUserMessage: () => { const pendingPrompt = this.pendingPrompt if (!pendingPrompt) return undefined @@ -873,6 +900,7 @@ export class AIChatManager { } } const isFirstUserTurn = !this.displayMessages.some((message) => message.role === 'user') + let webSearchUnavailable = false try { const oldSelectedContext = this.contextManager?.getSelectedContext() ?? [] if (this.mode === AIMode.SCRIPT || this.mode === AIMode.FLOW) { @@ -1063,7 +1091,10 @@ export class AIChatManager { } const addedMessages = await this.chatRequest({ - ...params + ...params, + onWebSearchUnavailable: () => { + webSearchUnavailable = true + } }) this.messages = [...this.messages, ...(addedMessages ?? [])] if (this.autoAcceptEditsActive) { @@ -1078,11 +1109,7 @@ export class AIChatManager { } catch (err) { console.error(err) this.flagLastMessageAsError() - if (err instanceof Error) { - sendUserToast('Failed to send request: ' + err.message, true) - } else { - sendUserToast('Failed to send request', true) - } + sendUserToast(getSendRequestErrorMessage(err, webSearchUnavailable), true) } finally { this.loading = false } diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index b1c77df84d..5d74196a2e 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -4,12 +4,25 @@ import type { CurrentEditor } from '$lib/components/flows/types' import type { ReviewChangesOpts } from './monaco-adapter' import { AIChatManager, AIMode, AIAutonomyMode } from './AIChatManager.svelte' +const mocks = vi.hoisted(() => ({ + getCurrentModel: vi.fn(), + tryGetCurrentModel: vi.fn(), + isWebSearchEnabledForProvider: vi.fn(), + logAiChat: vi.fn(), + sendUserToast: vi.fn(), + getOpenaiClient: vi.fn(), + getAnthropicClient: vi.fn(), + runChatLoop: vi.fn() +})) + vi.mock('monaco-editor', () => ({ Selection: class Selection {} })) vi.mock('$lib/gen', () => ({ - WorkspaceService: {}, + WorkspaceService: { + logAiChat: mocks.logAiChat + }, ScriptService: {}, FlowService: {}, JobService: {} @@ -26,18 +39,23 @@ vi.mock('$lib/stores', () => ({ })) vi.mock('$lib/toast', () => ({ - sendUserToast: vi.fn() + sendUserToast: mocks.sendUserToast })) vi.mock('$lib/aiStore', () => ({ - getCurrentModel: () => undefined, - tryGetCurrentModel: () => undefined, - getCombinedCustomPrompt: () => '' + getCurrentModel: mocks.getCurrentModel, + tryGetCurrentModel: mocks.tryGetCurrentModel, + getCombinedCustomPrompt: () => '', + isWebSearchEnabledForProvider: mocks.isWebSearchEnabledForProvider })) vi.mock('../lib', () => ({ getModelContextWindow: () => 128000, - workspaceAIClients: { subscribe: () => () => undefined } + workspaceAIClients: { + subscribe: () => () => undefined, + getOpenaiClient: mocks.getOpenaiClient, + getAnthropicClient: mocks.getAnthropicClient + } })) vi.mock('./api/apiTools', () => ({ @@ -45,7 +63,7 @@ vi.mock('./api/apiTools', () => ({ })) vi.mock('./chatLoop', () => ({ - runChatLoop: vi.fn() + runChatLoop: mocks.runChatLoop })) vi.mock('./global/gate', () => ({ @@ -59,6 +77,21 @@ vi.mock('esm-env', async (importOriginal) => ({ BROWSER: true })) +beforeEach(() => { + vi.clearAllMocks() + mocks.getCurrentModel.mockReturnValue(undefined) + mocks.tryGetCurrentModel.mockReturnValue(undefined) + mocks.isWebSearchEnabledForProvider.mockReturnValue(true) + mocks.logAiChat.mockResolvedValue(undefined) + mocks.getOpenaiClient.mockReturnValue({}) + mocks.getAnthropicClient.mockReturnValue({}) + mocks.runChatLoop.mockResolvedValue({ + addedMessages: [], + tokenUsage: { prompt: 0, completion: 0, total: 0 }, + hitMaxIterations: false + }) +}) + function createFlowHelpers({ hasPendingChanges, acceptAllModuleActions, @@ -87,6 +120,61 @@ function createFlowHelpers({ } as unknown as FlowAIChatHelpers } +describe('AIChatManager request errors', () => { + const openaiModel = { provider: 'openai', model: 'gpt-4o' } + + beforeEach(() => { + localStorage.clear() + mocks.getCurrentModel.mockReturnValue(openaiModel) + mocks.tryGetCurrentModel.mockReturnValue(openaiModel) + }) + + it('does not add a web-search hint to generic request errors', async () => { + const manager = new AIChatManager() + manager.instructions = 'Search for recent docs' + mocks.isWebSearchEnabledForProvider.mockReturnValue(true) + mocks.runChatLoop.mockRejectedValueOnce(new Error('provider quota exceeded')) + + await manager.sendRequest() + + expect(mocks.sendUserToast).toHaveBeenLastCalledWith( + 'Failed to send request: provider quota exceeded', + true + ) + }) + + it('adds the web-search hint when fallback happened and the request still fails', async () => { + const manager = new AIChatManager() + manager.instructions = 'Search for recent docs' + mocks.isWebSearchEnabledForProvider.mockReturnValue(true) + mocks.runChatLoop.mockImplementationOnce(async (config) => { + config.onWebSearchUnavailable?.() + throw new Error('provider quota exceeded') + }) + + await manager.sendRequest() + + expect(mocks.sendUserToast).toHaveBeenLastCalledWith( + 'Failed to send request: provider quota exceeded. Web search is unavailable for this provider/model/key. Disable web search in workspace settings and try again.', + true + ) + }) + + it('does not add the web search settings hint when web search is disabled', async () => { + const manager = new AIChatManager() + manager.instructions = 'Search for recent docs' + mocks.isWebSearchEnabledForProvider.mockReturnValue(false) + mocks.runChatLoop.mockRejectedValueOnce(new Error('provider quota exceeded')) + + await manager.sendRequest() + + expect(mocks.sendUserToast).toHaveBeenLastCalledWith( + 'Failed to send request: provider quota exceeded', + true + ) + }) +}) + describe('AIChatManager autonomy mode', () => { beforeEach(() => { localStorage.clear() diff --git a/frontend/src/lib/components/copilot/chat/ChatContextPicker.svelte b/frontend/src/lib/components/copilot/chat/ChatContextPicker.svelte index def6ea737b..c083f9eab8 100644 --- a/frontend/src/lib/components/copilot/chat/ChatContextPicker.svelte +++ b/frontend/src/lib/components/copilot/chat/ChatContextPicker.svelte @@ -58,8 +58,11 @@ at message-prep time by `AIChatManager` — see PR #9216. // Chat tree leaves carry either a workspace path (resolved to content // at pick time) or a runtime ContextElement (added directly). type ChatLeafData = WorkspaceItem | ContextElement + type DrillPickerHandle = { + handleKeydown: (e: KeyboardEvent) => void + } - let inner = $state | undefined>(undefined) + let inner = $state(undefined) export function handleKeydown(e: KeyboardEvent) { inner?.handleKeydown(e) diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index 2505cf2398..753c0b04d1 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -24,6 +24,29 @@ interface ParsedCompletionResult { tokenUsage: ChatTokenUsage } +type WebSearchStatus = 'searching' | 'completed' | 'failed' + +function setAnthropicWebSearchStatus( + callbacks: ToolCallbacks & { onMessageEnd: () => void }, + toolId: string, + status: WebSearchStatus, + errorCode?: string +) { + const isLoading = status === 'searching' + const failed = status === 'failed' + callbacks.onMessageEnd() + callbacks.setToolStatus(`anthropic_web_search:${toolId}`, { + content: failed ? 'Web search failed' : isLoading ? 'Searching the web...' : 'Searched the web', + error: failed ? `Web search failed${errorCode ? `: ${errorCode}` : ''}` : undefined, + isLoading, + isStreamingArguments: false, + needsConfirmation: false, + toolName: 'web_search', + showDetails: false, + autoCollapseDetails: true + }) +} + export async function getAnthropicCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, @@ -31,6 +54,7 @@ export async function getAnthropicCompletion( options?: { forceModelProvider?: AIProviderModel anthropicClient?: Anthropic + webSearch?: boolean reasoningEffort?: string } ): Promise { @@ -40,7 +64,17 @@ export async function getAnthropicCompletion( forceModelProvider: options?.forceModelProvider }) const { system, messages: anthropicMessages } = convertOpenAIToAnthropicMessages(messages) - const anthropicTools = convertOpenAIToolsToAnthropic(tools) + let anthropicTools = convertOpenAIToolsToAnthropic(tools) + + // Enable Anthropic's server-side web search tool. The proxy forwards the body + // verbatim, so this reaches Anthropic as a native server tool that it executes + // itself (no client round-trip). + if (options?.webSearch) { + anthropicTools = [ + ...(anthropicTools ?? []), + { type: 'web_search_20250305', name: 'web_search', max_uses: 5 } + ] + } const client = options?.anthropicClient ?? workspaceAIClients.getAnthropicClient() @@ -117,6 +151,8 @@ export async function parseAnthropicCompletion( showDetails: tool?.showDetails, autoCollapseDetails: tool?.autoCollapseDetails }) + } else if (block.type === 'server_tool_use' && block.name === 'web_search') { + setAnthropicWebSearchStatus(callbacks, block.id, 'searching') } } }) @@ -173,6 +209,14 @@ export async function parseAnthropicCompletion( messages.push(assistantMessage) addedMessages.push(assistantMessage) callbacks.onMessageEnd() + } else if (block.type === 'web_search_tool_result') { + const errorCode = Array.isArray(block.content) ? undefined : block.content.error_code + setAnthropicWebSearchStatus( + callbacks, + block.tool_use_id, + errorCode ? 'failed' : 'completed', + errorCode + ) } else if (block.type === 'tool_use') { // Convert Anthropic tool calls to OpenAI format for compatibility toolCallsToProcess.push({ diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts new file mode 100644 index 0000000000..af619b93bf --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts @@ -0,0 +1,294 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { randomUUID } from '$lib/utils/uuid' +import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs' +import { runChatLoop, type ChatLoopConfig } from './chatLoop' +import type { ReasoningProviderModel } from '../reasoningRegistry' + +const mocks = vi.hoisted(() => ({ + getCompletion: vi.fn(), + parseOpenAICompletion: vi.fn(), + providerSupportsWebSearch: vi.fn(), + getOpenAIResponsesCompletion: vi.fn(), + parseOpenAIResponsesCompletion: vi.fn(), + getAnthropicCompletion: vi.fn(), + parseAnthropicCompletion: vi.fn(), + resolveEffectiveReasoning: vi.fn() +})) + +vi.mock('../lib', () => ({ + getCompletion: mocks.getCompletion, + parseOpenAICompletion: mocks.parseOpenAICompletion, + providerSupportsWebSearch: mocks.providerSupportsWebSearch +})) + +vi.mock('../reasoningRegistry', () => ({ + resolveEffectiveReasoning: mocks.resolveEffectiveReasoning +})) + +vi.mock('./openai-responses', () => ({ + getOpenAIResponsesCompletion: mocks.getOpenAIResponsesCompletion, + parseOpenAIResponsesCompletion: mocks.parseOpenAIResponsesCompletion +})) + +vi.mock('./anthropic', () => ({ + getAnthropicCompletion: mocks.getAnthropicCompletion, + parseAnthropicCompletion: mocks.parseAnthropicCompletion +})) + +const tokenUsage = { prompt: 0, completion: 0, total: 0 } + +function createCallbacks(): ChatLoopConfig['callbacks'] { + return { + onNewToken: vi.fn(), + onMessageEnd: vi.fn(), + setToolStatus: vi.fn(), + removeToolStatus: vi.fn() + } +} + +function createConfig({ + workspace, + modelProvider = { provider: 'openai', model: 'gpt-4.1' }, + callbacks = createCallbacks(), + onWebSearchUnavailable +}: { + workspace: string + modelProvider?: ReasoningProviderModel + callbacks?: ChatLoopConfig['callbacks'] + onWebSearchUnavailable?: ChatLoopConfig['onWebSearchUnavailable'] +}): ChatLoopConfig { + const messages: ChatCompletionMessageParam[] = [{ role: 'user', content: 'search this' }] + + return { + messages, + systemMessage: { role: 'system', content: '' }, + tools: [], + helpers: undefined, + abortController: new AbortController(), + callbacks, + modelProvider, + clients: { + openai: {} as ChatLoopConfig['clients']['openai'], + anthropic: {} as ChatLoopConfig['clients']['anthropic'] + }, + workspace, + maxIterations: 1, + onWebSearchUnavailable + } +} + +describe('runChatLoop web search fallback', () => { + beforeEach(() => { + vi.resetAllMocks() + mocks.providerSupportsWebSearch.mockImplementation( + (provider) => provider === 'openai' || provider === 'anthropic' + ) + mocks.resolveEffectiveReasoning.mockReturnValue(undefined) + mocks.parseOpenAICompletion.mockResolvedValue({ + shouldContinue: false, + tokenUsage + }) + mocks.parseOpenAIResponsesCompletion.mockResolvedValue({ + shouldContinue: false, + tokenUsage + }) + mocks.parseAnthropicCompletion.mockResolvedValue({ + shouldContinue: false, + tokenUsage + }) + }) + + it('retries once without OpenAI web search and caches unsupported provider models', async () => { + const callbacks = createCallbacks() + const onWebSearchUnavailable = vi.fn() + const workspace = `workspace-${randomUUID()}` + + mocks.getOpenAIResponsesCompletion + .mockRejectedValueOnce( + Object.assign(new Error("Hosted tool 'web_search' is not supported with this model"), { + status: 400, + error: { type: 'invalid_request_error' } + }) + ) + .mockResolvedValue({}) + + await runChatLoop(createConfig({ workspace, callbacks, onWebSearchUnavailable })) + + expect(mocks.getOpenAIResponsesCompletion).toHaveBeenCalledTimes(2) + expect(mocks.getOpenAIResponsesCompletion.mock.calls[0][3]).toEqual( + expect.objectContaining({ webSearch: true }) + ) + expect(mocks.getOpenAIResponsesCompletion.mock.calls[1][3]).toEqual( + expect.objectContaining({ webSearch: false }) + ) + expect(onWebSearchUnavailable).toHaveBeenCalledTimes(1) + expect(callbacks.setToolStatus).not.toHaveBeenCalled() + + await runChatLoop(createConfig({ workspace })) + + expect(mocks.getOpenAIResponsesCompletion).toHaveBeenCalledTimes(3) + expect(mocks.getOpenAIResponsesCompletion.mock.calls[2][3]).toEqual( + expect.objectContaining({ webSearch: false }) + ) + }) + + it('does not treat unrelated OpenAI tool errors as web search failures', async () => { + const callbacks = createCallbacks() + const workspace = `workspace-${randomUUID()}` + + mocks.getOpenAIResponsesCompletion.mockRejectedValueOnce( + new Error('Unknown tool call: lookup_customer') + ) + mocks.getCompletion.mockResolvedValue({}) + + await runChatLoop(createConfig({ workspace, callbacks })) + + expect(mocks.getOpenAIResponsesCompletion).toHaveBeenCalledTimes(1) + expect(mocks.getOpenAIResponsesCompletion.mock.calls[0][3]).toEqual( + expect.objectContaining({ webSearch: true }) + ) + expect(mocks.getCompletion).toHaveBeenCalledTimes(1) + expect(callbacks.setToolStatus).not.toHaveBeenCalled() + + await runChatLoop(createConfig({ workspace })) + + expect(mocks.getOpenAIResponsesCompletion).toHaveBeenCalledTimes(2) + expect(mocks.getOpenAIResponsesCompletion.mock.calls[1][3]).toEqual( + expect.objectContaining({ webSearch: true }) + ) + }) + + it('treats OpenAI hosted-tool incompatibility as web search unavailable', async () => { + const callbacks = createCallbacks() + const onWebSearchUnavailable = vi.fn() + const workspace = `workspace-${randomUUID()}` + + mocks.getOpenAIResponsesCompletion + .mockRejectedValueOnce( + Object.assign(new Error('Hosted tools are not supported with this model'), { + status: 400, + error: { type: 'invalid_request_error' } + }) + ) + .mockResolvedValue({}) + + await runChatLoop(createConfig({ workspace, callbacks, onWebSearchUnavailable })) + + expect(mocks.getOpenAIResponsesCompletion).toHaveBeenCalledTimes(2) + expect(mocks.getOpenAIResponsesCompletion.mock.calls[0][3]).toEqual( + expect.objectContaining({ webSearch: true }) + ) + expect(mocks.getOpenAIResponsesCompletion.mock.calls[1][3]).toEqual( + expect.objectContaining({ webSearch: false }) + ) + expect(onWebSearchUnavailable).toHaveBeenCalledTimes(1) + expect(callbacks.setToolStatus).not.toHaveBeenCalled() + }) + + it('does not cache malformed web search requests as unavailable', async () => { + const callbacks = createCallbacks() + const workspace = `workspace-${randomUUID()}` + + mocks.getOpenAIResponsesCompletion.mockRejectedValueOnce( + Object.assign(new Error('Invalid value for web_search.search_context_size'), { + status: 400, + error: { type: 'invalid_request_error' } + }) + ) + mocks.getCompletion.mockResolvedValue({}) + + await runChatLoop(createConfig({ workspace, callbacks })) + + expect(mocks.getOpenAIResponsesCompletion).toHaveBeenCalledTimes(1) + expect(mocks.getCompletion).toHaveBeenCalledTimes(1) + expect(callbacks.setToolStatus).not.toHaveBeenCalled() + + await runChatLoop(createConfig({ workspace })) + + expect(mocks.getOpenAIResponsesCompletion).toHaveBeenCalledTimes(2) + expect(mocks.getOpenAIResponsesCompletion.mock.calls[1][3]).toEqual( + expect.objectContaining({ webSearch: true }) + ) + }) + + it('does not cache web-search rate limits as unavailable', async () => { + const callbacks = createCallbacks() + const workspace = `workspace-${randomUUID()}` + + mocks.getOpenAIResponsesCompletion.mockRejectedValueOnce( + Object.assign(new Error('Rate limit exceeded for web_search'), { + status: 429, + error: { type: 'rate_limit_error' } + }) + ) + mocks.getCompletion.mockResolvedValue({}) + + await runChatLoop(createConfig({ workspace, callbacks })) + + expect(mocks.getOpenAIResponsesCompletion).toHaveBeenCalledTimes(1) + expect(mocks.getCompletion).toHaveBeenCalledTimes(1) + expect(callbacks.setToolStatus).not.toHaveBeenCalled() + + await runChatLoop(createConfig({ workspace })) + + expect(mocks.getOpenAIResponsesCompletion).toHaveBeenCalledTimes(2) + expect(mocks.getOpenAIResponsesCompletion.mock.calls[1][3]).toEqual( + expect.objectContaining({ webSearch: true }) + ) + }) + + it('retries once without Anthropic web search when the model rejects it', async () => { + const callbacks = createCallbacks() + const onWebSearchUnavailable = vi.fn() + const workspace = `workspace-${randomUUID()}` + const modelProvider: ReasoningProviderModel = { + provider: 'anthropic', + model: 'claude-3-5-sonnet-latest' + } + + mocks.getAnthropicCompletion + .mockRejectedValueOnce( + Object.assign(new Error('Your organization must enable web search in the Claude Console'), { + status: 400, + error: { type: 'invalid_request_error' } + }) + ) + .mockResolvedValue({}) + + await runChatLoop( + createConfig({ workspace, callbacks, modelProvider, onWebSearchUnavailable }) + ) + + expect(mocks.getAnthropicCompletion).toHaveBeenCalledTimes(2) + expect(mocks.getAnthropicCompletion.mock.calls[0][3]).toEqual( + expect.objectContaining({ webSearch: true }) + ) + expect(mocks.getAnthropicCompletion.mock.calls[1][3]).toEqual( + expect.objectContaining({ webSearch: false }) + ) + expect(onWebSearchUnavailable).toHaveBeenCalledTimes(1) + expect(callbacks.setToolStatus).not.toHaveBeenCalled() + }) + + it('does not retry Anthropic request errors that only mention web search input', async () => { + const callbacks = createCallbacks() + const workspace = `workspace-${randomUUID()}` + const modelProvider: ReasoningProviderModel = { + provider: 'anthropic', + model: 'claude-sonnet-4-6' + } + const error = Object.assign(new Error('Invalid web_search query parameter'), { + status: 400, + error: { type: 'invalid_request_error' } + }) + + mocks.getAnthropicCompletion.mockRejectedValueOnce(error) + + await expect(runChatLoop(createConfig({ workspace, callbacks, modelProvider }))).rejects.toBe( + error + ) + + expect(mocks.getAnthropicCompletion).toHaveBeenCalledTimes(1) + expect(callbacks.setToolStatus).not.toHaveBeenCalled() + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts index a87f9cf333..46b8c881c0 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -5,7 +5,7 @@ import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/chat/completions.mjs' -import { getCompletion, parseOpenAICompletion } from '../lib' +import { getCompletion, parseOpenAICompletion, providerSupportsWebSearch } from '../lib' import { resolveEffectiveReasoning, type ReasoningProviderModel } from '../reasoningRegistry' import { getAnthropicCompletion, parseAnthropicCompletion } from './anthropic' import { getOpenAIResponsesCompletion, parseOpenAIResponsesCompletion } from './openai-responses' @@ -35,10 +35,17 @@ export interface ChatLoopConfig { modelProvider: ReasoningProviderModel clients: ChatClients workspace: string + /** + * Enable provider-native web search. Defaults to true for compatible providers + * and is re-read each iteration so model/provider changes take effect. Explicit + * true is still ignored for providers without native web search support. + */ + webSearch?: boolean /** Maximum iterations for the loop. undefined = unlimited (production). */ maxIterations?: number skipResponsesApi?: boolean onSkipResponsesApi?: () => void + onWebSearchUnavailable?: () => void /** Return a pending user message to inject between iterations, or undefined. */ getPendingUserMessage?: () => ChatCompletionUserMessageParam | undefined /** Called before each iteration (e.g. to refresh tool schemas). */ @@ -51,6 +58,101 @@ export interface ChatLoopResult { hitMaxIterations: boolean } +const unsupportedWebSearchCache = new Set() +const WEB_SEARCH_UNAVAILABLE_STATUS_CODES = new Set([400, 403, 404]) + +function getWebSearchCacheKey(workspace: string, modelProvider: ReasoningProviderModel): string { + return [workspace, modelProvider.provider, modelProvider.model].join(':') +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null +} + +function appendTextPart(parts: string[], value: unknown) { + if (typeof value === 'string' && value.trim()) { + parts.push(value) + } +} + +function getErrorText(err: unknown): string { + const parts: string[] = [] + if (err instanceof Error) { + appendTextPart(parts, err.message) + } + if (typeof err === 'string') { + appendTextPart(parts, err) + } + if (isRecord(err)) { + appendTextPart(parts, err.message) + appendTextPart(parts, err.type) + appendTextPart(parts, err.code) + appendTextPart(parts, err.param) + + const nested = err.error + if (isRecord(nested)) { + appendTextPart(parts, nested.message) + appendTextPart(parts, nested.type) + appendTextPart(parts, nested.code) + appendTextPart(parts, nested.param) + } else { + appendTextPart(parts, nested) + } + } + if (parts.length > 0) { + return parts.join(' ') + } + try { + return JSON.stringify(err) + } catch { + return String(err) + } +} + +function getErrorStatus(err: unknown): number | undefined { + if (!isRecord(err)) { + return undefined + } + const candidates = [err.status] + if (isRecord(err.response)) { + candidates.push(err.response.status) + } + if (isRecord(err.error)) { + candidates.push(err.error.status) + } + return candidates.find((status): status is number => typeof status === 'number') +} + +function hasWebSearchUnavailableSignal(err: unknown): boolean { + const message = getErrorText(err).toLowerCase() + const webSearchTerm = '(?:web[_ -]?search|web search|web-search)' + const unavailableTerm = + '(?:not supported|unsupported|not available|unavailable|disabled|not enabled|enable web search|forbidden|not permitted|permission|policy|blocked|access)' + const patterns = [ + new RegExp(`${webSearchTerm}.*${unavailableTerm}`), + new RegExp(`${unavailableTerm}.*${webSearchTerm}`), + /\bmust\s+enable\s+web[_ -]?search\b/, + /\bweb search options\b.*\bnot supported\b/, + /\bhosted tools?\b.*\b(?:not supported|unsupported)\b/, + /\bhosted tool ['"]web_search(?:_preview)?['"].*\b(?:not supported|unsupported)\b/ + ] + return patterns.some((pattern) => pattern.test(message)) +} + +function shouldRetryWithoutWebSearch(err: unknown): boolean { + if (!hasWebSearchUnavailableSignal(err)) { + return false + } + const status = getErrorStatus(err) + return status === undefined || WEB_SEARCH_UNAVAILABLE_STATUS_CODES.has(status) +} + +function markWebSearchUnsupported(cacheKey: string, err: unknown, onWebSearchUnavailable?: () => void) { + unsupportedWebSearchCache.add(cacheKey) + console.warn('Native web search unavailable; retrying without web search:', err) + onWebSearchUnavailable?.() +} + export async function runChatLoop(config: ChatLoopConfig): Promise { const { messages, @@ -84,6 +186,11 @@ export async function runChatLoop(config: ChatLoopConfig): Promise => { + const completion = await getOpenAIResponsesCompletion( + messageParams, + abortController, + toolDefs, + { + forceModelProvider: modelProvider, + openaiClient: clients.openai, + webSearch: useWebSearch, + reasoningEffort + } + ) + const continueCompletion = await parseOpenAIResponsesCompletion( + completion, + callbacks, + messages, + addedMessages, + tools, + helpers, + parseOptions + ) + tokenUsage = addChatTokenUsage(tokenUsage, continueCompletion.tokenUsage) + return continueCompletion.shouldContinue + } + let useCompletionsApi = skipResponsesApi if (!skipResponsesApi) { try { - const completion = await getOpenAIResponsesCompletion( - messageParams, - abortController, - toolDefs, - { - forceModelProvider: modelProvider, - openaiClient: clients.openai, - reasoningEffort - } - ) - const continueCompletion = await parseOpenAIResponsesCompletion( - completion, - callbacks, - messages, - addedMessages, - tools, - helpers, - parseOptions - ) - tokenUsage = addChatTokenUsage(tokenUsage, continueCompletion.tokenUsage) - if (!continueCompletion.shouldContinue) { + if (!(await runOpenAIResponses(webSearch))) { break } } catch (err) { - console.warn('OpenAI Responses API failed, falling back to Completions API:', err) - const errorMessage = err instanceof Error ? err.message : String(err) + let fallbackError = err + if (webSearch && shouldRetryWithoutWebSearch(err)) { + markWebSearchUnsupported(webSearchCacheKey, err, config.onWebSearchUnavailable) + try { + if (!(await runOpenAIResponses(false))) { + break + } + continue + } catch (retryErr) { + fallbackError = retryErr + } + } + + console.warn( + 'OpenAI Responses API failed, falling back to Completions API:', + fallbackError + ) + const errorMessage = getErrorText(fallbackError) if (errorMessage.includes('Responses API is not enabled')) { skipResponsesApi = true onSkipResponsesApi?.() @@ -146,6 +274,11 @@ export async function runChatLoop(config: ChatLoopConfig): Promise => { + const completion = await getAnthropicCompletion( + messageParams, + abortController, + toolDefs, + { + forceModelProvider: modelProvider, + anthropicClient: clients.anthropic, + webSearch: useWebSearch, + reasoningEffort + } + ) + if (!completion) { + return true + } const continueCompletion = await parseAnthropicCompletion( completion, callbacks, @@ -185,9 +327,22 @@ export async function runChatLoop(config: ChatLoopConfig): Promise `openai_web_search:${itemId}` + +function setOpenAIWebSearchStatus( + callbacks: ToolCallbacks & { onMessageEnd: () => void }, + itemId: string, + status: WebSearchStatus +) { + const isLoading = status === 'in_progress' || status === 'searching' + const failed = status === 'failed' + callbacks.onMessageEnd() + callbacks.setToolStatus(openAIWebSearchToolId(itemId), { + content: failed ? 'Web search failed' : isLoading ? 'Searching the web...' : 'Searched the web', + error: failed ? 'Web search failed' : undefined, + isLoading, + isStreamingArguments: false, + needsConfirmation: false, + toolName: 'web_search', + showDetails: false, + autoCollapseDetails: true + }) +} + // Conversion utilities for Responses API function convertMessagesToResponsesInput(messages: ChatCompletionMessageParam[]): { instructions?: string @@ -136,6 +161,7 @@ export async function getOpenAIResponsesCompletion( options?: { forceModelProvider?: AIProviderModel openaiClient?: OpenAI + webSearch?: boolean reasoningEffort?: string } ) { @@ -152,6 +178,12 @@ export async function getOpenAIResponsesCompletion( options?.reasoningEffort ) + // Enable OpenAI's built-in web search tool. The proxy forwards the body + // verbatim, so this reaches OpenAI as a native server-side tool. + if (options?.webSearch && providerSupportsWebSearch(provider)) { + responsesConfig.tools = [...(responsesConfig.tools ?? []), { type: 'web_search' }] + } + const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient() const runner = client.responses.stream( @@ -294,9 +326,23 @@ export async function parseOpenAIResponsesCompletion( showDetails: tool?.showDetails, autoCollapseDetails: tool?.autoCollapseDetails }) + } else if (item.type === 'web_search_call' && item.id) { + setOpenAIWebSearchStatus(callbacks, item.id, item.status) } }) + runner.on('response.web_search_call.in_progress', (event) => { + setOpenAIWebSearchStatus(callbacks, event.item_id, 'in_progress') + }) + + runner.on('response.web_search_call.searching', (event) => { + setOpenAIWebSearchStatus(callbacks, event.item_id, 'searching') + }) + + runner.on('response.web_search_call.completed', (event) => { + setOpenAIWebSearchStatus(callbacks, event.item_id, 'completed') + }) + // Stream function call arguments incrementally runner.on('response.function_call_arguments.delta', (event) => { if (currentStreamingTool?.shouldStream && currentStreamingTool.itemId === event.item_id) { @@ -371,6 +417,12 @@ export async function parseOpenAIResponsesCompletion( const finalResponse = await runner.finalResponse() const tokenUsage = openAIResponsesUsageToChatTokenUsage(finalResponse.usage) + for (const item of finalResponse.output ?? []) { + if (item.type === 'web_search_call') { + setOpenAIWebSearchStatus(callbacks, item.id, item.status) + } + } + // Process tool calls if any if (toolCallsToProcess.length > 0) { const assistantWithTools = { diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index a63a4f1b5a..2977da328a 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -711,6 +711,18 @@ const PROMPTS_CONFIGS = { gen: GEN_CONFIG } +/** + * Whether a provider can use native web search automatically in the web chat. + * Azure OpenAI can expose Responses API `web_search` for some deployments, but + * it is subscription/admin controlled and routes data through Grounding with + * Bing, so do not silently enable it until there is explicit Azure-specific UI. + * Providers behind OpenAI-compatible/native-translation proxy paths have no + * forwardable native web-search tool. + */ +export function providerSupportsWebSearch(provider: AIProvider | undefined): boolean { + return provider === 'openai' || provider === 'anthropic' +} + export function getProviderAndCompletionConfig({ messages, stream, diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 8723d4486c..2080ecd332 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -9,7 +9,7 @@ } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' - import { AI_PROVIDERS, fetchAvailableModels } from '../copilot/lib' + import { AI_PROVIDERS, fetchAvailableModels, providerSupportsWebSearch } from '../copilot/lib' import { supportsAutocomplete } from '../copilot/utils' import TestAiKey from '../copilot/TestAIKey.svelte' import Label from '../Label.svelte' @@ -88,8 +88,21 @@ return JSON.parse(JSON.stringify(v)) } + function normalizeProviderSettings( + providers: Exclude + ): Exclude { + return Object.fromEntries( + Object.entries(providers).map(([provider, config]) => [ + provider, + providerSupportsWebSearch(provider as AIProvider) + ? { ...config, web_search_enabled: config.web_search_enabled ?? true } + : config + ]) + ) + } + function applyConfig(config: AIConfig | undefined) { - aiProviders = clone(config?.providers ?? {}) + aiProviders = normalizeProviderSettings(clone(config?.providers ?? {})) defaultModel = config?.default_model?.model metadataModel = config?.metadata_model?.model codeCompletionModel = config?.code_completion_model?.model @@ -390,7 +403,10 @@ models: availableAiModels[provider].length > 0 ? [availableAiModels[provider][0]] - : [] + : [], + ...(providerSupportsWebSearch(provider as AIProvider) + ? { web_search_enabled: true } + : {}) } } } else { @@ -459,6 +475,22 @@ If you don't see the model you want, you can type it manually in the selector.

+ + {#if providerSupportsWebSearch(provider as AIProvider)} + + {/if} {/if}