From cc2f638de6cebeffb9fee1d4835a0cfd565af86c Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 6 Jul 2026 18:36:21 +0200 Subject: [PATCH] fix(ai): centralize Anthropic Messages API routing across completion paths (#9960) Co-authored-by: Claude Opus 4.8 (1M context) --- .../copilot/lib.anthropicRouting.test.ts | 214 ++++++++++++++++++ frontend/src/lib/components/copilot/lib.ts | 165 ++++++++++---- 2 files changed, 330 insertions(+), 49 deletions(-) create mode 100644 frontend/src/lib/components/copilot/lib.anthropicRouting.test.ts diff --git a/frontend/src/lib/components/copilot/lib.anthropicRouting.test.ts b/frontend/src/lib/components/copilot/lib.anthropicRouting.test.ts new file mode 100644 index 0000000000..2f2b0add26 --- /dev/null +++ b/frontend/src/lib/components/copilot/lib.anthropicRouting.test.ts @@ -0,0 +1,214 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AIProviderModel } from '$lib/gen' +import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' + +// getCurrentModel/getMetadataModel are read per call, so a hoisted holder lets +// each test point the routing at a different provider/model. +const h = vi.hoisted(() => ({ currentModel: undefined as AIProviderModel | undefined })) + +vi.mock('monaco-editor', () => ({ editor: {} })) + +vi.mock('$lib/stores', () => ({ + workspaceStore: { subscribe: () => () => undefined } +})) + +vi.mock('$lib/components/flows/flowTree', () => ({ + findModuleInModules: () => undefined +})) + +vi.mock('$lib/gen', () => ({ + OpenAPI: { BASE: '/api', TOKEN: undefined }, + ResourceService: {}, + ScriptService: {}, + FlowService: {}, + JobService: {}, + ScheduleService: {}, + HttpTriggerService: {}, + WebsocketTriggerService: {}, + KafkaTriggerService: {}, + NatsTriggerService: {}, + PostgresTriggerService: {}, + MqttTriggerService: {}, + SqsTriggerService: {}, + GcpTriggerService: {}, + AzureTriggerService: {} +})) + +vi.mock('$lib/utils', () => ({ + emptyString: (value: string | undefined | null) => !value, + generateRandomString: () => 'generated_id' +})) + +vi.mock('$lib/scripts', () => ({ + scriptLangToEditorLang: (language: string) => language +})) + +vi.mock('$lib/aiStore', () => ({ + getCurrentModel: () => h.currentModel, + getMetadataModel: () => h.currentModel, + copilotInfo: { + subscribe: (run: (value: unknown) => void) => { + run({}) + return () => undefined + } + } +})) + +vi.mock('@leeoniya/ufuzzy', () => ({ + default: class { + search() { + return [[], [], []] + } + } +})) + +function streamOf(chunks: unknown[]): any { + return (async function* () { + for (const chunk of chunks) { + yield chunk + } + })() +} + +function textDelta(text: string) { + return { type: 'content_block_delta', delta: { type: 'text_delta', text } } +} + +const messages: ChatCompletionMessageParam[] = [{ role: 'user', content: 'hi' }] + +let anthropicCreate: ReturnType +let anthropicStream: ReturnType +let openaiCreate: ReturnType + +async function setupClients() { + const { workspaceAIClients } = await import('./lib') + + anthropicCreate = vi.fn().mockResolvedValue({ + content: [ + { type: 'text', text: 'Hel' }, + { type: 'thinking', thinking: 'ignored' }, + { type: 'text', text: 'lo' } + ] + }) + anthropicStream = vi + .fn() + .mockReturnValue( + streamOf([ + { type: 'message_start' }, + textDelta('Hel'), + { type: 'content_block_delta', delta: { type: 'input_json_delta', partial_json: '{' } }, + textDelta('lo'), + { type: 'message_stop' } + ]) + ) + openaiCreate = vi.fn().mockResolvedValue({ choices: [{ message: { content: 'openai text' } }] }) + + vi.spyOn(workspaceAIClients, 'getAnthropicClient').mockReturnValue({ + messages: { create: anthropicCreate, stream: anthropicStream } + } as any) + vi.spyOn(workspaceAIClients, 'getOpenaiClient').mockReturnValue({ + chat: { completions: { create: openaiCreate } } + } as any) +} + +beforeEach(async () => { + await setupClients() +}) + +afterEach(() => { + vi.restoreAllMocks() + h.currentModel = undefined +}) + +describe('Anthropic Messages API routing', () => { + it('getNonStreamingCompletion routes Foundry Claude through the Anthropic client', async () => { + const { getNonStreamingCompletion } = await import('./lib') + h.currentModel = { provider: 'azure_foundry', model: 'claude-sonnet-5' } + + const response = await getNonStreamingCompletion(messages, new AbortController()) + + expect(anthropicCreate).toHaveBeenCalledTimes(1) + expect(openaiCreate).not.toHaveBeenCalled() + // text blocks concatenated, non-text blocks dropped + expect(response).toBe('Hello') + + const headers = anthropicCreate.mock.calls[0][1].headers + // X-Provider must carry the real provider so the backend resolves Foundry + // credentials/URL; the SDK header selects the Messages API path. + expect(headers['X-Provider']).toBe('azure_foundry') + expect(headers['X-Anthropic-SDK']).toBe('true') + }) + + it('getNonStreamingCompletion routes native Anthropic through the Anthropic client', async () => { + const { getNonStreamingCompletion } = await import('./lib') + h.currentModel = { provider: 'anthropic', model: 'claude-opus-4-8' } + + await getNonStreamingCompletion(messages, new AbortController()) + + expect(anthropicCreate).toHaveBeenCalledTimes(1) + expect(anthropicCreate.mock.calls[0][1].headers['X-Provider']).toBe('anthropic') + }) + + it('getNonStreamingCompletion keeps non-Claude Foundry models on the OpenAI path', async () => { + const { getNonStreamingCompletion } = await import('./lib') + h.currentModel = { provider: 'azure_foundry', model: 'gpt-4o' } + + await getNonStreamingCompletion(messages, new AbortController()) + + expect(anthropicCreate).not.toHaveBeenCalled() + expect(openaiCreate).toHaveBeenCalledTimes(1) + }) + + it('getCompletion adapts the Anthropic stream into OpenAI text chunks', async () => { + const { getCompletion, getResponseFromEvent } = await import('./lib') + h.currentModel = { provider: 'azure_foundry', model: 'claude-sonnet-5' } + + const completion = await getCompletion(messages, new AbortController()) + + let text = '' + let chunks = 0 + for await (const part of completion) { + chunks++ + text += getResponseFromEvent(part) + } + + expect(anthropicStream).toHaveBeenCalledTimes(1) + // only the two text deltas surface; message_start/stop and input_json are dropped + expect(chunks).toBe(2) + expect(text).toBe('Hello') + }) + + it('testKey routes Foundry Claude through the Anthropic client', async () => { + const { testKey } = await import('./lib') + + await testKey({ + resourcePath: 'u/admin/foundry', + model: 'claude-sonnet-5', + abortController: new AbortController(), + messages, + aiProvider: 'azure_foundry' + }) + + expect(anthropicCreate).toHaveBeenCalledTimes(1) + const headers = anthropicCreate.mock.calls[0][1].headers + expect(headers['X-Provider']).toBe('azure_foundry') + expect(headers['X-Resource-Path']).toBe('u/admin/foundry') + }) + + it('getFimCompletion no-ops for Anthropic Messages API models', async () => { + const { getFimCompletion } = await import('./lib') + const fetchSpy = vi.spyOn(globalThis, 'fetch') + + for (const provider of ['anthropic', 'azure_foundry'] as const) { + const result = await getFimCompletion( + 'prefix', + 'suffix', + { provider, model: 'claude-sonnet-5' }, + new AbortController() + ) + expect(result).toBeUndefined() + } + // no autocomplete request should be issued for these models + expect(fetchSpy).not.toHaveBeenCalled() + }) +}) diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 323b552cca..808f9e0988 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -302,10 +302,10 @@ export function getModelMaxTokens(provider: AIProvider, model: string) { return 8192 } -function getModelSpecificConfig( - modelProvider: AIProviderModel, - tools?: OpenAI.Chat.Completions.ChatCompletionTool[] -) { +// Resolves the completion token cap for a model: the workspace's per-model +// override when set, otherwise the built-in default. Shared by the OpenAI and +// Anthropic request paths so both honor the same limit. +function resolveMaxTokens(modelProvider: AIProviderModel): number { const defaultMaxTokens = getModelMaxTokens(modelProvider.provider, modelProvider.model) const modelKey = `${modelProvider.provider}:${modelProvider.model}` let customMaxTokensStore: Record | undefined @@ -314,7 +314,14 @@ function getModelSpecificConfig( } catch { // copilotInfo store may not be initialized in vitest } - const maxTokens = customMaxTokensStore?.[modelKey] ?? defaultMaxTokens + return customMaxTokensStore?.[modelKey] ?? defaultMaxTokens +} + +function getModelSpecificConfig( + modelProvider: AIProviderModel, + tools?: OpenAI.Chat.Completions.ChatCompletionTool[] +) { + const maxTokens = resolveMaxTokens(modelProvider) if ( (modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai' || @@ -466,23 +473,10 @@ export async function testKey({ throw new Error('Missing a model to test') } - // Providers served through the Anthropic Messages API (native Anthropic, and - // Claude deployments on Azure Foundry) must use the Anthropic SDK path rather - // than OpenAI chat completions. Mirrors the chat loop's routing so the test - // key exercises the same request shape the chat actually sends. - if (usesAnthropicMessagesApi(aiProvider, modelToTest)) { - await testAnthropicKey({ - apiKey, - workspace, - resourcePath, - model: modelToTest, - abortController, - messages, - aiProvider - }) - return - } - + // getNonStreamingCompletion routes Anthropic-Messages-API models (native + // Anthropic and Claude on Azure Foundry) through the Anthropic SDK and + // everything else through OpenAI chat completions, so the test exercises the + // same request shape the feature actually sends. await getNonStreamingCompletion(messages, abortController, { apiKey, workspace, @@ -494,30 +488,35 @@ export async function testKey({ }) } -async function testAnthropicKey({ - apiKey, - workspace, - resourcePath, - model, - abortController, - messages, - aiProvider -}: { +// Providers served through the Anthropic Messages API (native Anthropic, and +// Claude deployments on Azure Foundry) require the Anthropic SDK request shape: +// OpenAI chat-completions requests fail against them because the proxy forwards +// the body verbatim and, for Foundry, rewrites the URL to the /anthropic/v1 +// surface that only serves /messages. This centralizes the client/header/message +// setup so every completion entry point routes them the same way the chat does. +interface AnthropicCompletionParams { + messages: ChatCompletionMessageParam[] + modelProvider: AIProviderModel + abortController: AbortController apiKey?: string workspace?: string resourcePath?: string - model: string - abortController: AbortController - messages: ChatCompletionMessageParam[] - aiProvider: AIProvider -}) { +} + +function buildAnthropicProxyRequest({ + messages, + modelProvider, + apiKey, + workspace, + resourcePath +}: Omit) { const { system, messages: anthropicMessages } = convertOpenAIToAnthropicMessages(messages) // X-Provider must be the real provider (e.g. azure_foundry) so the backend // resolves the right credentials and Anthropic URL; the SDK headers tell it to // route through the Anthropic Messages API. const headers: Record = { - 'X-Provider': aiProvider, + 'X-Provider': modelProvider.provider, 'anthropic-version': '2023-06-01', 'X-Anthropic-SDK': 'true' } @@ -528,24 +527,65 @@ async function testAnthropicKey({ headers['X-API-Key'] = apiKey } - const anthropicClient = apiKey + const client = apiKey ? createAnthropicProxyClient(getAiProxyBaseURL()) : workspace ? workspaceAIClients.createAnthropicClient(workspace) : workspaceAIClients.getAnthropicClient() - await anthropicClient.messages.create( - { - model, - max_tokens: 100, - messages: anthropicMessages, - ...(system && { system }) - }, - { - signal: abortController.signal, - headers + const body = { + model: modelProvider.model, + max_tokens: resolveMaxTokens(modelProvider), + messages: anthropicMessages, + ...(system && { system }) + } + + return { client, headers, body } +} + +async function getAnthropicNonStreamingCompletion({ + abortController, + ...params +}: AnthropicCompletionParams): Promise { + const { client, headers, body } = buildAnthropicProxyRequest(params) + + const message = await client.messages.create(body, { + signal: abortController.signal, + headers + }) + + return message.content.map((block) => (block.type === 'text' ? block.text : '')).join('') +} + +// Adapts an Anthropic Messages stream into the OpenAI ChatCompletionChunk shape +// the completion consumers already iterate, so they need no Anthropic-specific +// handling. Only text deltas are surfaced (these paths don't use tool calls). +function getAnthropicStreamingCompletion({ + abortController, + ...params +}: AnthropicCompletionParams): Stream { + const { client, headers, body } = buildAnthropicProxyRequest(params) + + const stream = client.messages.stream(body, { + signal: abortController.signal, + headers + }) + + async function* toOpenAIChunks(): AsyncGenerator { + for await (const event of stream) { + if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') { + yield { + id: '', + object: 'chat.completion.chunk', + created: 0, + model: params.modelProvider.model, + choices: [{ index: 0, delta: { content: event.delta.text }, finish_reason: null }] + } + } } - ) + } + + return toOpenAIChunks() as unknown as Stream } interface BaseOptions { @@ -773,6 +813,19 @@ export async function getNonStreamingCompletion( forceModelProvider?: AIProviderModel } ) { + const modelProvider = options?.forceModelProvider ?? getCurrentModel() + + if (usesAnthropicMessagesApi(modelProvider.provider, modelProvider.model)) { + return getAnthropicNonStreamingCompletion({ + messages, + modelProvider, + abortController, + apiKey: options?.apiKey, + workspace: options?.workspace, + resourcePath: options?.resourcePath + }) + } + let response: string | undefined = '' const { provider, config } = getProviderAndCompletionConfig({ messages, @@ -846,6 +899,14 @@ export async function getFimCompletion( providerModel: AIProviderModel, abortController: AbortController ): Promise { + // The Anthropic Messages API has no fill-in-the-middle endpoint, and Foundry + // Claude deployments don't expose the OpenAI-compatible completions surface the + // FIM proxy targets. Skip autocomplete for these models rather than issuing a + // request that can't succeed. + if (usesAnthropicMessagesApi(providerModel.provider, providerModel.model)) { + return undefined + } + const fetchOptions: { signal: AbortSignal headers: Record @@ -908,6 +969,12 @@ export async function getCompletion( reasoningEffort?: string } ): Promise> { + const modelProvider = options?.forceModelProvider ?? getCurrentModel() + + if (usesAnthropicMessagesApi(modelProvider.provider, modelProvider.model)) { + return getAnthropicStreamingCompletion({ messages, modelProvider, abortController }) + } + const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true,