fix(ai): centralize Anthropic Messages API routing across completion paths (#9960)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-08-19 08:01:25 +00:00 · 2026-07-06 18:36:21 +02:00
parent dc6b99775b
commit cc2f638de6
2 changed files with 330 additions and 49 deletions
@@ -0,0 +1,214 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import type { AIProviderModel } from '$lib/gen'
+import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs'
+
+// getCurrentModel/getMetadataModel are read per call, so a hoisted holder lets
+// each test point the routing at a different provider/model.
+const h = vi.hoisted(() => ({ currentModel: undefined as AIProviderModel | undefined }))
+
+vi.mock('monaco-editor', () => ({ editor: {} }))
+
+vi.mock('$lib/stores', () => ({
+	workspaceStore: { subscribe: () => () => undefined }
+}))
+
+vi.mock('$lib/components/flows/flowTree', () => ({
+	findModuleInModules: () => undefined
+}))
+
+vi.mock('$lib/gen', () => ({
+	OpenAPI: { BASE: '/api', TOKEN: undefined },
+	ResourceService: {},
+	ScriptService: {},
+	FlowService: {},
+	JobService: {},
+	ScheduleService: {},
+	HttpTriggerService: {},
+	WebsocketTriggerService: {},
+	KafkaTriggerService: {},
+	NatsTriggerService: {},
+	PostgresTriggerService: {},
+	MqttTriggerService: {},
+	SqsTriggerService: {},
+	GcpTriggerService: {},
+	AzureTriggerService: {}
+}))
+
+vi.mock('$lib/utils', () => ({
+	emptyString: (value: string | undefined | null) => !value,
+	generateRandomString: () => 'generated_id'
+}))
+
+vi.mock('$lib/scripts', () => ({
+	scriptLangToEditorLang: (language: string) => language
+}))
+
+vi.mock('$lib/aiStore', () => ({
+	getCurrentModel: () => h.currentModel,
+	getMetadataModel: () => h.currentModel,
+	copilotInfo: {
+		subscribe: (run: (value: unknown) => void) => {
+			run({})
+			return () => undefined
+		}
+	}
+}))
+
+vi.mock('@leeoniya/ufuzzy', () => ({
+	default: class {
+		search() {
+			return [[], [], []]
+		}
+	}
+}))
+
+function streamOf(chunks: unknown[]): any {
+	return (async function* () {
+		for (const chunk of chunks) {
+			yield chunk
+		}
+	})()
+}
+
+function textDelta(text: string) {
+	return { type: 'content_block_delta', delta: { type: 'text_delta', text } }
+}
+
+const messages: ChatCompletionMessageParam[] = [{ role: 'user', content: 'hi' }]
+
+let anthropicCreate: ReturnType<typeof vi.fn>
+let anthropicStream: ReturnType<typeof vi.fn>
+let openaiCreate: ReturnType<typeof vi.fn>
+
+async function setupClients() {
+	const { workspaceAIClients } = await import('./lib')
+
+	anthropicCreate = vi.fn().mockResolvedValue({
+		content: [
+			{ type: 'text', text: 'Hel' },
+			{ type: 'thinking', thinking: 'ignored' },
+			{ type: 'text', text: 'lo' }
+		]
+	})
+	anthropicStream = vi
+		.fn()
+		.mockReturnValue(
+			streamOf([
+				{ type: 'message_start' },
+				textDelta('Hel'),
+				{ type: 'content_block_delta', delta: { type: 'input_json_delta', partial_json: '{' } },
+				textDelta('lo'),
+				{ type: 'message_stop' }
+			])
+		)
+	openaiCreate = vi.fn().mockResolvedValue({ choices: [{ message: { content: 'openai text' } }] })
+
+	vi.spyOn(workspaceAIClients, 'getAnthropicClient').mockReturnValue({
+		messages: { create: anthropicCreate, stream: anthropicStream }
+	} as any)
+	vi.spyOn(workspaceAIClients, 'getOpenaiClient').mockReturnValue({
+		chat: { completions: { create: openaiCreate } }
+	} as any)
+}
+
+beforeEach(async () => {
+	await setupClients()
+})
+
+afterEach(() => {
+	vi.restoreAllMocks()
+	h.currentModel = undefined
+})
+
+describe('Anthropic Messages API routing', () => {
+	it('getNonStreamingCompletion routes Foundry Claude through the Anthropic client', async () => {
+		const { getNonStreamingCompletion } = await import('./lib')
+		h.currentModel = { provider: 'azure_foundry', model: 'claude-sonnet-5' }
+
+		const response = await getNonStreamingCompletion(messages, new AbortController())
+
+		expect(anthropicCreate).toHaveBeenCalledTimes(1)
+		expect(openaiCreate).not.toHaveBeenCalled()
+		// text blocks concatenated, non-text blocks dropped
+		expect(response).toBe('Hello')
+
+		const headers = anthropicCreate.mock.calls[0][1].headers
+		// X-Provider must carry the real provider so the backend resolves Foundry
+		// credentials/URL; the SDK header selects the Messages API path.
+		expect(headers['X-Provider']).toBe('azure_foundry')
+		expect(headers['X-Anthropic-SDK']).toBe('true')
+	})
+
+	it('getNonStreamingCompletion routes native Anthropic through the Anthropic client', async () => {
+		const { getNonStreamingCompletion } = await import('./lib')
+		h.currentModel = { provider: 'anthropic', model: 'claude-opus-4-8' }
+
+		await getNonStreamingCompletion(messages, new AbortController())
+
+		expect(anthropicCreate).toHaveBeenCalledTimes(1)
+		expect(anthropicCreate.mock.calls[0][1].headers['X-Provider']).toBe('anthropic')
+	})
+
+	it('getNonStreamingCompletion keeps non-Claude Foundry models on the OpenAI path', async () => {
+		const { getNonStreamingCompletion } = await import('./lib')
+		h.currentModel = { provider: 'azure_foundry', model: 'gpt-4o' }
+
+		await getNonStreamingCompletion(messages, new AbortController())
+
+		expect(anthropicCreate).not.toHaveBeenCalled()
+		expect(openaiCreate).toHaveBeenCalledTimes(1)
+	})
+
+	it('getCompletion adapts the Anthropic stream into OpenAI text chunks', async () => {
+		const { getCompletion, getResponseFromEvent } = await import('./lib')
+		h.currentModel = { provider: 'azure_foundry', model: 'claude-sonnet-5' }
+
+		const completion = await getCompletion(messages, new AbortController())
+
+		let text = ''
+		let chunks = 0
+		for await (const part of completion) {
+			chunks++
+			text += getResponseFromEvent(part)
+		}
+
+		expect(anthropicStream).toHaveBeenCalledTimes(1)
+		// only the two text deltas surface; message_start/stop and input_json are dropped
+		expect(chunks).toBe(2)
+		expect(text).toBe('Hello')
+	})
+
+	it('testKey routes Foundry Claude through the Anthropic client', async () => {
+		const { testKey } = await import('./lib')
+
+		await testKey({
+			resourcePath: 'u/admin/foundry',
+			model: 'claude-sonnet-5',
+			abortController: new AbortController(),
+			messages,
+			aiProvider: 'azure_foundry'
+		})
+
+		expect(anthropicCreate).toHaveBeenCalledTimes(1)
+		const headers = anthropicCreate.mock.calls[0][1].headers
+		expect(headers['X-Provider']).toBe('azure_foundry')
+		expect(headers['X-Resource-Path']).toBe('u/admin/foundry')
+	})
+
+	it('getFimCompletion no-ops for Anthropic Messages API models', async () => {
+		const { getFimCompletion } = await import('./lib')
+		const fetchSpy = vi.spyOn(globalThis, 'fetch')
+
+		for (const provider of ['anthropic', 'azure_foundry'] as const) {
+			const result = await getFimCompletion(
+				'prefix',
+				'suffix',
+				{ provider, model: 'claude-sonnet-5' },
+				new AbortController()
+			)
+			expect(result).toBeUndefined()
+		}
+		// no autocomplete request should be issued for these models
+		expect(fetchSpy).not.toHaveBeenCalled()
+	})
+})
@@ -302,10 +302,10 @@ export function getModelMaxTokens(provider: AIProvider, model: string) {
 	return 8192
 }

-function getModelSpecificConfig(
-	modelProvider: AIProviderModel,
-	tools?: OpenAI.Chat.Completions.ChatCompletionTool[]
-) {
+// Resolves the completion token cap for a model: the workspace's per-model
+// override when set, otherwise the built-in default. Shared by the OpenAI and
+// Anthropic request paths so both honor the same limit.
+function resolveMaxTokens(modelProvider: AIProviderModel): number {
 	const defaultMaxTokens = getModelMaxTokens(modelProvider.provider, modelProvider.model)
 	const modelKey = `${modelProvider.provider}:${modelProvider.model}`
 	let customMaxTokensStore: Record<string, number> | undefined
@@ -314,7 +314,14 @@ function getModelSpecificConfig(
 	} catch {
 		// copilotInfo store may not be initialized in vitest
 	}
-	const maxTokens = customMaxTokensStore?.[modelKey] ?? defaultMaxTokens
+	return customMaxTokensStore?.[modelKey] ?? defaultMaxTokens
+}
+
+function getModelSpecificConfig(
+	modelProvider: AIProviderModel,
+	tools?: OpenAI.Chat.Completions.ChatCompletionTool[]
+) {
+	const maxTokens = resolveMaxTokens(modelProvider)
 	if (
 		(modelProvider.provider === 'openai' ||
 			modelProvider.provider === 'azure_openai' ||
@@ -466,23 +473,10 @@ export async function testKey({
 		throw new Error('Missing a model to test')
 	}

-	// Providers served through the Anthropic Messages API (native Anthropic, and
-	// Claude deployments on Azure Foundry) must use the Anthropic SDK path rather
-	// than OpenAI chat completions. Mirrors the chat loop's routing so the test
-	// key exercises the same request shape the chat actually sends.
-	if (usesAnthropicMessagesApi(aiProvider, modelToTest)) {
-		await testAnthropicKey({
-			apiKey,
-			workspace,
-			resourcePath,
-			model: modelToTest,
-			abortController,
-			messages,
-			aiProvider
-		})
-		return
-	}
-
+	// getNonStreamingCompletion routes Anthropic-Messages-API models (native
+	// Anthropic and Claude on Azure Foundry) through the Anthropic SDK and
+	// everything else through OpenAI chat completions, so the test exercises the
+	// same request shape the feature actually sends.
 	await getNonStreamingCompletion(messages, abortController, {
 		apiKey,
 		workspace,
@@ -494,30 +488,35 @@ export async function testKey({
 	})
 }

-async function testAnthropicKey({
-	apiKey,
-	workspace,
-	resourcePath,
-	model,
-	abortController,
-	messages,
-	aiProvider
-}: {
+// Providers served through the Anthropic Messages API (native Anthropic, and
+// Claude deployments on Azure Foundry) require the Anthropic SDK request shape:
+// OpenAI chat-completions requests fail against them because the proxy forwards
+// the body verbatim and, for Foundry, rewrites the URL to the /anthropic/v1
+// surface that only serves /messages. This centralizes the client/header/message
+// setup so every completion entry point routes them the same way the chat does.
+interface AnthropicCompletionParams {
+	messages: ChatCompletionMessageParam[]
+	modelProvider: AIProviderModel
+	abortController: AbortController
 	apiKey?: string
 	workspace?: string
 	resourcePath?: string
-	model: string
-	abortController: AbortController
-	messages: ChatCompletionMessageParam[]
-	aiProvider: AIProvider
-}) {
+}
+
+function buildAnthropicProxyRequest({
+	messages,
+	modelProvider,
+	apiKey,
+	workspace,
+	resourcePath
+}: Omit<AnthropicCompletionParams, 'abortController'>) {
 	const { system, messages: anthropicMessages } = convertOpenAIToAnthropicMessages(messages)

 	// X-Provider must be the real provider (e.g. azure_foundry) so the backend
 	// resolves the right credentials and Anthropic URL; the SDK headers tell it to
 	// route through the Anthropic Messages API.
 	const headers: Record<string, string> = {
-		'X-Provider': aiProvider,
+		'X-Provider': modelProvider.provider,
 		'anthropic-version': '2023-06-01',
 		'X-Anthropic-SDK': 'true'
 	}
@@ -528,24 +527,65 @@ async function testAnthropicKey({
 		headers['X-API-Key'] = apiKey
 	}

-	const anthropicClient = apiKey
+	const client = apiKey
 		? createAnthropicProxyClient(getAiProxyBaseURL())
 		: workspace
 			? workspaceAIClients.createAnthropicClient(workspace)
 			: workspaceAIClients.getAnthropicClient()

-	await anthropicClient.messages.create(
-		{
-			model,
-			max_tokens: 100,
-			messages: anthropicMessages,
-			...(system && { system })
-		},
-		{
-			signal: abortController.signal,
-			headers
+	const body = {
+		model: modelProvider.model,
+		max_tokens: resolveMaxTokens(modelProvider),
+		messages: anthropicMessages,
+		...(system && { system })
+	}
+
+	return { client, headers, body }
+}
+
+async function getAnthropicNonStreamingCompletion({
+	abortController,
+	...params
+}: AnthropicCompletionParams): Promise<string> {
+	const { client, headers, body } = buildAnthropicProxyRequest(params)
+
+	const message = await client.messages.create(body, {
+		signal: abortController.signal,
+		headers
+	})
+
+	return message.content.map((block) => (block.type === 'text' ? block.text : '')).join('')
+}
+
+// Adapts an Anthropic Messages stream into the OpenAI ChatCompletionChunk shape
+// the completion consumers already iterate, so they need no Anthropic-specific
+// handling. Only text deltas are surfaced (these paths don't use tool calls).
+function getAnthropicStreamingCompletion({
+	abortController,
+	...params
+}: AnthropicCompletionParams): Stream<ChatCompletionChunk> {
+	const { client, headers, body } = buildAnthropicProxyRequest(params)
+
+	const stream = client.messages.stream(body, {
+		signal: abortController.signal,
+		headers
+	})
+
+	async function* toOpenAIChunks(): AsyncGenerator<ChatCompletionChunk> {
+		for await (const event of stream) {
+			if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') {
+				yield {
+					id: '',
+					object: 'chat.completion.chunk',
+					created: 0,
+					model: params.modelProvider.model,
+					choices: [{ index: 0, delta: { content: event.delta.text }, finish_reason: null }]
+				}
+			}
 		}
-	)
+	}
+
+	return toOpenAIChunks() as unknown as Stream<ChatCompletionChunk>
 }

 interface BaseOptions {
@@ -773,6 +813,19 @@ export async function getNonStreamingCompletion(
 		forceModelProvider?: AIProviderModel
 	}
 ) {
+	const modelProvider = options?.forceModelProvider ?? getCurrentModel()
+
+	if (usesAnthropicMessagesApi(modelProvider.provider, modelProvider.model)) {
+		return getAnthropicNonStreamingCompletion({
+			messages,
+			modelProvider,
+			abortController,
+			apiKey: options?.apiKey,
+			workspace: options?.workspace,
+			resourcePath: options?.resourcePath
+		})
+	}
+
 	let response: string | undefined = ''
 	const { provider, config } = getProviderAndCompletionConfig({
 		messages,
@@ -846,6 +899,14 @@ export async function getFimCompletion(
 	providerModel: AIProviderModel,
 	abortController: AbortController
 ): Promise<string | undefined> {
+	// The Anthropic Messages API has no fill-in-the-middle endpoint, and Foundry
+	// Claude deployments don't expose the OpenAI-compatible completions surface the
+	// FIM proxy targets. Skip autocomplete for these models rather than issuing a
+	// request that can't succeed.
+	if (usesAnthropicMessagesApi(providerModel.provider, providerModel.model)) {
+		return undefined
+	}
+
 	const fetchOptions: {
 		signal: AbortSignal
 		headers: Record<string, string>
@@ -908,6 +969,12 @@ export async function getCompletion(
 		reasoningEffort?: string
 	}
 ): Promise<Stream<ChatCompletionChunk>> {
+	const modelProvider = options?.forceModelProvider ?? getCurrentModel()
+
+	if (usesAnthropicMessagesApi(modelProvider.provider, modelProvider.model)) {
+		return getAnthropicStreamingCompletion({ messages, modelProvider, abortController })
+	}
+
 	const { provider, config } = getProviderAndCompletionConfig({
 		messages,
 		stream: true,