From fec40086961174fea25b4e1f796991152b84b211 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Mon, 18 May 2026 12:40:18 +0200 Subject: [PATCH] fix: preserve ai reasoning content (#9208) * fix: preserve ai reasoning content * fix: avoid text-only reasoning replay * feat: add deepseek ai eval models --- .../frontend/core/shared/baseEvalRunner.ts | 3 +- .../core/shared/providerConfig.test.ts | 7 ++ .../frontend/core/shared/providerConfig.ts | 3 + ai_evals/core/models.test.ts | 25 ++++-- ai_evals/core/models.ts | 44 +++++++-- ai_evals/modes/frontendCommon.test.ts | 9 +- ai_evals/modes/frontendCommon.ts | 8 +- .../copilot/chat/openaiReasoning.ts | 51 +++++++++++ .../src/lib/components/copilot/lib.test.ts | 90 +++++++++++++++++++ frontend/src/lib/components/copilot/lib.ts | 66 +++++++++----- 10 files changed, 269 insertions(+), 37 deletions(-) create mode 100644 frontend/src/lib/components/copilot/chat/openaiReasoning.ts diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 7fd43ccb87..5f4ea2c307 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -236,7 +236,8 @@ function toFrontendEvalProvider( if ( provider === "anthropic" || provider === "openai" || - provider === "googleai" + provider === "googleai" || + provider === "deepseek" ) { return provider; } diff --git a/ai_evals/adapters/frontend/core/shared/providerConfig.test.ts b/ai_evals/adapters/frontend/core/shared/providerConfig.test.ts index 77d9154da3..819300bd62 100644 --- a/ai_evals/adapters/frontend/core/shared/providerConfig.test.ts +++ b/ai_evals/adapters/frontend/core/shared/providerConfig.test.ts @@ -27,6 +27,13 @@ describe("resolveEvalModelProvider", () => { }); }); + it("infers deepseek from DeepSeek model ids", () => { + expect(resolveEvalModelProvider("deepseek-v4-flash")).toEqual({ + provider: "deepseek", + model: "deepseek-v4-flash", + }); + }); + it("preserves an explicit provider", () => { expect(resolveEvalModelProvider("gemini-2.5-pro", "googleai")).toEqual({ provider: "googleai", diff --git a/ai_evals/adapters/frontend/core/shared/providerConfig.ts b/ai_evals/adapters/frontend/core/shared/providerConfig.ts index 15372049fe..9d9e315217 100644 --- a/ai_evals/adapters/frontend/core/shared/providerConfig.ts +++ b/ai_evals/adapters/frontend/core/shared/providerConfig.ts @@ -83,6 +83,9 @@ export function resolveEvalModelProvider( if (model.startsWith("gemini")) { return { provider: "googleai", model }; } + if (model.startsWith("deepseek")) { + return { provider: "deepseek", model }; + } if (model.startsWith("gpt") || model.startsWith("o")) { return { provider: "openai", model }; } diff --git a/ai_evals/core/models.test.ts b/ai_evals/core/models.test.ts index 86bf1c6a9a..a11fe40530 100644 --- a/ai_evals/core/models.test.ts +++ b/ai_evals/core/models.test.ts @@ -11,19 +11,34 @@ describe("resolveEvalModel", () => { provider: "googleai", model: "gemini-2.5-pro", }); - expect(resolveEvalModel("script", "gemini-3-flash-preview").frontend).toEqual({ + expect( + resolveEvalModel("script", "gemini-3-flash-preview").frontend, + ).toEqual({ provider: "googleai", model: "gemini-3-flash-preview", }); - expect(resolveEvalModel("flow", "gemini-3.1-pro-preview").frontend).toEqual({ - provider: "googleai", - model: "gemini-3.1-pro-preview", + expect(resolveEvalModel("flow", "gemini-3.1-pro-preview").frontend).toEqual( + { + provider: "googleai", + model: "gemini-3.1-pro-preview", + }, + ); + }); + + it("supports DeepSeek aliases for frontend evals", () => { + expect(resolveEvalModel("flow", "deepseek").frontend).toEqual({ + provider: "deepseek", + model: "deepseek-v4-flash", + }); + expect(resolveEvalModel("script", "deepseek-v4-pro").frontend).toEqual({ + provider: "deepseek", + model: "deepseek-v4-pro", }); }); it("rejects Gemini aliases for cli evals", () => { expect(() => resolveEvalModel("cli", "gemini")).toThrow( - "Model gemini-flash is not supported for cli mode" + "Model gemini-flash is not supported for cli mode", ); }); }); diff --git a/ai_evals/core/models.ts b/ai_evals/core/models.ts index 82f3b3f69b..054a27bc1c 100644 --- a/ai_evals/core/models.ts +++ b/ai_evals/core/models.ts @@ -1,7 +1,7 @@ import type { EvalMode } from "./types"; export interface FrontendEvalModelConfig { - provider: "anthropic" | "openai" | "googleai"; + provider: "anthropic" | "openai" | "googleai" | "deepseek"; model: string; } @@ -117,15 +117,40 @@ export const EVAL_MODELS: EvalModelSpec[] = [ { id: "gemini-3.1-pro-preview", label: "Gemini 3.1 Pro Preview", - aliases: ["gemini-3.1-pro-preview", "gemini-3.1-pro", "gemini-3-pro-preview"], + aliases: [ + "gemini-3.1-pro-preview", + "gemini-3.1-pro", + "gemini-3-pro-preview", + ], frontend: { provider: "googleai", model: "gemini-3.1-pro-preview", }, }, + { + id: "deepseek-v4-flash", + label: "DeepSeek V4 Flash", + aliases: ["deepseek", "deepseek-v4", "deepseek-v4-flash"], + frontend: { + provider: "deepseek", + model: "deepseek-v4-flash", + }, + }, + { + id: "deepseek-v4-pro", + label: "DeepSeek V4 Pro", + aliases: ["deepseek-pro", "deepseek-v4-pro"], + frontend: { + provider: "deepseek", + model: "deepseek-v4-pro", + }, + }, ]; -export function resolveEvalModel(mode: EvalMode, alias?: string): EvalModelSpec { +export function resolveEvalModel( + mode: EvalMode, + alias?: string, +): EvalModelSpec { const spec = alias ? findEvalModel(alias) : getDefaultEvalModel(mode); if (!spec) { throw new Error(`Unknown model: ${alias}`); @@ -152,14 +177,19 @@ export function getEvalModelHelpText(): string { }).join("\n"); } -export function formatRunModelLabel(mode: EvalMode, model: EvalModelSpec): string { +export function formatRunModelLabel( + mode: EvalMode, + model: EvalModelSpec, +): string { if (mode === "cli") { return `${model.cli!.provider}:${model.cli!.model}`; } return `${model.frontend!.provider}:${model.frontend!.model}`; } -export function getFrontendEvalModel(model: EvalModelSpec): FrontendEvalModelConfig { +export function getFrontendEvalModel( + model: EvalModelSpec, +): FrontendEvalModelConfig { if (!model.frontend) { throw new Error(`Model ${model.id} does not support frontend evals`); } @@ -180,6 +210,8 @@ function getDefaultEvalModel(mode: EvalMode): EvalModelSpec { function findEvalModel(alias: string): EvalModelSpec | undefined { const normalized = alias.trim().toLowerCase(); return EVAL_MODELS.find((model) => - [model.id, ...model.aliases].some((candidate) => candidate.toLowerCase() === normalized) + [model.id, ...model.aliases].some( + (candidate) => candidate.toLowerCase() === normalized, + ), ); } diff --git a/ai_evals/modes/frontendCommon.test.ts b/ai_evals/modes/frontendCommon.test.ts index cac10ffcab..897ac3f8a3 100644 --- a/ai_evals/modes/frontendCommon.test.ts +++ b/ai_evals/modes/frontendCommon.test.ts @@ -5,12 +5,14 @@ const ORIGINAL_ENV = { ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY, OPENAI_API_KEY: process.env.OPENAI_API_KEY, GEMINI_API_KEY: process.env.GEMINI_API_KEY, + DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY, }; afterEach(() => { process.env.ANTHROPIC_API_KEY = ORIGINAL_ENV.ANTHROPIC_API_KEY; process.env.OPENAI_API_KEY = ORIGINAL_ENV.OPENAI_API_KEY; process.env.GEMINI_API_KEY = ORIGINAL_ENV.GEMINI_API_KEY; + process.env.DEEPSEEK_API_KEY = ORIGINAL_ENV.DEEPSEEK_API_KEY; }); describe("getFrontendApiKey", () => { @@ -19,10 +21,15 @@ describe("getFrontendApiKey", () => { expect(getFrontendApiKey("googleai")).toBe("gemini-test-key"); }); + it("reads the DeepSeek API key for deepseek models", () => { + process.env.DEEPSEEK_API_KEY = "deepseek-test-key"; + expect(getFrontendApiKey("deepseek")).toBe("deepseek-test-key"); + }); + it("throws a provider-specific error when the key is missing", () => { delete process.env.GEMINI_API_KEY; expect(() => getFrontendApiKey("googleai")).toThrow( - "GEMINI_API_KEY is required for frontend evals" + "GEMINI_API_KEY is required for frontend evals", ); }); }); diff --git a/ai_evals/modes/frontendCommon.ts b/ai_evals/modes/frontendCommon.ts index f121551d86..b81907b42d 100644 --- a/ai_evals/modes/frontendCommon.ts +++ b/ai_evals/modes/frontendCommon.ts @@ -1,12 +1,16 @@ import type { FrontendEvalModelConfig } from "../core/models"; -export function getFrontendApiKey(provider: FrontendEvalModelConfig["provider"]): string { +export function getFrontendApiKey( + provider: FrontendEvalModelConfig["provider"], +): string { const envName = provider === "anthropic" ? "ANTHROPIC_API_KEY" : provider === "googleai" ? "GEMINI_API_KEY" - : "OPENAI_API_KEY"; + : provider === "deepseek" + ? "DEEPSEEK_API_KEY" + : "OPENAI_API_KEY"; const apiKey = process.env[envName]; if (!apiKey) { throw new Error(`${envName} is required for frontend evals`); diff --git a/frontend/src/lib/components/copilot/chat/openaiReasoning.ts b/frontend/src/lib/components/copilot/chat/openaiReasoning.ts new file mode 100644 index 0000000000..da809641a0 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/openaiReasoning.ts @@ -0,0 +1,51 @@ +import type { + ChatCompletionChunk, + ChatCompletionMessageFunctionToolCall, + ChatCompletionMessageParam +} from 'openai/resources/index.mjs' + +type ChatCompletionDeltaWithReasoning = ChatCompletionChunk.Choice.Delta & { + reasoning_content?: string | null +} + +type ChatCompletionAssistantMessageWithReasoning = ChatCompletionMessageParam & { + role: 'assistant' + reasoning_content?: string +} + +export type ReasoningContentState = { + hasReasoningContent: boolean + reasoningContent: string +} + +export function getReasoningContentDelta( + delta: ChatCompletionChunk.Choice.Delta +): string | null | undefined { + return (delta as ChatCompletionDeltaWithReasoning).reasoning_content +} + +export function buildAssistantTextMessage( + content: string +): ChatCompletionAssistantMessageWithReasoning { + return { + role: 'assistant', + content + } +} + +export function buildAssistantToolCallMessage({ + content, + reasoning, + toolCalls +}: { + content: string + reasoning: ReasoningContentState + toolCalls: ChatCompletionMessageFunctionToolCall[] +}): ChatCompletionAssistantMessageWithReasoning { + return { + role: 'assistant', + ...(content || reasoning.hasReasoningContent ? { content } : {}), + ...(reasoning.hasReasoningContent ? { reasoning_content: reasoning.reasoningContent } : {}), + tool_calls: toolCalls + } +} diff --git a/frontend/src/lib/components/copilot/lib.test.ts b/frontend/src/lib/components/copilot/lib.test.ts index a559401068..f1eec17cbb 100644 --- a/frontend/src/lib/components/copilot/lib.test.ts +++ b/frontend/src/lib/components/copilot/lib.test.ts @@ -1,6 +1,23 @@ +import type { + ChatCompletionChunk, + ChatCompletionMessageFunctionToolCall, + ChatCompletionMessageParam +} from 'openai/resources/index.mjs' import { describe, expect, it } from 'vitest' +import { + buildAssistantTextMessage, + buildAssistantToolCallMessage, + getReasoningContentDelta +} from './chat/openaiReasoning' import { getDefaultChatTemperature, modelDisallowsSamplingParams } from './modelConfig' +type AssistantMessageWithReasoning = ChatCompletionMessageParam & { + role: 'assistant' + content?: string + reasoning_content?: string + tool_calls?: ChatCompletionMessageFunctionToolCall[] +} + describe('modelConfig', () => { it('flags Opus 4.7 model IDs via includes matching', () => { expect(modelDisallowsSamplingParams('claude-opus-4-7')).toBe(true) @@ -25,3 +42,76 @@ describe('modelConfig', () => { expect(getDefaultChatTemperature({ provider: 'anthropic', model: 'claude-sonnet-4-6' })).toBe(0) }) }) + +describe('openaiReasoning', () => { + it('reads provider-specific reasoning_content deltas', () => { + expect( + getReasoningContentDelta({ + reasoning_content: 'thinking' + } as ChatCompletionChunk.Choice.Delta & { reasoning_content: string }) + ).toBe('thinking') + }) + + it('preserves DeepSeek reasoning_content on assistant tool-call messages', () => { + const toolCalls: ChatCompletionMessageFunctionToolCall[] = [ + { + id: 'call_1', + type: 'function', + function: { + name: 'lookup', + arguments: '{"query":"docs"}' + } + } + ] + + const assistantMessage = buildAssistantToolCallMessage({ + content: 'I will look that up.', + reasoning: { + hasReasoningContent: true, + reasoningContent: 'First, I need a lookup.' + }, + toolCalls + }) as AssistantMessageWithReasoning + + expect(assistantMessage).toMatchObject({ + role: 'assistant', + content: 'I will look that up.', + reasoning_content: 'First, I need a lookup.', + tool_calls: [ + { + id: 'call_1', + type: 'function', + function: { + name: 'lookup', + arguments: '{"query":"docs"}' + } + } + ] + }) + }) + + it('does not preserve reasoning_content on text-only assistant messages', () => { + expect(buildAssistantTextMessage('done')).toEqual({ + role: 'assistant', + content: 'done' + }) + }) + + it('keeps empty reasoning_content when the provider emitted the field', () => { + const assistantMessage = buildAssistantToolCallMessage({ + content: '', + reasoning: { + hasReasoningContent: true, + reasoningContent: '' + }, + toolCalls: [] + }) as AssistantMessageWithReasoning + + expect(assistantMessage).toMatchObject({ + role: 'assistant', + content: '', + reasoning_content: '', + tool_calls: [] + }) + }) +}) diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index e98ff035db..f919103400 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -31,6 +31,11 @@ import { openAICompletionsUsageToChatTokenUsage, type ChatTokenUsage } from './chat/tokenUsage' +import { + buildAssistantTextMessage, + buildAssistantToolCallMessage, + getReasoningContentDelta +} from './chat/openaiReasoning' export const SUPPORTED_LANGUAGES = new Set(Object.keys(GEN_CONFIG.prompts)) @@ -960,6 +965,9 @@ export async function parseOpenAICompletion( let tokenUsage = emptyChatTokenUsage() let answer = '' + let assistantContent = '' + let reasoningContent = '' + let hasReasoningContent = false for await (const chunk of completion) { if ('usage' in chunk && chunk.usage) { tokenUsage = openAICompletionsUsageToChatTokenUsage(chunk.usage) @@ -968,9 +976,11 @@ export async function parseOpenAICompletion( continue } const c = chunk as ChatCompletionChunk + const choice = c.choices[0] + const delta = choice.delta // Check for malformed function call error (e.g. from Gemini models) - const finishReason = c.choices[0].finish_reason + const finishReason = choice.finish_reason if ( finishReason && typeof finishReason === 'string' && @@ -979,12 +989,19 @@ export async function parseOpenAICompletion( malformedFunctionCallError = true } - const delta = c.choices[0].delta.content - if (delta) { - answer += delta - callbacks.onNewToken(delta) + const reasoningDelta = getReasoningContentDelta(delta) + if (typeof reasoningDelta === 'string') { + hasReasoningContent = true + reasoningContent += reasoningDelta } - const toolCalls = c.choices[0].delta.tool_calls || [] + + const contentDelta = delta.content + if (contentDelta) { + answer += contentDelta + assistantContent += contentDelta + callbacks.onNewToken(contentDelta) + } + const toolCalls = delta.tool_calls || [] if (toolCalls.length > 0 && answer) { // if tool calls are present but we have some textual content already, we need to display it to the user first callbacks.onMessageEnd() @@ -1059,8 +1076,12 @@ export async function parseOpenAICompletion( } } - if (answer) { - const toAdd = { role: 'assistant' as const, content: answer } + const toolCalls = Object.values(finalToolCalls).filter( + (toolCall) => toolCall.id !== undefined && toolCall.function?.arguments !== undefined + ) as ChatCompletionMessageFunctionToolCall[] + + if (answer && toolCalls.length === 0) { + const toAdd = buildAssistantTextMessage(answer) addedMessages.push(toAdd) messages.push(toAdd) } @@ -1074,21 +1095,22 @@ export async function parseOpenAICompletion( } } - const toolCalls = Object.values(finalToolCalls).filter( - (toolCall) => toolCall.id !== undefined && toolCall.function?.arguments !== undefined - ) as ChatCompletionMessageFunctionToolCall[] - if (toolCalls.length > 0) { - const toAdd = { - role: 'assistant' as const, - tool_calls: toolCalls.map((t) => ({ - ...t, - function: { - ...t.function, - arguments: t.function.arguments || '{}' - } - })) - } + const normalizedToolCalls = toolCalls.map((t) => ({ + ...t, + function: { + ...t.function, + arguments: t.function.arguments || '{}' + } + })) + const toAdd = buildAssistantToolCallMessage({ + content: assistantContent, + reasoning: { + hasReasoningContent, + reasoningContent + }, + toolCalls: normalizedToolCalls + }) messages.push(toAdd) addedMessages.push(toAdd) for (const toolCall of toolCalls) {