starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-12 00:06:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: preserve ai reasoning content (#9208)

Browse Source 
* fix: preserve ai reasoning content

* fix: avoid text-only reasoning replay

* feat: add deepseek ai eval models
This commit is contained in:
centdix
2026-05-18 10:40:18 +00:00
committed by GitHub
parent bd32c5f951
commit fec4008696
10 changed files with 269 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts
+2 -1
View File
@@ -236,7 +236,8 @@ function toFrontendEvalProvider(
if (
provider === "anthropic" ||
provider === "openai" ||
provider === "googleai"
provider === "googleai" ||
provider === "deepseek"
) {
return provider;
}
ai_evals/adapters/frontend/core/shared/providerConfig.test.ts
+7
View File
@@ -27,6 +27,13 @@ describe("resolveEvalModelProvider", () => {
});
});
it("infers deepseek from DeepSeek model ids", () => {
expect(resolveEvalModelProvider("deepseek-v4-flash")).toEqual({
provider: "deepseek",
model: "deepseek-v4-flash",
});
});
it("preserves an explicit provider", () => {
expect(resolveEvalModelProvider("gemini-2.5-pro", "googleai")).toEqual({
provider: "googleai",
ai_evals/adapters/frontend/core/shared/providerConfig.ts
+3
View File
@@ -83,6 +83,9 @@ export function resolveEvalModelProvider(
if (model.startsWith("gemini")) {
return { provider: "googleai", model };
}
if (model.startsWith("deepseek")) {
return { provider: "deepseek", model };
}
if (model.startsWith("gpt") || model.startsWith("o")) {
return { provider: "openai", model };
}
ai_evals/core/models.test.ts
+20 -5
View File
@@ -11,19 +11,34 @@ describe("resolveEvalModel", () => {
provider: "googleai",
model: "gemini-2.5-pro",
});
expect(resolveEvalModel("script", "gemini-3-flash-preview").frontend).toEqual({
expect(
resolveEvalModel("script", "gemini-3-flash-preview").frontend,
).toEqual({
provider: "googleai",
model: "gemini-3-flash-preview",
});
expect(resolveEvalModel("flow", "gemini-3.1-pro-preview").frontend).toEqual({
provider: "googleai",
model: "gemini-3.1-pro-preview",
expect(resolveEvalModel("flow", "gemini-3.1-pro-preview").frontend).toEqual(
{
provider: "googleai",
model: "gemini-3.1-pro-preview",
},
);
});
it("supports DeepSeek aliases for frontend evals", () => {
expect(resolveEvalModel("flow", "deepseek").frontend).toEqual({
provider: "deepseek",
model: "deepseek-v4-flash",
});
expect(resolveEvalModel("script", "deepseek-v4-pro").frontend).toEqual({
provider: "deepseek",
model: "deepseek-v4-pro",
});
});
it("rejects Gemini aliases for cli evals", () => {
expect(() => resolveEvalModel("cli", "gemini")).toThrow(
"Model gemini-flash is not supported for cli mode"
"Model gemini-flash is not supported for cli mode",
);
});
});
ai_evals/core/models.ts
+38 -6
View File
@@ -1,7 +1,7 @@
import type { EvalMode } from "./types";
export interface FrontendEvalModelConfig {
provider: "anthropic" | "openai" | "googleai";
provider: "anthropic" | "openai" | "googleai" | "deepseek";
model: string;
}
@@ -117,15 +117,40 @@ export const EVAL_MODELS: EvalModelSpec[] = [
{
id: "gemini-3.1-pro-preview",
label: "Gemini 3.1 Pro Preview",
aliases: ["gemini-3.1-pro-preview", "gemini-3.1-pro", "gemini-3-pro-preview"],
aliases: [
"gemini-3.1-pro-preview",
"gemini-3.1-pro",
"gemini-3-pro-preview",
],
frontend: {
provider: "googleai",
model: "gemini-3.1-pro-preview",
},
},
{
id: "deepseek-v4-flash",
label: "DeepSeek V4 Flash",
aliases: ["deepseek", "deepseek-v4", "deepseek-v4-flash"],
frontend: {
provider: "deepseek",
model: "deepseek-v4-flash",
},
},
{
id: "deepseek-v4-pro",
label: "DeepSeek V4 Pro",
aliases: ["deepseek-pro", "deepseek-v4-pro"],
frontend: {
provider: "deepseek",
model: "deepseek-v4-pro",
},
},
];
export function resolveEvalModel(mode: EvalMode, alias?: string): EvalModelSpec {
export function resolveEvalModel(
mode: EvalMode,
alias?: string,
): EvalModelSpec {
const spec = alias ? findEvalModel(alias) : getDefaultEvalModel(mode);
if (!spec) {
throw new Error(`Unknown model: ${alias}`);
@@ -152,14 +177,19 @@ export function getEvalModelHelpText(): string {
}).join("\n");
}
export function formatRunModelLabel(mode: EvalMode, model: EvalModelSpec): string {
export function formatRunModelLabel(
mode: EvalMode,
model: EvalModelSpec,
): string {
if (mode === "cli") {
return `${model.cli!.provider}:${model.cli!.model}`;
}
return `${model.frontend!.provider}:${model.frontend!.model}`;
}
export function getFrontendEvalModel(model: EvalModelSpec): FrontendEvalModelConfig {
export function getFrontendEvalModel(
model: EvalModelSpec,
): FrontendEvalModelConfig {
if (!model.frontend) {
throw new Error(`Model ${model.id} does not support frontend evals`);
}
@@ -180,6 +210,8 @@ function getDefaultEvalModel(mode: EvalMode): EvalModelSpec {
function findEvalModel(alias: string): EvalModelSpec | undefined {
const normalized = alias.trim().toLowerCase();
return EVAL_MODELS.find((model) =>
[model.id, ...model.aliases].some((candidate) => candidate.toLowerCase() === normalized)
[model.id, ...model.aliases].some(
(candidate) => candidate.toLowerCase() === normalized,
),
);
}
ai_evals/modes/frontendCommon.test.ts
+8 -1
View File
@@ -5,12 +5,14 @@ const ORIGINAL_ENV = {
ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
OPENAI_API_KEY: process.env.OPENAI_API_KEY,
GEMINI_API_KEY: process.env.GEMINI_API_KEY,
DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY,
};
afterEach(() => {
process.env.ANTHROPIC_API_KEY = ORIGINAL_ENV.ANTHROPIC_API_KEY;
process.env.OPENAI_API_KEY = ORIGINAL_ENV.OPENAI_API_KEY;
process.env.GEMINI_API_KEY = ORIGINAL_ENV.GEMINI_API_KEY;
process.env.DEEPSEEK_API_KEY = ORIGINAL_ENV.DEEPSEEK_API_KEY;
});
describe("getFrontendApiKey", () => {
@@ -19,10 +21,15 @@ describe("getFrontendApiKey", () => {
expect(getFrontendApiKey("googleai")).toBe("gemini-test-key");
});
it("reads the DeepSeek API key for deepseek models", () => {
process.env.DEEPSEEK_API_KEY = "deepseek-test-key";
expect(getFrontendApiKey("deepseek")).toBe("deepseek-test-key");
});
it("throws a provider-specific error when the key is missing", () => {
delete process.env.GEMINI_API_KEY;
expect(() => getFrontendApiKey("googleai")).toThrow(
"GEMINI_API_KEY is required for frontend evals"
"GEMINI_API_KEY is required for frontend evals",
);
});
});
ai_evals/modes/frontendCommon.ts
+6 -2
View File
@@ -1,12 +1,16 @@
import type { FrontendEvalModelConfig } from "../core/models";
export function getFrontendApiKey(provider: FrontendEvalModelConfig["provider"]): string {
export function getFrontendApiKey(
provider: FrontendEvalModelConfig["provider"],
): string {
const envName =
provider === "anthropic"
? "ANTHROPIC_API_KEY"
: provider === "googleai"
? "GEMINI_API_KEY"
: "OPENAI_API_KEY";
: provider === "deepseek"
? "DEEPSEEK_API_KEY"
: "OPENAI_API_KEY";
const apiKey = process.env[envName];
if (!apiKey) {
throw new Error(`${envName} is required for frontend evals`);
frontend/src/lib/components/copilot/chat/openaiReasoning.ts
+51
View File
@@ -0,0 +1,51 @@
import type {
ChatCompletionChunk,
ChatCompletionMessageFunctionToolCall,
ChatCompletionMessageParam
} from 'openai/resources/index.mjs'
type ChatCompletionDeltaWithReasoning = ChatCompletionChunk.Choice.Delta & {
reasoning_content?: string | null
}
type ChatCompletionAssistantMessageWithReasoning = ChatCompletionMessageParam & {
role: 'assistant'
reasoning_content?: string
}
export type ReasoningContentState = {
hasReasoningContent: boolean
reasoningContent: string
}
export function getReasoningContentDelta(
delta: ChatCompletionChunk.Choice.Delta
): string | null | undefined {
return (delta as ChatCompletionDeltaWithReasoning).reasoning_content
}
export function buildAssistantTextMessage(
content: string
): ChatCompletionAssistantMessageWithReasoning {
return {
role: 'assistant',
content
}
}
export function buildAssistantToolCallMessage({
content,
reasoning,
toolCalls
}: {
content: string
reasoning: ReasoningContentState
toolCalls: ChatCompletionMessageFunctionToolCall[]
}): ChatCompletionAssistantMessageWithReasoning {
return {
role: 'assistant',
...(content || reasoning.hasReasoningContent ? { content } : {}),
...(reasoning.hasReasoningContent ? { reasoning_content: reasoning.reasoningContent } : {}),
tool_calls: toolCalls
}
}
frontend/src/lib/components/copilot/lib.test.ts
+90
View File
@@ -1,6 +1,23 @@
import type {
ChatCompletionChunk,
ChatCompletionMessageFunctionToolCall,
ChatCompletionMessageParam
} from 'openai/resources/index.mjs'
import { describe, expect, it } from 'vitest'
import {
buildAssistantTextMessage,
buildAssistantToolCallMessage,
getReasoningContentDelta
} from './chat/openaiReasoning'
import { getDefaultChatTemperature, modelDisallowsSamplingParams } from './modelConfig'
type AssistantMessageWithReasoning = ChatCompletionMessageParam & {
role: 'assistant'
content?: string
reasoning_content?: string
tool_calls?: ChatCompletionMessageFunctionToolCall[]
}
describe('modelConfig', () => {
it('flags Opus 4.7 model IDs via includes matching', () => {
expect(modelDisallowsSamplingParams('claude-opus-4-7')).toBe(true)
@@ -25,3 +42,76 @@ describe('modelConfig', () => {
expect(getDefaultChatTemperature({ provider: 'anthropic', model: 'claude-sonnet-4-6' })).toBe(0)
})
})
describe('openaiReasoning', () => {
it('reads provider-specific reasoning_content deltas', () => {
expect(
getReasoningContentDelta({
reasoning_content: 'thinking'
} as ChatCompletionChunk.Choice.Delta & { reasoning_content: string })
).toBe('thinking')
})
it('preserves DeepSeek reasoning_content on assistant tool-call messages', () => {
const toolCalls: ChatCompletionMessageFunctionToolCall[] = [
{
id: 'call_1',
type: 'function',
function: {
name: 'lookup',
arguments: '{"query":"docs"}'
}
}
]
const assistantMessage = buildAssistantToolCallMessage({
content: 'I will look that up.',
reasoning: {
hasReasoningContent: true,
reasoningContent: 'First, I need a lookup.'
},
toolCalls
}) as AssistantMessageWithReasoning
expect(assistantMessage).toMatchObject({
role: 'assistant',
content: 'I will look that up.',
reasoning_content: 'First, I need a lookup.',
tool_calls: [
{
id: 'call_1',
type: 'function',
function: {
name: 'lookup',
arguments: '{"query":"docs"}'
}
}
]
})
})
it('does not preserve reasoning_content on text-only assistant messages', () => {
expect(buildAssistantTextMessage('done')).toEqual({
role: 'assistant',
content: 'done'
})
})
it('keeps empty reasoning_content when the provider emitted the field', () => {
const assistantMessage = buildAssistantToolCallMessage({
content: '',
reasoning: {
hasReasoningContent: true,
reasoningContent: ''
},
toolCalls: []
}) as AssistantMessageWithReasoning
expect(assistantMessage).toMatchObject({
role: 'assistant',
content: '',
reasoning_content: '',
tool_calls: []
})
})
})
frontend/src/lib/components/copilot/lib.ts
+44 -22
View File
@@ -31,6 +31,11 @@ import {
openAICompletionsUsageToChatTokenUsage,
type ChatTokenUsage
} from './chat/tokenUsage'
import {
buildAssistantTextMessage,
buildAssistantToolCallMessage,
getReasoningContentDelta
} from './chat/openaiReasoning'
export const SUPPORTED_LANGUAGES = new Set(Object.keys(GEN_CONFIG.prompts))
@@ -960,6 +965,9 @@ export async function parseOpenAICompletion(
let tokenUsage = emptyChatTokenUsage()
let answer = ''
let assistantContent = ''
let reasoningContent = ''
let hasReasoningContent = false
for await (const chunk of completion) {
if ('usage' in chunk && chunk.usage) {
tokenUsage = openAICompletionsUsageToChatTokenUsage(chunk.usage)
@@ -968,9 +976,11 @@ export async function parseOpenAICompletion(
continue
}
const c = chunk as ChatCompletionChunk
const choice = c.choices[0]
const delta = choice.delta
// Check for malformed function call error (e.g. from Gemini models)
const finishReason = c.choices[0].finish_reason
const finishReason = choice.finish_reason
if (
finishReason &&
typeof finishReason === 'string' &&
@@ -979,12 +989,19 @@ export async function parseOpenAICompletion(
malformedFunctionCallError = true
}
const delta = c.choices[0].delta.content
if (delta) {
answer += delta
callbacks.onNewToken(delta)
const reasoningDelta = getReasoningContentDelta(delta)
if (typeof reasoningDelta === 'string') {
hasReasoningContent = true
reasoningContent += reasoningDelta
}
const toolCalls = c.choices[0].delta.tool_calls || []
const contentDelta = delta.content
if (contentDelta) {
answer += contentDelta
assistantContent += contentDelta
callbacks.onNewToken(contentDelta)
}
const toolCalls = delta.tool_calls || []
if (toolCalls.length > 0 && answer) {
// if tool calls are present but we have some textual content already, we need to display it to the user first
callbacks.onMessageEnd()
@@ -1059,8 +1076,12 @@ export async function parseOpenAICompletion(
}
}
if (answer) {
const toAdd = { role: 'assistant' as const, content: answer }
const toolCalls = Object.values(finalToolCalls).filter(
(toolCall) => toolCall.id !== undefined && toolCall.function?.arguments !== undefined
) as ChatCompletionMessageFunctionToolCall[]
if (answer && toolCalls.length === 0) {
const toAdd = buildAssistantTextMessage(answer)
addedMessages.push(toAdd)
messages.push(toAdd)
}
@@ -1074,21 +1095,22 @@ export async function parseOpenAICompletion(
}
}
const toolCalls = Object.values(finalToolCalls).filter(
(toolCall) => toolCall.id !== undefined && toolCall.function?.arguments !== undefined
) as ChatCompletionMessageFunctionToolCall[]
if (toolCalls.length > 0) {
const toAdd = {
role: 'assistant' as const,
tool_calls: toolCalls.map((t) => ({
...t,
function: {
...t.function,
arguments: t.function.arguments || '{}'
}
}))
}
const normalizedToolCalls = toolCalls.map((t) => ({
...t,
function: {
...t.function,
arguments: t.function.arguments || '{}'
}
}))
const toAdd = buildAssistantToolCallMessage({
content: assistantContent,
reasoning: {
hasReasoningContent,
reasoningContent
},
toolCalls: normalizedToolCalls
})
messages.push(toAdd)
addedMessages.push(toAdd)
for (const toolCall of toolCalls) {