From 2553fbfe31417bd985e7994eac695bf918f97ce2 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Thu, 28 May 2026 18:52:26 +0200 Subject: [PATCH] feat: add deepseek fim support (#9365) --- backend/windmill-ai/src/ai_providers.rs | 3 +- backend/windmill-ai/src/proxy/fim.rs | 116 ++++++++++++++++-- backend/windmill-api/src/ai.rs | 24 +++- .../copilot/autocomplete/request.ts | 4 +- frontend/src/lib/components/copilot/fim.ts | 39 ++++++ .../src/lib/components/copilot/lib.test.ts | 44 +++++++ frontend/src/lib/components/copilot/lib.ts | 23 +--- frontend/src/lib/components/copilot/utils.ts | 5 +- .../workspaceSettings/AISettings.svelte | 5 +- 9 files changed, 219 insertions(+), 44 deletions(-) create mode 100644 frontend/src/lib/components/copilot/fim.ts diff --git a/backend/windmill-ai/src/ai_providers.rs b/backend/windmill-ai/src/ai_providers.rs index b568c0accb..fb29454ca5 100644 --- a/backend/windmill-ai/src/ai_providers.rs +++ b/backend/windmill-ai/src/ai_providers.rs @@ -27,6 +27,7 @@ lazy_static::lazy_static! { } pub const OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; +pub const DEEPSEEK_BASE_URL: &str = "https://api.deepseek.com/v1"; pub const GOOGLE_AI_BASE_URL: &str = "https://generativelanguage.googleapis.com/v1beta"; /// Empty string signals BedrockClient::from_env() to use the region from AWS environment/config @@ -106,7 +107,7 @@ impl AIProvider { Ok(azure_base_path.unwrap_or("https://api.openai.com/v1".to_string())) } - AIProvider::DeepSeek => Ok("https://api.deepseek.com/v1".to_string()), + AIProvider::DeepSeek => Ok(DEEPSEEK_BASE_URL.to_string()), AIProvider::GoogleAI => Ok(GOOGLE_AI_BASE_URL.to_string()), AIProvider::Groq => Ok("https://api.groq.com/openai/v1".to_string()), AIProvider::OpenRouter => Ok("https://openrouter.ai/api/v1".to_string()), diff --git a/backend/windmill-ai/src/proxy/fim.rs b/backend/windmill-ai/src/proxy/fim.rs index 2d14fd23ce..3645476441 100644 --- a/backend/windmill-ai/src/proxy/fim.rs +++ b/backend/windmill-ai/src/proxy/fim.rs @@ -3,12 +3,13 @@ use serde::Deserialize; use serde_json::json; use windmill_common::error::{Error, Result}; -use crate::ai_providers::AIProvider; +use crate::ai_providers::{AIProvider, DEEPSEEK_BASE_URL}; #[derive(Debug, Eq, PartialEq)] pub struct FimProxyTransform { pub body: Bytes, pub path: String, + pub base_url: Option, } #[derive(Deserialize)] @@ -22,19 +23,49 @@ struct FimRequest { } pub fn supports_native_fim(provider: &AIProvider) -> bool { - matches!(provider, AIProvider::Mistral) + matches!(provider, AIProvider::Mistral | AIProvider::DeepSeek) +} + +fn deepseek_fim_base_url(base_url: &str) -> String { + let trimmed = base_url.trim_end_matches('/'); + let deepseek_root_base_url = DEEPSEEK_BASE_URL + .strip_suffix("/v1") + .unwrap_or(DEEPSEEK_BASE_URL); + + if trimmed == DEEPSEEK_BASE_URL || trimmed == deepseek_root_base_url { + return format!("{deepseek_root_base_url}/beta"); + } + + if let Some(prefix) = trimmed.strip_suffix("/v1") { + return format!("{prefix}/beta"); + } + + trimmed.to_string() } pub fn maybe_transform_fim_request( provider: &AIProvider, path: &str, + base_url: &str, body: &[u8], ) -> Result> { - if path.contains("fim/completions") && !supports_native_fim(provider) { - transform_fim_to_chat_completions(body).map(Some) - } else { - Ok(None) + if !path.contains("fim/completions") { + return Ok(None); } + + if matches!(provider, AIProvider::DeepSeek) { + return Ok(Some(FimProxyTransform { + body: Bytes::copy_from_slice(body), + path: "completions".to_string(), + base_url: Some(deepseek_fim_base_url(base_url)), + })); + } + + if !supports_native_fim(provider) { + return transform_fim_to_chat_completions(body).map(Some); + } + + Ok(None) } fn transform_fim_to_chat_completions(body: &[u8]) -> Result { @@ -64,7 +95,11 @@ fn transform_fim_to_chat_completions(body: &[u8]) -> Result { let body = serde_json::to_vec(&chat_req) .map_err(|e| Error::internal_err(format!("Failed to serialize chat request: {}", e)))?; - Ok(FimProxyTransform { body: Bytes::from(body), path: "chat/completions".to_string() }) + Ok(FimProxyTransform { + body: Bytes::from(body), + path: "chat/completions".to_string(), + base_url: None, + }) } #[cfg(test)] @@ -73,11 +108,62 @@ mod tests { #[test] fn mistral_keeps_native_fim_request() { - let transformed = - maybe_transform_fim_request(&AIProvider::Mistral, "fim/completions", br#"{}"#).unwrap(); + let transformed = maybe_transform_fim_request( + &AIProvider::Mistral, + "fim/completions", + "https://api.mistral.ai/v1", + br#"{}"#, + ) + .unwrap(); assert!(transformed.is_none()); assert!(supports_native_fim(&AIProvider::Mistral)); + assert!(supports_native_fim(&AIProvider::DeepSeek)); + assert!(!supports_native_fim(&AIProvider::OpenAI)); + } + + #[test] + fn deepseek_fim_base_url_uses_beta_endpoint() { + assert_eq!( + deepseek_fim_base_url("https://api.deepseek.com/v1"), + "https://api.deepseek.com/beta" + ); + assert_eq!( + deepseek_fim_base_url("https://api.deepseek.com/v1/"), + "https://api.deepseek.com/beta" + ); + assert_eq!( + deepseek_fim_base_url("https://api.deepseek.com"), + "https://api.deepseek.com/beta" + ); + assert_eq!( + deepseek_fim_base_url("https://proxy.example/deepseek/v1"), + "https://proxy.example/deepseek/beta" + ); + assert_eq!( + deepseek_fim_base_url("https://proxy.example/deepseek/beta"), + "https://proxy.example/deepseek/beta" + ); + } + + #[test] + fn deepseek_fim_request_uses_beta_completions_endpoint() { + let body = br#"{"model":"deepseek-v4-pro","prompt":"return ","suffix":";"}"#; + let transformed = maybe_transform_fim_request( + &AIProvider::DeepSeek, + "fim/completions", + DEEPSEEK_BASE_URL, + body, + ) + .unwrap() + .expect("DeepSeek FIM should be routed to the beta completions endpoint"); + + assert_eq!(transformed.path, "completions"); + assert_eq!( + transformed.base_url.as_deref(), + Some("https://api.deepseek.com/beta") + ); + assert_eq!(transformed.body, Bytes::copy_from_slice(body)); } #[test] @@ -85,6 +171,7 @@ mod tests { let transformed = maybe_transform_fim_request( &AIProvider::OpenAI, "fim/completions", + "https://api.openai.com/v1", br#"{ "model": "gpt-4.1", "prompt": "fn main() {", @@ -96,6 +183,7 @@ mod tests { .expect("OpenAI FIM should be transformed"); assert_eq!(transformed.path, "chat/completions"); + assert_eq!(transformed.base_url, None); let body: serde_json::Value = serde_json::from_slice(&transformed.body).unwrap(); assert_eq!(body["model"], "gpt-4.1"); @@ -111,9 +199,13 @@ mod tests { #[test] fn invalid_fim_body_is_bad_request() { - let err = - maybe_transform_fim_request(&AIProvider::OpenAI, "fim/completions", br#"{"model": 1}"#) - .unwrap_err(); + let err = maybe_transform_fim_request( + &AIProvider::OpenAI, + "fim/completions", + "https://api.openai.com/v1", + br#"{"model": 1}"#, + ) + .unwrap_err(); assert!(matches!(err, Error::BadRequest(_))); } diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 81f601e53f..21059df18a 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -627,7 +627,7 @@ async fn proxy( check_scopes(&authed, || format!("resources:read:{}", resource_path))?; } - let credentials = match workspace_cache { + let mut credentials = match workspace_cache { Some(request_cache) if !request_cache.is_expired() && forced_resource_path.is_none() => { request_cache.credentials } @@ -758,11 +758,23 @@ async fn proxy( } }; - if let Some(fim_transform) = maybe_transform_fim_request(&provider, &ai_path, &body)? { - tracing::debug!( - "Transforming FIM request to chat/completions with FIM tokens for provider {:?}", - provider - ); + if let Some(fim_transform) = + maybe_transform_fim_request(&provider, &ai_path, &credentials.base_url, &body)? + { + if fim_transform.base_url.is_some() { + tracing::debug!( + "Routing native FIM request through provider-specific endpoint for {:?}", + provider + ); + } else { + tracing::debug!( + "Transforming FIM request to chat/completions with FIM tokens for provider {:?}", + provider + ); + } + if let Some(base_url) = fim_transform.base_url { + credentials.base_url = base_url; + } body = fim_transform.body; ai_path = fim_transform.path; } diff --git a/frontend/src/lib/components/copilot/autocomplete/request.ts b/frontend/src/lib/components/copilot/autocomplete/request.ts index ef29ba9a19..cc3987ebe5 100644 --- a/frontend/src/lib/components/copilot/autocomplete/request.ts +++ b/frontend/src/lib/components/copilot/autocomplete/request.ts @@ -30,9 +30,9 @@ export async function autocompleteRequest( throw new Error('No code completion model selected') } - // Only add context lines for Mistral (native FIM) - other providers use chat completion + // Only add context lines for native FIM providers - other providers use chat completion // too much context degrades significantly the quality of the completion - if (providerModel.provider === 'mistral') { + if (providerModel.provider === 'mistral' || providerModel.provider === 'deepseek') { let commentSymbol = getCommentSymbol(context.scriptLang) let contextLines = comment( commentSymbol, diff --git a/frontend/src/lib/components/copilot/fim.ts b/frontend/src/lib/components/copilot/fim.ts new file mode 100644 index 0000000000..16c5d40431 --- /dev/null +++ b/frontend/src/lib/components/copilot/fim.ts @@ -0,0 +1,39 @@ +import type { AIProvider } from '$lib/gen' +import { z } from 'zod' + +const chatFimResponseSchema = z.object({ + choices: z.array( + z.object({ + message: z.object({ + content: z.string().optional() + }), + finish_reason: z.string().optional() + }) + ) +}) + +const deepseekFimResponseSchema = z.object({ + choices: z.array( + z.object({ + text: z.string().optional(), + finish_reason: z.string().optional() + }) + ) +}) + +export function parseFimCompletionChoice( + body: unknown, + provider: AIProvider +): { content: string | undefined; finish_reason: string | undefined } | undefined { + if (provider === 'deepseek') { + const parsedBody = deepseekFimResponseSchema.parse(body) + const choice = parsedBody.choices[0] + return choice ? { content: choice.text, finish_reason: choice.finish_reason } : undefined + } + + const parsedBody = chatFimResponseSchema.parse(body) + const choice = parsedBody.choices[0] + return choice + ? { content: choice.message.content, finish_reason: choice.finish_reason } + : undefined +} diff --git a/frontend/src/lib/components/copilot/lib.test.ts b/frontend/src/lib/components/copilot/lib.test.ts index f1eec17cbb..369987d53f 100644 --- a/frontend/src/lib/components/copilot/lib.test.ts +++ b/frontend/src/lib/components/copilot/lib.test.ts @@ -9,7 +9,9 @@ import { buildAssistantToolCallMessage, getReasoningContentDelta } from './chat/openaiReasoning' +import { parseFimCompletionChoice } from './fim' import { getDefaultChatTemperature, modelDisallowsSamplingParams } from './modelConfig' +import { supportsAutocomplete } from './utils' type AssistantMessageWithReasoning = ChatCompletionMessageParam & { role: 'assistant' @@ -43,6 +45,48 @@ describe('modelConfig', () => { }) }) +describe('fim autocomplete', () => { + it('allows DeepSeek v4 pro and Codestral autocomplete models', () => { + expect(supportsAutocomplete('codestral-latest')).toBe(true) + expect(supportsAutocomplete('Codestral-2501')).toBe(true) + expect(supportsAutocomplete('codestral-embed')).toBe(false) + expect(supportsAutocomplete('deepseek-v4-pro')).toBe(true) + expect(supportsAutocomplete('deepseek-chat')).toBe(false) + }) + + it('parses chat-shaped native FIM responses', () => { + expect( + parseFimCompletionChoice( + { + choices: [ + { + message: { content: 'cache[key] = factory()' }, + finish_reason: 'stop' + } + ] + }, + 'mistral' + ) + ).toEqual({ content: 'cache[key] = factory()', finish_reason: 'stop' }) + }) + + it('parses DeepSeek native FIM completion responses', () => { + expect( + parseFimCompletionChoice( + { + choices: [ + { + text: 'items?.length ?? 0', + finish_reason: 'stop' + } + ] + }, + 'deepseek' + ) + ).toEqual({ content: 'items?.length ?? 0', finish_reason: 'stop' }) + }) +}) + describe('openaiReasoning', () => { it('reads provider-specific reasoning_content deltas', () => { expect( diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index de87579a23..00d2334acc 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -16,7 +16,6 @@ import { OpenAPI, ResourceService, type Script } from '../../gen' import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' import { getDefaultChatTemperature } from './modelConfig' import { formatResourceTypes } from './utils' -import { z } from 'zod' import { processToolCall, type Tool, type ToolCallbacks } from './chat/shared' import { getNonStreamingOpenAIResponsesCompletion, @@ -36,6 +35,7 @@ import { buildAssistantToolCallMessage, getReasoningContentDelta } from './chat/openaiReasoning' +import { parseFimCompletionChoice } from './fim' export const SUPPORTED_LANGUAGES = new Set(Object.keys(GEN_CONFIG.prompts)) @@ -74,7 +74,7 @@ export const AI_PROVIDERS: Record = { }, deepseek: { label: 'DeepSeek', - defaultModels: ['deepseek-chat', 'deepseek-reasoner'] + defaultModels: ['deepseek-v4-pro', 'deepseek-chat', 'deepseek-reasoner'] }, googleai: { label: 'Google AI', @@ -816,17 +816,6 @@ export async function getNonStreamingCompletion( return response } -const mistralFimResponseSchema = z.object({ - choices: z.array( - z.object({ - message: z.object({ - content: z.string().optional() - }), - finish_reason: z.string() - }) - ) -}) - export const FIM_MAX_TOKENS = 256 const FIM_MAX_LINES = 8 export async function getFimCompletion( @@ -864,12 +853,10 @@ export async function getFimCompletion( ) const body = await response.json() - const parsedBody = mistralFimResponseSchema.parse(body) + const choice = parseFimCompletionChoice(body, providerModel.provider) - const choice = parsedBody.choices[0] - - if (choice && choice.message.content !== undefined) { - let lines = choice.message.content.split('\n') + if (choice?.content !== undefined) { + let lines = choice.content.split('\n') // If finish_reason is 'length', remove the last line if (choice.finish_reason === 'length') { diff --git a/frontend/src/lib/components/copilot/utils.ts b/frontend/src/lib/components/copilot/utils.ts index 8e5150cda6..50f6513b26 100644 --- a/frontend/src/lib/components/copilot/utils.ts +++ b/frontend/src/lib/components/copilot/utils.ts @@ -171,10 +171,9 @@ export function yamlStringifyExceptKeys(obj: any, keys: string[]) { /** * Checks if a model supports FIM (Fill-in-the-Middle) autocomplete. - * Currently only Codestral models (non-embedding) support this. + * Currently Codestral models (non-embedding) and DeepSeek FIM support this. */ export function supportsAutocomplete(model: string): boolean { const lower = model.toLowerCase() - return lower.includes('codestral') && !lower.includes('embed') + return (lower.includes('codestral') && !lower.includes('embed')) || lower === 'deepseek-v4-pro' } - diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index dd3cc6e0d0..922492d6ac 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -353,7 +353,7 @@ {#if showWorkspaceOverrideEditor}
- {#each Object.entries(AI_PROVIDERS) as [provider, details]} + {#each Object.entries(AI_PROVIDERS) as [provider, details] (provider)}