diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 866b29404f..a8d7a9e122 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -6,7 +6,7 @@ use http::{HeaderMap, Method}; use quick_cache::sync::Cache; use reqwest::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; +use serde_json::{json, value::RawValue}; use std::collections::HashMap; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::ai_providers::{AIProvider, ProviderConfig, ProviderModel}; @@ -438,6 +438,53 @@ pub struct AIConfig { pub max_tokens_per_model: Option>, } +// FIM (Fill-in-the-Middle) simulation for providers that don't support native FIM +#[derive(Deserialize, Debug)] +struct FimRequest { + model: String, + prompt: String, // code before cursor + suffix: Option, // code after cursor + temperature: Option, + max_tokens: Option, + stop: Option>, +} + +/// Checks if the AI provider supports native FIM (Fill-in-the-Middle) endpoint +fn supports_native_fim(provider: &AIProvider) -> bool { + matches!(provider, AIProvider::Mistral) +} + +/// Transforms a FIM request to chat/completions format for providers that don't support native FIM. +fn transform_fim_to_chat_completions(body: &Bytes) -> Result<(Bytes, String)> { + let fim_req: FimRequest = serde_json::from_slice(body) + .map_err(|e| Error::internal_err(format!("Failed to parse FIM request: {}", e)))?; + + let suffix = fim_req.suffix.unwrap_or_default(); + + let system_prompt = "You are a code completion assistant. Complete the code at the position between the given prefix and suffix. Output ONLY the code that goes at the cursor - no explanations, no markdown, no repeating the prefix or suffix."; + + let user_content = format!( + "\n{}\n\n\n{}", + fim_req.prompt, suffix + ); + + let chat_req = json!({ + "model": fim_req.model, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_content} + ], + "temperature": fim_req.temperature.unwrap_or(0.0), + "max_tokens": fim_req.max_tokens.unwrap_or(256), + "stop": fim_req.stop + }); + + let chat_body = serde_json::to_vec(&chat_req) + .map_err(|e| Error::internal_err(format!("Failed to serialize chat request: {}", e)))?; + + Ok((Bytes::from(chat_body), "chat/completions".to_string())) +} + pub fn global_service() -> Router { Router::new().route("/proxy/*ai", post(global_proxy).get(global_proxy)) } @@ -516,10 +563,10 @@ async fn global_proxy( async fn proxy( authed: ApiAuthed, Extension(db): Extension, - Path((w_id, ai_path)): Path<(String, String)>, + Path((w_id, mut ai_path)): Path<(String, String)>, method: Method, headers: HeaderMap, - body: Bytes, + mut body: Bytes, ) -> impl IntoResponse { let provider = headers .get("X-Provider") @@ -600,6 +647,19 @@ async fn proxy( } }; + // Check if this is a FIM request to a provider that doesn't support native FIM endpoint + // For such providers, transform to use FIM sentinel tokens with the chat/completions endpoint + let is_fim_request = ai_path.contains("fim/completions"); + if is_fim_request && !supports_native_fim(&provider) { + tracing::debug!( + "Transforming FIM request to chat/completions with FIM tokens for provider {:?}", + provider + ); + let (chat_body, chat_path) = transform_fim_to_chat_completions(&body)?; + body = chat_body; + ai_path = chat_path; + } + // Extract model and streaming flag for Bedrock transformation (only for POST requests) let (model, is_streaming) = if matches!(provider, AIProvider::AWSBedrock) && method == Method::POST { diff --git a/frontend/src/lib/components/copilot/autocomplete/Autocompletor.ts b/frontend/src/lib/components/copilot/autocomplete/Autocompletor.ts index c25bff9bc5..8f62956952 100644 --- a/frontend/src/lib/components/copilot/autocomplete/Autocompletor.ts +++ b/frontend/src/lib/components/copilot/autocomplete/Autocompletor.ts @@ -5,6 +5,7 @@ import { LRUCache } from 'lru-cache' import { autocompleteRequest } from './request' import { FIM_MAX_TOKENS, getModelContextWindow } from '../lib' import { setGlobalCSS } from '../shared' +import { supportsAutocomplete } from '../utils' import { get } from 'svelte/store' import type { MonacoLanguageClient } from 'monaco-languageclient' import { copilotInfo } from '$lib/aiStore' @@ -202,12 +203,8 @@ export class Autocompletor { } static isProviderModelSupported(providerModel: AIProviderModel | undefined) { - return ( - providerModel && - providerModel.provider === 'mistral' && - providerModel.model.startsWith('codestral-') && - !providerModel.model.startsWith('codestral-embed') - ) + if (!providerModel) return false + return supportsAutocomplete(providerModel.model) } dispose() { diff --git a/frontend/src/lib/components/copilot/autocomplete/request.ts b/frontend/src/lib/components/copilot/autocomplete/request.ts index 8589213483..51fe0cf720 100644 --- a/frontend/src/lib/components/copilot/autocomplete/request.ts +++ b/frontend/src/lib/components/copilot/autocomplete/request.ts @@ -23,26 +23,30 @@ export async function autocompleteRequest( }, abortController: AbortController ) { - let commentSymbol = getCommentSymbol(context.scriptLang) - let contextLines = comment( - commentSymbol, - 'You are a code completion assistant. You are given three important contexts (, , ) to help you complete the code.\n' - ) - contextLines += comment(commentSymbol, 'LANGUAGE CONTEXT:\n') - contextLines += comment(commentSymbol, getLangContext(context.scriptLang) + '\n') - contextLines += comment(commentSymbol, 'DIAGNOSTICS:\n') - contextLines += comment(commentSymbol, context.markers.map((m) => m.message).join('\n') + '\n') - contextLines += comment(commentSymbol, 'LIBRARY METHODS:\n') - contextLines += comment(commentSymbol, context.libraries + '\n') - - context.prefix = contextLines + '\n' + context.prefix - const providerModel = get(copilotInfo).codeCompletionModel if (!providerModel) { throw new Error('No code completion model selected') } + // Only add context lines for Mistral (native FIM) - other providers use chat completion + // too much context degrades significantly the quality of the completion + if (providerModel.provider === 'mistral') { + let commentSymbol = getCommentSymbol(context.scriptLang) + let contextLines = comment( + commentSymbol, + 'You are a code completion assistant. You are given three important contexts (, , ) to help you complete the code.\n' + ) + contextLines += comment(commentSymbol, 'LANGUAGE CONTEXT:\n') + contextLines += comment(commentSymbol, getLangContext(context.scriptLang) + '\n') + contextLines += comment(commentSymbol, 'DIAGNOSTICS:\n') + contextLines += comment(commentSymbol, context.markers.map((m) => m.message).join('\n') + '\n') + contextLines += comment(commentSymbol, 'LIBRARY METHODS:\n') + contextLines += comment(commentSymbol, context.libraries + '\n') + + context.prefix = contextLines + '\n' + context.prefix + } + try { const completion = await getFimCompletion( context.prefix, diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 240b3bb77c..c541df4a9d 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -14,7 +14,7 @@ import Anthropic from '@anthropic-ai/sdk' import { get, type Writable } from 'svelte/store' import { OpenAPI, ResourceService, type Script } from '../../gen' import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' -import { formatResourceTypes } from './utils' +import { formatResourceTypes, isMistralFamily } from './utils' import { z } from 'zod' import { processToolCall, type Tool, type ToolCallbacks } from './chat/shared' import { @@ -290,6 +290,7 @@ function getModelSpecificConfig( const modelKey = `${modelProvider.provider}:${modelProvider.model}` const customMaxTokensStore = get(copilotInfo)?.maxTokensPerModel const maxTokens = customMaxTokensStore?.[modelKey] ?? defaultMaxTokens + const isMistralModel = isMistralFamily(modelProvider.model) if ( (modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai') && (modelProvider.model.startsWith('o') || modelProvider.model.startsWith('gpt-5')) @@ -297,7 +298,8 @@ function getModelSpecificConfig( return { model: modelProvider.model, ...(tools && tools.length > 0 ? { tools } : {}), - max_completion_tokens: maxTokens + max_completion_tokens: maxTokens, + ...(isMistralModel ? { seed: undefined } : {}) } } else { return { @@ -314,7 +316,8 @@ function getModelSpecificConfig( temperature: 0 }), ...(tools && tools.length > 0 ? { tools } : {}), - max_tokens: maxTokens + max_tokens: maxTokens, + ...(isMistralModel ? { seed: undefined } : {}) } } } diff --git a/frontend/src/lib/components/copilot/utils.ts b/frontend/src/lib/components/copilot/utils.ts index 2ac122dc4d..b13e433737 100644 --- a/frontend/src/lib/components/copilot/utils.ts +++ b/frontend/src/lib/components/copilot/utils.ts @@ -168,3 +168,21 @@ export function yamlStringifyExceptKeys(obj: any, keys: string[]) { } }) } + +/** + * Checks if a model supports FIM (Fill-in-the-Middle) autocomplete. + * Currently only Codestral models (non-embedding) support this. + */ +export function supportsAutocomplete(model: string): boolean { + const lower = model.toLowerCase() + return lower.includes('codestral') && !lower.includes('embed') +} + +/** + * Checks if a model belongs to the Mistral family. + * Used for provider-specific configurations (e.g., excluding seed parameter). + */ +export function isMistralFamily(model: string): boolean { + const lower = model.toLowerCase() + return lower.includes('mistral') || lower.includes('codestral') +} diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 20776a2ef5..d8fb7299f5 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -3,6 +3,7 @@ import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { AI_PROVIDERS, fetchAvailableModels } from '../copilot/lib' + import { supportsAutocomplete } from '../copilot/utils' import TestAiKey from '../copilot/TestAIKey.svelte' import Description from '../Description.svelte' import Label from '../Label.svelte' @@ -166,9 +167,7 @@ } } - const autocompleteModels = $derived( - selectedAiModels.filter((m) => m.startsWith('codestral-') && !m.startsWith('codestral-embed')) - ) + const autocompleteModels = $derived(selectedAiModels.filter(supportsAutocomplete))