feat(aichat): handle codestral from any provider (#7649)

* fix for codestral

* enable codestral

* fim with completion

* reduce context when using completion

* refactor: extract model detection utilities for Codestral/Mistral

Consolidate duplicated model detection logic into shared utilities
in copilot/utils.ts to improve maintainability and ensure consistency
across autocomplete support checks and Mistral-specific configurations.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: add cursor marker to FIM-to-chat transformation prompt

Add explicit <CURSOR/> marker between prefix and suffix in the
FIM-to-chat transformation to help chat models better understand
where the completion should be inserted.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

backend/windmill-api/src/ai.rs

@@ -6,7 +6,7 @@ use http::{HeaderMap, Method};
 use quick_cache::sync::Cache;
 use reqwest::{Client, RequestBuilder};
 use serde::{Deserialize, Serialize};
-use serde_json::value::RawValue;
+use serde_json::{json, value::RawValue};
 use std::collections::HashMap;
 use windmill_audit::{audit_oss::audit_log, ActionKind};
 use windmill_common::ai_providers::{AIProvider, ProviderConfig, ProviderModel};
@@ -438,6 +438,53 @@ pub struct AIConfig {
     pub max_tokens_per_model: Option<HashMap<String, i32>>,
 }
 
+// FIM (Fill-in-the-Middle) simulation for providers that don't support native FIM
+#[derive(Deserialize, Debug)]
+struct FimRequest {
+    model: String,
+    prompt: String,         // code before cursor
+    suffix: Option<String>, // code after cursor
+    temperature: Option<f32>,
+    max_tokens: Option<u32>,
+    stop: Option<Vec<String>>,
+}
+
+/// Checks if the AI provider supports native FIM (Fill-in-the-Middle) endpoint
+fn supports_native_fim(provider: &AIProvider) -> bool {
+    matches!(provider, AIProvider::Mistral)
+}
+
+/// Transforms a FIM request to chat/completions format for providers that don't support native FIM.
+fn transform_fim_to_chat_completions(body: &Bytes) -> Result<(Bytes, String)> {
+    let fim_req: FimRequest = serde_json::from_slice(body)
+        .map_err(|e| Error::internal_err(format!("Failed to parse FIM request: {}", e)))?;
+
+    let suffix = fim_req.suffix.unwrap_or_default();
+
+    let system_prompt = "You are a code completion assistant. Complete the code at the <CURSOR/> position between the given prefix and suffix. Output ONLY the code that goes at the cursor - no explanations, no markdown, no repeating the prefix or suffix.";
+
+    let user_content = format!(
+        "<PREFIX>\n{}\n<CURSOR/>\n<SUFFIX>\n{}",
+        fim_req.prompt, suffix
+    );
+
+    let chat_req = json!({
+        "model": fim_req.model,
+        "messages": [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_content}
+        ],
+        "temperature": fim_req.temperature.unwrap_or(0.0),
+        "max_tokens": fim_req.max_tokens.unwrap_or(256),
+        "stop": fim_req.stop
+    });
+
+    let chat_body = serde_json::to_vec(&chat_req)
+        .map_err(|e| Error::internal_err(format!("Failed to serialize chat request: {}", e)))?;
+
+    Ok((Bytes::from(chat_body), "chat/completions".to_string()))
+}
+
 pub fn global_service() -> Router {
     Router::new().route("/proxy/*ai", post(global_proxy).get(global_proxy))
 }
@@ -516,10 +563,10 @@ async fn global_proxy(
 async fn proxy(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
-    Path((w_id, ai_path)): Path<(String, String)>,
+    Path((w_id, mut ai_path)): Path<(String, String)>,
     method: Method,
     headers: HeaderMap,
-    body: Bytes,
+    mut body: Bytes,
 ) -> impl IntoResponse {
     let provider = headers
         .get("X-Provider")
@@ -600,6 +647,19 @@ async fn proxy(
         }
     };
 
+    // Check if this is a FIM request to a provider that doesn't support native FIM endpoint
+    // For such providers, transform to use FIM sentinel tokens with the chat/completions endpoint
+    let is_fim_request = ai_path.contains("fim/completions");
+    if is_fim_request && !supports_native_fim(&provider) {
+        tracing::debug!(
+            "Transforming FIM request to chat/completions with FIM tokens for provider {:?}",
+            provider
+        );
+        let (chat_body, chat_path) = transform_fim_to_chat_completions(&body)?;
+        body = chat_body;
+        ai_path = chat_path;
+    }
+
     // Extract model and streaming flag for Bedrock transformation (only for POST requests)
     let (model, is_streaming) =
         if matches!(provider, AIProvider::AWSBedrock) && method == Method::POST {

frontend/src/lib/components/copilot/autocomplete/Autocompletor.ts

@@ -5,6 +5,7 @@ import { LRUCache } from 'lru-cache'
 import { autocompleteRequest } from './request'
 import { FIM_MAX_TOKENS, getModelContextWindow } from '../lib'
 import { setGlobalCSS } from '../shared'
+import { supportsAutocomplete } from '../utils'
 import { get } from 'svelte/store'
 import type { MonacoLanguageClient } from 'monaco-languageclient'
 import { copilotInfo } from '$lib/aiStore'
@@ -202,12 +203,8 @@ export class Autocompletor {
 	}
 
 	static isProviderModelSupported(providerModel: AIProviderModel | undefined) {
-		return (
-			providerModel &&
-			providerModel.provider === 'mistral' &&
-			providerModel.model.startsWith('codestral-') &&
-			!providerModel.model.startsWith('codestral-embed')
-		)
+		if (!providerModel) return false
+		return supportsAutocomplete(providerModel.model)
 	}
 
 	dispose() {

frontend/src/lib/components/copilot/autocomplete/request.ts

@@ -23,26 +23,30 @@ export async function autocompleteRequest(
 	},
 	abortController: AbortController
 ) {
-	let commentSymbol = getCommentSymbol(context.scriptLang)
-	let contextLines = comment(
-		commentSymbol,
-		'You are a code completion assistant. You are given three important contexts (<LANGUAGE CONTEXT>, <DIAGNOSTICS>, <LIBRARY METHODS>) to help you complete the code.\n'
-	)
-	contextLines += comment(commentSymbol, 'LANGUAGE CONTEXT:\n')
-	contextLines += comment(commentSymbol, getLangContext(context.scriptLang) + '\n')
-	contextLines += comment(commentSymbol, 'DIAGNOSTICS:\n')
-	contextLines += comment(commentSymbol, context.markers.map((m) => m.message).join('\n') + '\n')
-	contextLines += comment(commentSymbol, 'LIBRARY METHODS:\n')
-	contextLines += comment(commentSymbol, context.libraries + '\n')
-
-	context.prefix = contextLines + '\n' + context.prefix
-
 	const providerModel = get(copilotInfo).codeCompletionModel
 
 	if (!providerModel) {
 		throw new Error('No code completion model selected')
 	}
 
+	// Only add context lines for Mistral (native FIM) - other providers use chat completion
+	// too much context degrades significantly the quality of the completion
+	if (providerModel.provider === 'mistral') {
+		let commentSymbol = getCommentSymbol(context.scriptLang)
+		let contextLines = comment(
+			commentSymbol,
+			'You are a code completion assistant. You are given three important contexts (<LANGUAGE CONTEXT>, <DIAGNOSTICS>, <LIBRARY METHODS>) to help you complete the code.\n'
+		)
+		contextLines += comment(commentSymbol, 'LANGUAGE CONTEXT:\n')
+		contextLines += comment(commentSymbol, getLangContext(context.scriptLang) + '\n')
+		contextLines += comment(commentSymbol, 'DIAGNOSTICS:\n')
+		contextLines += comment(commentSymbol, context.markers.map((m) => m.message).join('\n') + '\n')
+		contextLines += comment(commentSymbol, 'LIBRARY METHODS:\n')
+		contextLines += comment(commentSymbol, context.libraries + '\n')
+
+		context.prefix = contextLines + '\n' + context.prefix
+	}
+
 	try {
 		const completion = await getFimCompletion(
 			context.prefix,

frontend/src/lib/components/copilot/lib.ts

@@ -14,7 +14,7 @@ import Anthropic from '@anthropic-ai/sdk'
 import { get, type Writable } from 'svelte/store'
 import { OpenAPI, ResourceService, type Script } from '../../gen'
 import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts'
-import { formatResourceTypes } from './utils'
+import { formatResourceTypes, isMistralFamily } from './utils'
 import { z } from 'zod'
 import { processToolCall, type Tool, type ToolCallbacks } from './chat/shared'
 import {
@@ -290,6 +290,7 @@ function getModelSpecificConfig(
 	const modelKey = `${modelProvider.provider}:${modelProvider.model}`
 	const customMaxTokensStore = get(copilotInfo)?.maxTokensPerModel
 	const maxTokens = customMaxTokensStore?.[modelKey] ?? defaultMaxTokens
+	const isMistralModel = isMistralFamily(modelProvider.model)
 	if (
 		(modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai') &&
 		(modelProvider.model.startsWith('o') || modelProvider.model.startsWith('gpt-5'))
@@ -297,7 +298,8 @@ function getModelSpecificConfig(
 		return {
 			model: modelProvider.model,
 			...(tools && tools.length > 0 ? { tools } : {}),
-			max_completion_tokens: maxTokens
+			max_completion_tokens: maxTokens,
+			...(isMistralModel ? { seed: undefined } : {})
 		}
 	} else {
 		return {
@@ -314,7 +316,8 @@ function getModelSpecificConfig(
 						temperature: 0
 					}),
 			...(tools && tools.length > 0 ? { tools } : {}),
-			max_tokens: maxTokens
+			max_tokens: maxTokens,
+			...(isMistralModel ? { seed: undefined } : {})
 		}
 	}
 }

frontend/src/lib/components/copilot/utils.ts

@@ -168,3 +168,21 @@ export function yamlStringifyExceptKeys(obj: any, keys: string[]) {
 		}
 	})
 }
+
+/**
+ * Checks if a model supports FIM (Fill-in-the-Middle) autocomplete.
+ * Currently only Codestral models (non-embedding) support this.
+ */
+export function supportsAutocomplete(model: string): boolean {
+	const lower = model.toLowerCase()
+	return lower.includes('codestral') && !lower.includes('embed')
+}
+
+/**
+ * Checks if a model belongs to the Mistral family.
+ * Used for provider-specific configurations (e.g., excluding seed parameter).
+ */
+export function isMistralFamily(model: string): boolean {
+	const lower = model.toLowerCase()
+	return lower.includes('mistral') || lower.includes('codestral')
+}

frontend/src/lib/components/workspaceSettings/AISettings.svelte

@@ -3,6 +3,7 @@
 	import { workspaceStore } from '$lib/stores'
 	import { sendUserToast } from '$lib/toast'
 	import { AI_PROVIDERS, fetchAvailableModels } from '../copilot/lib'
+	import { supportsAutocomplete } from '../copilot/utils'
 	import TestAiKey from '../copilot/TestAIKey.svelte'
 	import Description from '../Description.svelte'
 	import Label from '../Label.svelte'
@@ -166,9 +167,7 @@
 		}
 	}
 
-	const autocompleteModels = $derived(
-		selectedAiModels.filter((m) => m.startsWith('codestral-') && !m.startsWith('codestral-embed'))
-	)
+	const autocompleteModels = $derived(selectedAiModels.filter(supportsAutocomplete))
 </script>
 
 <div class="flex flex-col gap-4 mt-4">