feat: Support mistral anthropic for ai (#4692)

* wip: openai proxy and other ai proxy integration

* fixing migration script

* wip: support different ai provider in front, fix proxy openai

* wip: adding frontend ai provider

* updated copilot types

* wip: working on anthropic integration

* done AI proxy front

* adding new type and support for anthropic

* updating gitignore

* adding streaming response

* added streaming prompt

* push lib/gen

* wip: fixing anthropic

* anthropic fully supported

* fix backend missing var error and fully support stream event for anthropic

* remove gen directory

* fixing openapi file

* add support for mistral, and update create workspace components

* remove deref.json

* remove package-json

* openapi

* fix ui enable code

* added utility function for init workspace ai provider

* fix workspace switch bug

* update anthropic property and fixed frontend error

* fix workspace settings

* update error message and fix typo migration file

* chore: update openapi file

* fix dev file

* add .sqlx

* all

* update sqlx

---------

Co-authored-by: dieriba <t.dieriba@gmail.com>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
Co-authored-by: Ruben Fiszel <ruben@rubenfiszel.com>

.gitignore

@@ -6,4 +6,4 @@ frontend/src/routes/test.svelte
 CaddyfileRemoteMalo
 *.swp
 **/.idea/
-.direnv
+.direnv

backend/.gitignore

@@ -2,7 +2,7 @@ target/
 .env
 oauth.json
 oauth2.json
-windmill-api/openapi-deref.yaml
 tracing.folded
 heaptrack*
 index/
+windmill-api/openapi-*.*

backend/.sqlx/query-0230bd64d2b6719c3536a106e18a68c7b43b3a9a9efa92b5517132e9c0d8a25b.json

@@ -1,12 +1,12 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT openai_resource_path, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1",
+  "query": "SELECT ai_resource, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1",
   "describe": {
     "columns": [
       {
         "ordinal": 0,
-        "name": "openai_resource_path",
-        "type_info": "Varchar"
+        "name": "ai_resource",
+        "type_info": "Jsonb"
       },
       {
         "ordinal": 1,
@@ -24,5 +24,5 @@
       false
     ]
   },
-  "hash": "ceb97024ebf1a1c00ea1ed4952f66b229ca4622c537cc252d8ed52b4d24270ee"
+  "hash": "0230bd64d2b6719c3536a106e18a68c7b43b3a9a9efa92b5517132e9c0d8a25b"
 }

backend/.sqlx/query-0331a81262e2d3c1bcfaeb64617b11eb68ab4599d64e5e5af639a9ac5d791fd0.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT\n                -- slack_team_id, \n                -- slack_name, \n                -- slack_command_script, \n                -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n                auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n                CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n                CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n                webhook, \n                deploy_to, \n                error_handler, \n                openai_resource_path, \n                code_completion_enabled, \n                error_handler_extra_args, \n                error_handler_muted_on_cancel, \n                large_file_storage, \n                git_sync,\n                default_app,\n                default_scripts,\n                workspace.name\n            FROM workspace_settings\n            LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id\n            WHERE workspace_id = $1",
+  "query": "SELECT\n                -- slack_team_id, \n                -- slack_name, \n                -- slack_command_script, \n                -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n                auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n                CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n                CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n                webhook, \n                deploy_to, \n                error_handler, \n                ai_resource, \n                code_completion_enabled, \n                error_handler_extra_args, \n                error_handler_muted_on_cancel, \n                large_file_storage, \n                git_sync,\n                default_app,\n                default_scripts,\n                workspace.name\n            FROM workspace_settings\n            LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id\n            WHERE workspace_id = $1",
   "describe": {
     "columns": [
       {
@@ -35,8 +35,8 @@
       },
       {
         "ordinal": 6,
-        "name": "openai_resource_path",
-        "type_info": "Varchar"
+        "name": "ai_resource",
+        "type_info": "Jsonb"
       },
       {
         "ordinal": 7,
@@ -102,5 +102,5 @@
       false
     ]
   },
-  "hash": "188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3"
+  "hash": "0331a81262e2d3c1bcfaeb64617b11eb68ab4599d64e5e5af639a9ac5d791fd0"
 }

backend/.sqlx/query-034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE workspace_settings SET openai_resource_path = $1, code_completion_enabled = $2 WHERE workspace_id = $3",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Bool",
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304"
-}

backend/.sqlx/query-1610c79b8d238e3a35d795f34a2a0100b933b7f199d41cc2f554d57c811d55f3.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE workspace_settings SET ai_resource = $1, code_completion_enabled = $2 WHERE workspace_id = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb",
+        "Bool",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "1610c79b8d238e3a35d795f34a2a0100b933b7f199d41cc2f554d57c811d55f3"
+}

backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json

@@ -65,8 +65,8 @@
       },
       {
         "ordinal": 12,
-        "name": "openai_resource_path",
-        "type_info": "Varchar"
+        "name": "ai_resource",
+        "type_info": "Jsonb"
       },
       {
         "ordinal": 13,

backend/.sqlx/query-286fa00c088df146c08c1934556f06a4243c66787ab8759a8045e60effd2fb77.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT value\n            FROM resource\n            WHERE path = $1 AND workspace_id = $2",
+  "query": "SELECT value\n                FROM resource\n                WHERE path = $1 AND workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -19,5 +19,5 @@
       true
     ]
   },
-  "hash": "acdaa5151f8f7f37bb8c8c5a7d146789887e47db9695fc26b1dfaedd735e1e60"
+  "hash": "286fa00c088df146c08c1934556f06a4243c66787ab8759a8045e60effd2fb77"
 }

backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json

@@ -65,8 +65,8 @@
       },
       {
         "ordinal": 12,
-        "name": "openai_resource_path",
-        "type_info": "Varchar"
+        "name": "ai_resource",
+        "type_info": "Jsonb"
       },
       {
         "ordinal": 13,

backend/.sqlx/query-6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE workspace_settings SET openai_resource_path = NULL, code_completion_enabled = $1 WHERE workspace_id = $2",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Bool",
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03"
-}

backend/.sqlx/query-6940ddc5ee8d2a1048213d46f52d964b199604ba66dedbe9f89cea727d726a2d.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE workspace_settings SET ai_resource = NULL, code_completion_enabled = $1 WHERE workspace_id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Bool",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "6940ddc5ee8d2a1048213d46f52d964b199604ba66dedbe9f89cea727d726a2d"
+}

backend/.sqlx/query-ceda377fa534656ac12a7f41db77db1f054ec751855c8db7352b0e9c20b63ef8.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT ai_resource FROM workspace_settings WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "ai_resource",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "ceda377fa534656ac12a7f41db77db1f054ec751855c8db7352b0e9c20b63ef8"
+}

backend/.sqlx/query-f8b689ec3e09f14dc02ac9e9f98d252a14d3a2acfadcd9cd6ba27ca9799f4706.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT openai_resource_path FROM workspace_settings WHERE workspace_id = $1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "openai_resource_path",
-        "type_info": "Varchar"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      true
-    ]
-  },
-  "hash": "f8b689ec3e09f14dc02ac9e9f98d252a14d3a2acfadcd9cd6ba27ca9799f4706"
-}

backend/migrations/20241105133531_alter_openai_resource_path_column_in_workspace_settings.down.sql

@@ -0,0 +1,7 @@
+-- Add down migration script here
+ALTER TABLE workspace_settings
+ALTER COLUMN ai_resource TYPE text
+USING ai_resource->>'path';
+
+ALTER TABLE workspace_settings
+RENAME COLUMN ai_resource to openai_resource_path;

backend/migrations/20241105133531_alter_openai_resource_path_column_in_workspace_settings.up.sql

@@ -0,0 +1,7 @@
+-- Add up migration script here
+ALTER TABLE workspace_settings
+ALTER COLUMN openai_resource_path TYPE jsonb
+USING jsonb_build_object('provider', 'openai', 'path', openai_resource_path);
+
+ALTER TABLE workspace_settings
+RENAME COLUMN openai_resource_path TO ai_resource;

backend/windmill-api/openapi.yaml

@@ -1681,8 +1681,8 @@ paths:
                     type: string
                   deploy_to:
                     type: string
-                  openai_resource_path:
-                    type: string
+                  ai_resource:
+                    $ref: "#/components/schemas/AiResource"
                   code_completion_enabled:
                     type: boolean
                   error_handler:
@@ -1966,8 +1966,8 @@ paths:
               required:
                 - code_completion_enabled
               properties:
-                openai_resource_path:
-                  type: string
+                ai_resource:
+                  $ref: "#/components/schemas/AiResource"
                 code_completion_enabled:
                   type: boolean
       responses:
@@ -1995,12 +1995,15 @@ paths:
               schema:
                 type: object
                 properties:
-                  exists_openai_resource_path:
+                  ai_provider:
+                    type: string
+                  exists_ai_resource:
                     type: boolean
                   code_completion_enabled:
                     type: boolean
                 required:
-                  - exists_openai_resource_path
+                  - ai_provider
+                  - exists_ai_resource
                   - code_completion_enabled
 
   /w/{workspace}/workspaces/edit_error_handler:
@@ -10194,6 +10197,16 @@ components:
   schemas:
     $ref: "../../openflow.openapi.yaml#/components/schemas"
 
+    AiResource:
+      type: object
+      properties:
+        path:
+          type: string
+        provider:
+          type: string
+      required:
+        - path
+        - provider
     Script:
       type: object
       properties:
@@ -11012,7 +11025,7 @@ components:
             - "jobs.disapproval"
             - "jobs.delete"
             - "account.delete"
-            - "openai.request"
+            - "ai.request"
             - "resources.create"
             - "resources.update"
             - "resources.delete"

backend/windmill-api/src/ai.rs

@@ -0,0 +1,507 @@
+use crate::{
+    db::{ApiAuthed, DB},
+    variables::decrypt,
+};
+use anthropic::AnthropicCache;
+use axum::{
+    body::Bytes,
+    extract::{Path, Query},
+    response::IntoResponse,
+    routing::post,
+    Extension, Router,
+};
+use lazy_static::lazy_static;
+use mistral::MistralCache;
+use openai::OpenaiCache;
+use quick_cache::sync::Cache;
+use reqwest::{Client, RequestBuilder};
+use serde::{Deserialize, Deserializer};
+use windmill_audit::audit_ee::audit_log;
+use windmill_audit::ActionKind;
+use windmill_common::error::{to_anyhow, Result};
+use windmill_common::variables::build_crypt;
+
+use windmill_common::error::Error;
+
+use serde_json::value::{RawValue, Value};
+use std::collections::HashMap;
+
+lazy_static::lazy_static! {
+    static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
+        .timeout(std::time::Duration::from_secs(60 * 5))
+        .user_agent("windmill/beta")
+        .build().unwrap();
+}
+
+trait AiRequest {
+    fn prepare_request(self, path: &str, body: Bytes) -> Result<RequestBuilder>;
+}
+
+mod openai {
+    use super::*;
+
+    use super::{get_variable_or_self, KeyCache};
+
+    const API_VERSION: &str = "2023-05-15";
+
+    #[derive(Deserialize, Debug)]
+    struct OpenaiResource {
+        api_key: String,
+        organization_id: Option<String>,
+    }
+
+    #[derive(Deserialize, Debug)]
+    struct OpenaiClientCredentialsOauthResource {
+        client_id: String,
+        client_secret: String,
+        token_url: String,
+        user: Option<String>,
+    }
+
+    #[derive(Deserialize, Debug)]
+    #[serde(untagged, rename_all = "snake_case")]
+    enum OpenaiConfig {
+        Resource(OpenaiResource),
+        ClientCredentialsOauthResource(OpenaiClientCredentialsOauthResource),
+    }
+
+    lazy_static::lazy_static! {
+        pub static ref OPENAI_AZURE_BASE_PATH: Option<String> = std::env::var("OPENAI_AZURE_BASE_PATH").ok();
+    }
+
+    #[derive(Deserialize, Debug)]
+    struct OpenaiCredentials {
+        access_token: String,
+    }
+
+    #[derive(Clone, Debug, Deserialize)]
+    pub struct OpenaiCache {
+        api_key: String,
+        organization_id: Option<String>,
+        azure_base_path: Option<String>,
+        user: Option<String>,
+    }
+
+    impl OpenaiCache {
+        pub fn new(
+            api_key: String,
+            organization_id: Option<String>,
+            azure_base_path: Option<String>,
+            user: Option<String>,
+        ) -> Self {
+            Self { api_key, organization_id, azure_base_path, user }
+        }
+    }
+
+    const BASE_URL: &str = "https://api.openai.com/v1";
+    impl AiRequest for OpenaiCache {
+        fn prepare_request(self, openai_path: &str, mut body: Bytes) -> Result<RequestBuilder> {
+            let OpenaiCache { api_key, azure_base_path, organization_id, user } = self;
+            if user.is_some() {
+                tracing::debug!("Adding user to request body");
+                let mut json_body: HashMap<String, Box<RawValue>> = serde_json::from_slice(&body)
+                    .map_err(|e| {
+                    Error::InternalErr(format!("Failed to parse request body: {}", e))
+                })?;
+
+                let user_json_string = serde_json::Value::String(user.unwrap()).to_string(); // makes sure to escape characters
+
+                json_body.insert(
+                    "user".to_string(),
+                    RawValue::from_string(user_json_string)
+                        .map_err(|e| Error::InternalErr(format!("Failed to parse user: {}", e)))?,
+                );
+
+                body = serde_json::to_vec(&json_body)
+                    .map_err(|e| {
+                        Error::InternalErr(format!("Failed to reserialize request body: {}", e))
+                    })?
+                    .into();
+            }
+
+            let base_url = if let Some(base_url) = azure_base_path {
+                base_url
+            } else {
+                BASE_URL.to_string()
+            };
+            let url = format!("{}/{}", base_url, openai_path);
+            let mut request = HTTP_CLIENT
+                .post(url)
+                .header("content-type", "application/json")
+                .body(body);
+
+            if base_url != BASE_URL {
+                request = request
+                    .header("api-key", api_key)
+                    .query(&[("api-version", API_VERSION)])
+            } else {
+                request = request.header("authorization", format!("Bearer {}", api_key))
+            }
+
+            if let Some(org_id) = organization_id {
+                request = request.header("OpenAI-Organization", org_id);
+            }
+
+            Ok(request)
+        }
+    }
+
+    async fn get_openai_key_using_credentials_flow(
+        mut resource: OpenaiClientCredentialsOauthResource,
+        db: &DB,
+        w_id: &str,
+    ) -> Result<String> {
+        resource.client_id = get_variable_or_self(resource.client_id, db, w_id).await?;
+        resource.client_secret = get_variable_or_self(resource.client_secret, db, w_id).await?;
+        resource.token_url = get_variable_or_self(resource.token_url, db, w_id).await?;
+        let mut params = HashMap::new();
+        params.insert("grant_type", "client_credentials");
+        let response = HTTP_CLIENT
+            .post(resource.token_url)
+            .form(&params)
+            .basic_auth(resource.client_id, Some(resource.client_secret))
+            .send()
+            .await
+            .map_err(|err| {
+                Error::InternalErr(format!(
+                    "Failed to get OpenAI credentials using credentials flow: {}",
+                    err
+                ))
+            })?;
+        let response = response.json::<OpenaiCredentials>().await.map_err(|err| {
+            Error::InternalErr(format!(
+                "Failed to parse OpenAI credentials from credentials flow: {}",
+                err
+            ))
+        })?;
+        Ok(response.access_token)
+    }
+
+    pub async fn get_cached_value(db: &DB, w_id: &str, resource: Value) -> Result<KeyCache> {
+        let config = serde_json::from_value(resource)
+            .map_err(|e| Error::InternalErr(format!("validating openai resource {e:#}")))?;
+
+        let mut user = None::<String>;
+        let mut resource = match config {
+            OpenaiConfig::Resource(resource) => {
+                tracing::debug!("Getting OpenAI key from static resource");
+                resource
+            }
+            OpenaiConfig::ClientCredentialsOauthResource(resource) => {
+                tracing::debug!("Getting OpenAI key with client credentials flow");
+                user = resource.user.clone();
+                let token = get_openai_key_using_credentials_flow(resource, db, w_id).await?;
+                OpenaiResource { api_key: token, organization_id: None }
+            }
+        };
+
+        resource.api_key = get_variable_or_self(resource.api_key, db, w_id).await?;
+
+        if let Some(organization_id) = resource.organization_id {
+            resource.organization_id = Some(get_variable_or_self(organization_id, db, w_id).await?);
+        }
+
+        if user.is_some() {
+            user = Some(get_variable_or_self(user.unwrap(), db, w_id).await?);
+        }
+
+        let azure_base_path = sqlx::query_scalar!(
+            "SELECT value
+            FROM global_settings
+            WHERE name = 'openai_azure_base_path'",
+        )
+        .fetch_optional(db)
+        .await?;
+
+        let azure_base_path = if let Some(azure_base_path) = azure_base_path {
+            Some(
+                serde_json::from_value::<String>(azure_base_path).map_err(|e| {
+                    Error::InternalErr(format!("validating openai azure base path {e:#}"))
+                })?,
+            )
+        } else {
+            OPENAI_AZURE_BASE_PATH.clone()
+        };
+
+        let workspace_cache = OpenaiCache::new(
+            resource.api_key.clone(),
+            resource.organization_id.clone(),
+            azure_base_path.clone(),
+            user.clone(),
+        );
+        Ok(KeyCache::Openai(workspace_cache))
+    }
+}
+
+mod anthropic {
+
+    use super::*;
+
+    #[derive(Clone, Deserialize, Debug)]
+    pub struct AnthropicCache {
+        #[serde(rename = "apiKey")]
+        pub api_key: String,
+    }
+
+    const API_VERSION: &str = "2023-06-01";
+
+    impl AnthropicCache {
+        pub fn new(api_key: String) -> Self {
+            Self { api_key }
+        }
+    }
+
+    const BASE_URL: &str = "https://api.anthropic.com";
+    impl AiRequest for AnthropicCache {
+        fn prepare_request(self, anthropic_path: &str, body: Bytes) -> Result<RequestBuilder> {
+            let AnthropicCache { api_key } = self;
+            let url = format!("{}/{}", BASE_URL, anthropic_path);
+            let request = HTTP_CLIENT
+                .post(url)
+                .header("x-api-key", api_key)
+                .header("anthropic-version", API_VERSION)
+                .header("content-type", "application/json")
+                .body(body);
+            Ok(request)
+        }
+    }
+
+    pub async fn get_cached_value(db: &DB, w_id: &str, resource: Value) -> Result<KeyCache> {
+        let mut resource: AnthropicCache = serde_json::from_value(resource)
+            .map_err(|e| Error::InternalErr(format!("validating anthropic resource {e:#}")))?;
+        resource.api_key = get_variable_or_self(resource.api_key, db, w_id).await?;
+        let workspace_cache = AnthropicCache::new(resource.api_key);
+        Ok(KeyCache::Anthropic(workspace_cache))
+    }
+}
+
+mod mistral {
+    use super::*;
+    #[derive(Deserialize, Clone, Debug)]
+    pub struct MistralCache {
+        #[serde(rename = "apiKey")]
+        pub api_key: String,
+    }
+
+    impl MistralCache {
+        pub fn new(api_key: String) -> Self {
+            Self { api_key }
+        }
+    }
+
+    const BASE_URL: &str = "https://api.mistral.ai";
+    impl AiRequest for MistralCache {
+        fn prepare_request(self, mistral_path: &str, body: Bytes) -> Result<RequestBuilder> {
+            let MistralCache { api_key } = self;
+
+            let url = format!("{}/{}", BASE_URL, mistral_path);
+            let request = HTTP_CLIENT
+                .post(url)
+                .header("content-type", "application/json")
+                .header("Accept", "application/json")
+                .header("authorization", format!("Bearer {}", api_key))
+                .body(body);
+            Ok(request)
+        }
+    }
+
+    pub async fn get_cached_value(db: &DB, w_id: &str, resource: Value) -> Result<KeyCache> {
+        let mut resource: MistralCache = serde_json::from_value(resource)
+            .map_err(|e| Error::InternalErr(format!("validating mistral resource {e:#}")))?;
+        resource.api_key = get_variable_or_self(resource.api_key, db, w_id).await?;
+
+        let workspace_cache = MistralCache::new(resource.api_key);
+        Ok(KeyCache::Mistral(workspace_cache))
+    }
+}
+
+#[derive(Clone, Debug)]
+pub enum KeyCache {
+    Openai(OpenaiCache),
+    Anthropic(AnthropicCache),
+    Mistral(MistralCache),
+}
+
+#[derive(Clone, Debug)]
+pub struct AiCache {
+    pub path: String,
+    pub cached_key: KeyCache,
+    pub expires_at: std::time::Instant,
+}
+
+impl AiCache {
+    pub fn new(path: String, cached_key: KeyCache) -> Self {
+        Self {
+            path,
+            cached_key,
+            expires_at: std::time::Instant::now() + std::time::Duration::from_secs(60),
+        }
+    }
+    fn is_expired(&self) -> bool {
+        self.expires_at < std::time::Instant::now()
+    }
+}
+
+lazy_static! {
+    pub static ref AI_KEY_CACHE: Cache<String, AiCache> = Cache::new(500);
+}
+
+struct Variable {
+    value: String,
+    is_secret: bool,
+}
+
+#[derive(Deserialize, Debug)]
+struct ProxyQueryParams {
+    no_cache: Option<bool>,
+}
+
+async fn get_variable_or_self(path: String, db: &DB, w_id: &str) -> Result<String> {
+    if !path.starts_with("$var:") {
+        return Ok(path);
+    }
+    let path = path.strip_prefix("$var:").unwrap().to_string();
+    let mut variable = sqlx::query_as!(
+        Variable,
+        "SELECT value, is_secret
+        FROM variable
+        WHERE path = $1 AND workspace_id = $2",
+        &path,
+        &w_id
+    )
+    .fetch_one(db)
+    .await?;
+    if variable.is_secret {
+        let mc = build_crypt(db, w_id).await?;
+        variable.value = decrypt(&mc, variable.value)?;
+    }
+    Ok(variable.value)
+}
+
+#[derive(Deserialize, Debug)]
+pub struct AiResource {
+    pub path: String,
+    #[serde(deserialize_with = "check_if_valid_ai_provider")]
+    pub provider: String,
+}
+
+fn check_if_valid_ai_provider<'de, D>(provider: D) -> std::result::Result<String, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let provider = String::deserialize(provider)?;
+    match provider.as_str() {
+        "anthropic" | "openai" | "mistral" => Ok(provider),
+        _ => Err(serde::de::Error::custom(
+            "Only the following Ai providers are supported: openai, anthropic and mistral"
+                .to_string(),
+        )),
+    }
+}
+
+pub fn workspaced_service() -> Router {
+    let router = Router::new().route("/proxy/*ai", post(proxy));
+
+    router
+}
+
+async fn proxy(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path((w_id, ai_path)): Path<(String, String)>,
+    Query(query_params): Query<ProxyQueryParams>,
+    body: Bytes,
+) -> impl IntoResponse {
+    let workspace_cache = AI_KEY_CACHE.get(&w_id);
+    let ai_cache = match workspace_cache {
+        Some(cache) if !cache.is_expired() && !query_params.no_cache.unwrap_or(false) => {
+            cache.cached_key
+        }
+        _ => {
+            let ai_resource = sqlx::query_scalar!(
+                "SELECT ai_resource FROM workspace_settings WHERE workspace_id = $1",
+                &w_id
+            )
+            .fetch_one(&db)
+            .await?;
+
+            if ai_resource.is_none() {
+                return Err(Error::InternalErr("AI resource not configured".to_string()));
+            }
+
+            let ai_resource = serde_json::from_value::<AiResource>(ai_resource.unwrap())
+                .map_err(|e| Error::BadRequest(e.to_string()))?;
+            let ai_resource_path = ai_resource.path;
+
+            let resource = sqlx::query_scalar!(
+                "SELECT value
+                FROM resource
+                WHERE path = $1 AND workspace_id = $2",
+                &ai_resource_path,
+                &w_id
+            )
+            .fetch_optional(&db)
+            .await?
+            .ok_or_else(|| {
+                Error::InternalErr(format!(
+                    "Could not find the {} resource at path {ai_resource_path}, update the resource path in the workspace settings", ai_resource.provider
+                ))
+            })?;
+
+            if resource.is_none() {
+                return Err(Error::InternalErr(format!(
+                    "{} resource missing value",
+                    ai_resource.provider
+                )));
+            }
+
+            let resource = resource.unwrap();
+
+            let ai_cache = match ai_resource.provider.as_str() {
+                "openai" => openai::get_cached_value(&db, &w_id, resource).await,
+                "anthropic" => anthropic::get_cached_value(&db, &w_id, resource).await,
+                "mistral" => mistral::get_cached_value(&db, &w_id, resource).await,
+                provider => {
+                    return Err(Error::BadRequest(format!("{} is not supported", provider)))
+                }
+            };
+            let ai_cache = ai_cache?;
+            AI_KEY_CACHE.insert(
+                w_id.clone(),
+                AiCache::new(ai_resource_path, ai_cache.clone()),
+            );
+            ai_cache
+        }
+    };
+    let (path, request) = match ai_cache {
+        KeyCache::Openai(cached) => ("openai_path", cached.prepare_request(&ai_path, body)),
+        KeyCache::Anthropic(cached) => ("anthropic_path", cached.prepare_request(&ai_path, body)),
+        KeyCache::Mistral(cached) => ("mistral_path", cached.prepare_request(&ai_path, body)),
+    };
+
+    let response = request?.send().await.map_err(to_anyhow)?;
+
+    let mut tx = db.begin().await?;
+
+    audit_log(
+        &mut *tx,
+        &authed,
+        "ai.request",
+        ActionKind::Execute,
+        &w_id,
+        Some(&authed.email),
+        Some([(path, &format!("{:?}", ai_path)[..])].into()),
+    )
+    .await?;
+    tx.commit().await?;
+
+    if response.error_for_status_ref().is_err() {
+        let err_msg = response.text().await.unwrap_or("".to_string());
+        return Err(Error::AiError(err_msg));
+    }
+
+    let status_code = response.status();
+    let headers = response.headers().clone();
+    let stream = response.bytes_stream();
+    Ok((status_code, headers, axum::body::Body::from_stream(stream)))
+}

backend/windmill-api/src/lib.rs

@@ -62,13 +62,14 @@ mod http_triggers;
 mod indexer_ee;
 mod inputs;
 mod integration;
+mod ai;
+
 #[cfg(feature = "parquet")]
 mod job_helpers_ee;
 pub mod job_metrics;
 pub mod jobs;
 pub mod oauth2_ee;
 mod oidc_ee;
-mod openai;
 mod raw_apps;
 mod resources;
 mod saml_ee;
@@ -94,6 +95,7 @@ mod workspaces_ee;
 
 pub const DEFAULT_BODY_LIMIT: usize = 2097152 * 100; // 200MB
 
+
 lazy_static::lazy_static! {
 
     pub static ref REQUEST_SIZE_LIMIT: Arc<RwLock<usize>> = Arc::new(RwLock::new(DEFAULT_BODY_LIMIT));
@@ -278,7 +280,7 @@ pub async fn run_server(
                         .nest("/job_helpers", job_helpers_service)
                         .nest("/jobs", jobs::workspaced_service())
                         .nest("/oauth", oauth2_ee::workspaced_service())
-                        .nest("/openai", openai::workspaced_service())
+                        .nest("/ai", ai::workspaced_service())
                         .nest("/raw_apps", raw_apps::workspaced_service())
                         .nest("/resources", resources::workspaced_service())
                         .nest("/schedules", schedule::workspaced_service())

backend/windmill-api/src/openai.rs

@@ -1,344 +0,0 @@
-use std::collections::HashMap;
-
-use crate::db::{ApiAuthed, DB};
-
-use axum::{
-    body::Bytes,
-    extract::{Extension, Path, Query},
-    response::IntoResponse,
-    routing::post,
-    Router,
-};
-use quick_cache::sync::Cache;
-use reqwest::Client;
-use serde_json::value::RawValue;
-use windmill_audit::audit_ee::audit_log;
-use windmill_audit::ActionKind;
-use windmill_common::{
-    error::{to_anyhow, Error},
-    variables::build_crypt,
-};
-
-use crate::variables::decrypt;
-use serde::Deserialize;
-
-lazy_static::lazy_static! {
-    static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
-        .timeout(std::time::Duration::from_secs(60 * 5))
-        .user_agent("windmill/beta")
-        .build().unwrap();
-}
-
-pub fn workspaced_service() -> Router {
-    let router = Router::new().route("/proxy/*openai_path", post(proxy));
-
-    router
-}
-
-#[derive(Deserialize)]
-struct OpenaiResource {
-    api_key: String,
-    organization_id: Option<String>,
-}
-
-#[derive(Deserialize)]
-struct OpenaiClientCredentialsOauthResource {
-    client_id: String,
-    client_secret: String,
-    token_url: String,
-    user: Option<String>,
-}
-
-#[derive(Deserialize)]
-#[serde(untagged)]
-enum OpenaiConfig {
-    Resource(OpenaiResource),
-    ClientCredentialsOauthResource(OpenaiClientCredentialsOauthResource),
-}
-
-struct Variable {
-    value: String,
-    is_secret: bool,
-}
-async fn get_variable_or_self(path: String, db: &DB, w_id: &String) -> Result<String, Error> {
-    if !path.starts_with("$var:") {
-        return Ok(path);
-    }
-    let path = path.strip_prefix("$var:").unwrap().to_string();
-    let mut variable = sqlx::query_as!(
-        Variable,
-        "SELECT value, is_secret
-        FROM variable
-        WHERE path = $1 AND workspace_id = $2",
-        &path,
-        &w_id
-    )
-    .fetch_one(db)
-    .await?;
-    if variable.is_secret {
-        let mc = build_crypt(&db, &w_id).await?;
-        variable.value = decrypt(&mc, variable.value)?;
-    }
-    Ok(variable.value)
-}
-
-lazy_static::lazy_static! {
-    pub static ref OPENAI_AZURE_BASE_PATH: Option<String> = std::env::var("OPENAI_AZURE_BASE_PATH").ok();
-    static ref OPENAI_KEY_CACHE: Cache<String, OpenaiKeyCache> = Cache::new(500);
-}
-
-#[derive(Deserialize)]
-struct OpenaiCredentials {
-    access_token: String,
-}
-async fn get_openai_key_using_credentials_flow(
-    mut resource: OpenaiClientCredentialsOauthResource,
-    db: &DB,
-    w_id: &String,
-) -> Result<String, Error> {
-    resource.client_id = get_variable_or_self(resource.client_id, &db, &w_id).await?;
-    resource.client_secret = get_variable_or_self(resource.client_secret, &db, &w_id).await?;
-    resource.token_url = get_variable_or_self(resource.token_url, &db, &w_id).await?;
-    let mut params = HashMap::new();
-    params.insert("grant_type", "client_credentials");
-    let response = HTTP_CLIENT
-        .post(resource.token_url)
-        .form(&params)
-        .basic_auth(resource.client_id, Some(resource.client_secret))
-        .send()
-        .await
-        .map_err(|err| {
-            Error::InternalErr(format!(
-                "Failed to get OpenAI credentials using credentials flow: {}",
-                err
-            ))
-        })?;
-    let response = response.json::<OpenaiCredentials>().await.map_err(|err| {
-        Error::InternalErr(format!(
-            "Failed to parse OpenAI credentials from credentials flow: {}",
-            err
-        ))
-    })?;
-    Ok(response.access_token)
-}
-
-#[derive(Clone)]
-struct OpenaiKeyCache {
-    api_key: String,
-    organization_id: Option<String>,
-    azure_base_path: Option<String>,
-    user: Option<String>,
-    expires_at: std::time::Instant,
-}
-
-impl OpenaiKeyCache {
-    fn new(
-        api_key: String,
-        organization_id: Option<String>,
-        azure_base_path: Option<String>,
-        expires_at: std::time::Instant,
-        user: Option<String>,
-    ) -> Self {
-        Self { api_key, organization_id, azure_base_path, expires_at, user }
-    }
-    fn is_expired(&self) -> bool {
-        self.expires_at < std::time::Instant::now()
-    }
-}
-
-#[derive(Deserialize)]
-struct ProxyQueryParams {
-    no_cache: Option<bool>,
-}
-async fn proxy(
-    authed: ApiAuthed,
-    Extension(db): Extension<DB>,
-    Path((w_id, openai_path)): Path<(String, String)>,
-    Query(query_params): Query<ProxyQueryParams>,
-    mut body: Bytes,
-) -> impl IntoResponse {
-    let workspace_cache = OPENAI_KEY_CACHE.get(&w_id);
-    let (api_key, organization_id, azure_base_path, user) = if query_params
-        .no_cache
-        .unwrap_or(false)
-        || workspace_cache.is_none()
-        || workspace_cache.clone().unwrap().is_expired()
-    {
-        let openai_resource_path = sqlx::query_scalar!(
-            "SELECT openai_resource_path FROM workspace_settings WHERE workspace_id = $1",
-            &w_id
-        )
-        .fetch_one(&db)
-        .await?;
-
-        if openai_resource_path.is_none() {
-            return Err(Error::InternalErr(
-                "OpenAI resource not configured".to_string(),
-            ));
-        }
-
-        let openai_resource_path = openai_resource_path.unwrap();
-
-        let resource = sqlx::query_scalar!(
-            "SELECT value
-            FROM resource
-            WHERE path = $1 AND workspace_id = $2",
-            &openai_resource_path,
-            &w_id
-        )
-        .fetch_optional(&db)
-        .await?
-        .ok_or_else(|| {
-            Error::InternalErr(format!(
-                "Could not find the OpenAI resource at path {openai_resource_path}, update the resource path in the workspace settings"
-            ))
-        })?;
-
-        if resource.is_none() {
-            return Err(Error::InternalErr(
-                "OpenAI resource missing value".to_string(),
-            ));
-        }
-
-        let config: OpenaiConfig = serde_json::from_value(resource.unwrap())
-            .map_err(|e| Error::InternalErr(format!("validating openai resource {e:#}")))?;
-
-        let mut user = None::<String>;
-        let mut resource = match config {
-            OpenaiConfig::Resource(resource) => {
-                tracing::debug!("Getting OpenAI key from static resource");
-                resource
-            }
-            OpenaiConfig::ClientCredentialsOauthResource(resource) => {
-                tracing::debug!("Getting OpenAI key with client credentials flow");
-                user = resource.user.clone();
-                let token = get_openai_key_using_credentials_flow(resource, &db, &w_id).await?;
-                OpenaiResource { api_key: token, organization_id: None }
-            }
-        };
-
-        resource.api_key = get_variable_or_self(resource.api_key, &db, &w_id).await?;
-
-        if resource.organization_id.is_some() {
-            resource.organization_id =
-                Some(get_variable_or_self(resource.organization_id.unwrap(), &db, &w_id).await?);
-        }
-
-        if user.is_some() {
-            user = Some(get_variable_or_self(user.unwrap(), &db, &w_id).await?);
-        }
-
-        let expires_at = std::time::Instant::now() + std::time::Duration::from_secs(60);
-
-        let azure_base_path = sqlx::query_scalar!(
-            "SELECT value
-            FROM global_settings
-            WHERE name = 'openai_azure_base_path'",
-        )
-        .fetch_optional(&db)
-        .await?;
-
-        let azure_base_path = if let Some(azure_base_path) = azure_base_path {
-            Some(
-                serde_json::from_value::<String>(azure_base_path).map_err(|e| {
-                    Error::InternalErr(format!("validating openai azure base path {e:#}"))
-                })?,
-            )
-        } else {
-            OPENAI_AZURE_BASE_PATH.clone()
-        };
-
-        let workspace_cache = OpenaiKeyCache::new(
-            resource.api_key.clone(),
-            resource.organization_id.clone(),
-            azure_base_path.clone(),
-            expires_at,
-            user.clone(),
-        );
-        OPENAI_KEY_CACHE.insert(w_id.clone(), workspace_cache);
-        (
-            resource.api_key,
-            resource.organization_id,
-            azure_base_path,
-            user,
-        )
-    } else {
-        tracing::debug!("Using cached OpenAI key");
-        let workspace_cache = workspace_cache.unwrap();
-        (
-            workspace_cache.api_key.clone(),
-            workspace_cache.organization_id.clone(),
-            workspace_cache.azure_base_path.clone(),
-            workspace_cache.user.clone(),
-        )
-    };
-
-    if user.is_some() {
-        tracing::debug!("Adding user to request body");
-        let mut json_body: HashMap<String, Box<RawValue>> = serde_json::from_slice(&body)
-            .map_err(|e| Error::InternalErr(format!("Failed to parse request body: {}", e)))?;
-
-        let user_json_string = serde_json::Value::String(user.unwrap()).to_string(); // makes sure to escape characters
-
-        json_body.insert(
-            "user".to_string(),
-            RawValue::from_string(user_json_string)
-                .map_err(|e| Error::InternalErr(format!("Failed to parse user: {}", e)))?,
-        );
-
-        body = serde_json::to_vec(&json_body)
-            .map_err(|e| Error::InternalErr(format!("Failed to reserialize request body: {}", e)))?
-            .into();
-    }
-
-    let base_url = if let Some(base_url) = azure_base_path {
-        base_url
-    } else {
-        "https://api.openai.com/v1".to_string()
-    };
-
-    let url = format!("{}/{}", base_url, openai_path);
-    let mut request = HTTP_CLIENT
-        .post(url)
-        .header("content-type", "application/json")
-        .body(body);
-
-    if base_url != "https://api.openai.com/v1" {
-        request = request
-            .header("api-key", api_key)
-            .query(&[("api-version", "2023-05-15")])
-    } else {
-        request = request.header("authorization", format!("Bearer {}", api_key))
-    }
-
-    if let Some(org_id) = organization_id {
-        request = request.header("OpenAI-Organization", org_id);
-    }
-
-    let response = request.send().await.map_err(to_anyhow)?;
-
-    let mut tx = db.begin().await?;
-    audit_log(
-        &mut *tx,
-        &authed,
-        "openai.request",
-        ActionKind::Execute,
-        &w_id,
-        Some(&authed.email),
-        Some([("openai_path", &format!("{:?}", openai_path)[..])].into()),
-    )
-    .await?;
-    tx.commit().await?;
-
-    if response.error_for_status_ref().is_err() {
-        return Err(Error::OpenAIError(
-            response.text().await.unwrap_or("".to_string()),
-        ));
-    }
-
-    let status_code = response.status();
-    let headers = response.headers().clone();
-    let stream = response.bytes_stream();
-
-    Ok((status_code, headers, axum::body::Body::from_stream(stream)))
-}

backend/windmill-api/src/workspaces.rs

@@ -8,6 +8,7 @@
 
 use std::collections::HashMap;
 
+use crate::ai::{AiResource, AI_KEY_CACHE};
 use crate::db::ApiAuthed;
 use crate::users_ee::send_email_if_possible;
 use crate::utils::get_instance_username_or_create_pending;
@@ -168,7 +169,7 @@ pub struct WorkspaceSettings {
     pub plan: Option<String>,
     pub webhook: Option<String>,
     pub deploy_to: Option<String>,
-    pub openai_resource_path: Option<String>,
+    pub ai_resource: Option<serde_json::Value>,
     pub code_completion_enabled: bool,
     pub error_handler: Option<String>,
     pub error_handler_extra_args: Option<serde_json::Value>,
@@ -234,7 +235,7 @@ struct EditWebhook {
 
 #[derive(Deserialize)]
 struct EditCopilotConfig {
-    openai_resource_path: Option<String>,
+    ai_resource: Option<serde_json::Value>,
     code_completion_enabled: bool,
 }
 
@@ -645,23 +646,33 @@ async fn edit_copilot_config(
 
     let mut tx = db.begin().await?;
 
-    if let Some(openai_resource_path) = &eo.openai_resource_path {
+    if let Some(ai_resource) = &eo.ai_resource {
+        let path = serde_json::from_value::<AiResource>(ai_resource.clone())
+            .map_err(|e| Error::BadRequest(e.to_string()))?
+            .path;
         sqlx::query!(
-            "UPDATE workspace_settings SET openai_resource_path = $1, code_completion_enabled = $2 WHERE workspace_id = $3",
-            openai_resource_path,
+            "UPDATE workspace_settings SET ai_resource = $1, code_completion_enabled = $2 WHERE workspace_id = $3",
+            ai_resource,
             eo.code_completion_enabled,
             &w_id
         )
         .execute(&mut *tx)
         .await?;
+
+        if let Some(cached) = AI_KEY_CACHE.get(&w_id) {
+            if cached.path != path {
+                AI_KEY_CACHE.remove(&w_id);
+            }
+        }
     } else {
         sqlx::query!(
-            "UPDATE workspace_settings SET openai_resource_path = NULL, code_completion_enabled = $1 WHERE workspace_id = $2",
+            "UPDATE workspace_settings SET ai_resource = NULL, code_completion_enabled = $1 WHERE workspace_id = $2",
             eo.code_completion_enabled,
             &w_id,
         )
         .execute(&mut *tx)
         .await?;
+        AI_KEY_CACHE.remove(&w_id);
     }
     audit_log(
         &mut *tx,
@@ -672,10 +683,7 @@ async fn edit_copilot_config(
         Some(&authed.email),
         Some(
             [
-                (
-                    "openai_resource_path",
-                    &format!("{:?}", eo.openai_resource_path)[..],
-                ),
+                ("ai_resource", &format!("{:?}", eo.ai_resource)[..]),
                 (
                     "code_completion_enabled",
                     &format!("{:?}", eo.code_completion_enabled)[..],
@@ -692,7 +700,8 @@ async fn edit_copilot_config(
 
 #[derive(Serialize)]
 struct CopilotInfo {
-    pub exists_openai_resource_path: bool,
+    pub ai_provider: String,
+    pub exists_ai_resource: bool,
     pub code_completion_enabled: bool,
 }
 async fn get_copilot_info(
@@ -701,16 +710,32 @@ async fn get_copilot_info(
 ) -> JsonResult<CopilotInfo> {
     let mut tx = db.begin().await?;
     let record = sqlx::query!(
-        "SELECT openai_resource_path, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1",
+        "SELECT ai_resource, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1",
         &w_id
     )
     .fetch_one(&mut *tx)
     .await
-    .map_err(|e| Error::InternalErr(format!("getting openai_resource_path and code_completion_enabled: {e:#}")))?;
+    .map_err(|e| Error::InternalErr(format!("getting ai_resource and code_completion_enabled: {e:#}")))?;
     tx.commit().await?;
 
+    let (ai_provider, exists_ai_resource) = if let Some(ai_resource) = record.ai_resource {
+        let ai_resource = serde_json::from_value::<AiResource>(ai_resource);
+        let exist = ai_resource.is_ok();
+        (
+            if exist {
+                ai_resource.unwrap().provider
+            } else {
+                "".to_string()
+            },
+            exist,
+        )
+    } else {
+        ("".to_string(), false)
+    };
+
     Ok(Json(CopilotInfo {
-        exists_openai_resource_path: record.openai_resource_path.is_some(),
+        ai_provider,
+        exists_ai_resource,
         code_completion_enabled: record.code_completion_enabled,
     }))
 }
@@ -2223,7 +2248,7 @@ struct SimplifiedSettings {
     error_handler: Option<String>,
     error_handler_extra_args: Option<Value>,
     error_handler_muted_on_cancel: bool,
-    openai_resource_path: Option<String>,
+    ai_resource: Option<serde_json::Value>,
     code_completion_enabled: bool,
     large_file_storage: Option<Value>,
     git_sync: Option<Value>,
@@ -2588,7 +2613,7 @@ async fn tarball_workspace(
                 webhook, 
                 deploy_to, 
                 error_handler, 
-                openai_resource_path, 
+                ai_resource, 
                 code_completion_enabled, 
                 error_handler_extra_args, 
                 error_handler_muted_on_cancel, 

backend/windmill-common/src/error.rs

@@ -59,7 +59,7 @@ pub enum Error {
     #[error("Error: {0:#?}")]
     JsonErr(serde_json::Value),
     #[error("{0}")]
-    OpenAIError(String),
+    AiError(String),
     #[error("{0}")]
     AlreadyCompleted(String),
 }
@@ -90,7 +90,7 @@ impl IntoResponse for Error {
             Self::RequireAdmin(_) => axum::http::StatusCode::FORBIDDEN,
             Self::SqlErr(_)
             | Self::BadRequest(_)
-            | Self::OpenAIError(_)
+            | Self::AiError(_)
             | Self::QuotaExceeded(_) => axum::http::StatusCode::BAD_REQUEST,
             _ => axum::http::StatusCode::INTERNAL_SERVER_ERROR,
         };

frontend/package-lock.json

@@ -9,6 +9,7 @@
 			"version": "1.423.2",
 			"license": "AGPL-3.0",
 			"dependencies": {
+				"@anthropic-ai/sdk": "^0.32.1",
 				"@aws-crypto/sha256-js": "^4.0.0",
 				"@codingame/monaco-vscode-configuration-service-override": "~8.0.2",
 				"@codingame/monaco-vscode-files-service-override": "~8.0.2",
@@ -21,6 +22,7 @@
 				"@codingame/monaco-vscode-standalone-typescript-language-features": "~8.0.2",
 				"@json2csv/plainjs": "^7.0.6",
 				"@leeoniya/ufuzzy": "^1.0.8",
+				"@mistralai/mistralai": "^1.3.0",
 				"@popperjs/core": "^2.11.6",
 				"@redocly/json-to-json-schema": "^0.0.1",
 				"@tanstack/svelte-table": "^8.9.9",
@@ -175,6 +177,30 @@
 				"node": ">=6.0.0"
 			}
 		},
+		"node_modules/@anthropic-ai/sdk": {
+			"version": "0.32.1",
+			"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.32.1.tgz",
+			"integrity": "sha512-U9JwTrDvdQ9iWuABVsMLj8nJVwAyQz6QXvgLsVhryhCEPkLsbcP/MXxm+jYcAwLoV8ESbaTTjnD4kuAFa+Hyjg==",
+			"license": "MIT",
+			"dependencies": {
+				"@types/node": "^18.11.18",
+				"@types/node-fetch": "^2.6.4",
+				"abort-controller": "^3.0.0",
+				"agentkeepalive": "^4.2.1",
+				"form-data-encoder": "1.7.2",
+				"formdata-node": "^4.3.2",
+				"node-fetch": "^2.6.7"
+			}
+		},
+		"node_modules/@anthropic-ai/sdk/node_modules/@types/node": {
+			"version": "18.19.64",
+			"resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.64.tgz",
+			"integrity": "sha512-955mDqvO2vFf/oL7V3WiUtiz+BugyX8uVbaT2H8oj3+8dRyH2FLiNdowe7eNqRM7IOIZvzDH76EoAT+gwm6aIQ==",
+			"license": "MIT",
+			"dependencies": {
+				"undici-types": "~5.26.4"
+			}
+		},
 		"node_modules/@apidevtools/json-schema-ref-parser": {
 			"version": "11.6.1",
 			"resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-11.6.1.tgz",
@@ -3659,6 +3685,14 @@
 			"resolved": "https://registry.npmjs.org/@mapbox/unitbezier/-/unitbezier-0.0.0.tgz",
 			"integrity": "sha512-HPnRdYO0WjFjRTSwO3frz1wKaU649OBFPX3Zo/2WZvuRi6zMiRGui8SnPQiQABgqCf8YikDe5t3HViTVw1WUzA=="
 		},
+		"node_modules/@mistralai/mistralai": {
+			"version": "1.3.0",
+			"resolved": "https://registry.npmjs.org/@mistralai/mistralai/-/mistralai-1.3.0.tgz",
+			"integrity": "sha512-G5DPCSC8sEhG3LUEZDYLD7qEZWDuZXgaX3IcoC3a/ydm9jFuh2pRZtknsgMx2sU8d7kxRuxblY3fPH5C38wnhQ==",
+			"peerDependencies": {
+				"zod": ">= 3"
+			}
+		},
 		"node_modules/@nodelib/fs.scandir": {
 			"version": "2.1.5",
 			"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -13738,6 +13772,16 @@
 			"integrity": "sha512-A7AMeJfGefk317I/3tBoUYRcDcNavKEkpiPN/nQsBz/viI2GvT7BtrqdPD6rGqBFN8Ax7v4obf+Cl32JF9DDVw==",
 			"dev": true
 		},
+		"node_modules/zod": {
+			"version": "3.23.8",
+			"resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz",
+			"integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==",
+			"license": "MIT",
+			"peer": true,
+			"funding": {
+				"url": "https://github.com/sponsors/colinhacks"
+			}
+		},
 		"node_modules/zstddec": {
 			"version": "0.1.0",
 			"resolved": "https://registry.npmjs.org/zstddec/-/zstddec-0.1.0.tgz",

frontend/package.json

@@ -82,6 +82,7 @@
 	},
 	"type": "module",
 	"dependencies": {
+		"@anthropic-ai/sdk": "^0.32.1",
 		"@aws-crypto/sha256-js": "^4.0.0",
 		"@codingame/monaco-vscode-configuration-service-override": "~8.0.2",
 		"@codingame/monaco-vscode-files-service-override": "~8.0.2",
@@ -94,6 +95,7 @@
 		"@codingame/monaco-vscode-standalone-typescript-language-features": "~8.0.2",
 		"@json2csv/plainjs": "^7.0.6",
 		"@leeoniya/ufuzzy": "^1.0.8",
+		"@mistralai/mistralai": "^1.3.0",
 		"@popperjs/core": "^2.11.6",
 		"@redocly/json-to-json-schema": "^0.0.1",
 		"@tanstack/svelte-table": "^8.9.9",

frontend/src/lib/components/Dev.svelte

@@ -39,7 +39,6 @@
 	import { CornerDownLeft, Play } from 'lucide-svelte'
 	import Toggle from './Toggle.svelte'
 	import { setLicense } from '$lib/enterpriseUtils'
-	import { workspacedOpenai } from './copilot/lib'
 	import type { FlowCopilotContext, FlowCopilotModule } from './copilot/flow'
 	import { pickScript } from './flows/flowStateUtils'
 	import {
@@ -49,6 +48,7 @@
 	} from '$lib/relative_imports'
 	import Tooltip from './Tooltip.svelte'
 	import type { ScheduleTrigger, TriggerContext } from './triggers'
+	import { initAllAiWorkspace } from './copilot/lib'
 	import type { FlowPropPickerConfig, PropPickerContext } from './prop_picker'
 	import type { PickableProperties } from './flows/previousResults'
 	$: token = $page.url.searchParams.get('wm_token') ?? undefined
@@ -110,12 +110,13 @@
 
 	async function setCopilotInfo() {
 		if (workspace) {
-			workspacedOpenai.init(workspace, token)
+			initAllAiWorkspace(workspace)
 			try {
 				copilotInfo.set(await WorkspaceService.getCopilotInfo({ workspace }))
 			} catch (err) {
 				copilotInfo.set({
-					exists_openai_resource_path: false,
+					ai_provider: '',
+					exists_ai_resource: false,
 					code_completion_enabled: false
 				})
 

frontend/src/lib/components/Editor.svelte

@@ -171,6 +171,7 @@
 	import { initVim } from './monaco_keybindings'
 	import { buildWorkerDefinition } from '$lib/monaco_workers/build_workers'
 	import { parseTypescriptDeps } from '$lib/relative_imports'
+	import type { AiProviderTypes } from './copilot/lib'
 
 	// import EditorTheme from './EditorTheme.svelte'
 
@@ -593,11 +594,13 @@
 							token.onCancellationRequested(() => {
 								abortController?.abort()
 							})
+							const aiProvider = $copilotInfo.ai_provider
 							const insertText = await editorCodeCompletion(
 								textUntilPosition,
 								textAfterPosition,
 								lang,
-								abortController
+								abortController,
+								aiProvider as AiProviderTypes
 							)
 							if (insertText) {
 								items = [
@@ -624,7 +627,7 @@
 		)
 	}
 
-	$: $copilotInfo.exists_openai_resource_path &&
+	$: $copilotInfo.exists_ai_resource &&
 		$copilotInfo.code_completion_enabled &&
 		$codeCompletionSessionEnabled &&
 		initialized &&

frontend/src/lib/components/FlowBuilder.svelte

@@ -85,6 +85,7 @@
 	import type { FlowBuilderWhitelabelCustomUi } from './custom_ui'
 	import FlowYamlEditor from './flows/header/FlowYamlEditor.svelte'
 	import { type TriggerContext, type ScheduleTrigger } from './triggers'
+	import type { AiProviderTypes } from './copilot/lib'
 
 	export let initialPath: string = ''
 	export let pathStoreInit: string | undefined = undefined
@@ -789,6 +790,7 @@
 		try {
 			push(history, $flowStore)
 			let module = stepOnly ? $copilotModulesStore[0] : $copilotModulesStore[idx]
+			const aiProvider = $copilotInfo.ai_provider as AiProviderTypes
 
 			copilotLoading = true
 			copilotStatus = "Generating code for step '" + module.id + "'..."
@@ -918,7 +920,8 @@
 						  })
 						: undefined,
 					isFirstInLoop,
-					abortController
+					abortController,
+					aiProvider
 				)
 				unsubscribe()
 			}
@@ -934,7 +937,7 @@
 						pastModule.value.type === 'script')
 				) {
 					const stepSchema: Schema = JSON.parse(JSON.stringify($flowStateStore[module.id].schema)) // deep copy
-					if (isHubStep && pastModule !== undefined && $copilotInfo.exists_openai_resource_path) {
+					if (isHubStep && pastModule !== undefined && $copilotInfo.exists_ai_resource) {
 						// ask AI to set step inputs
 						abortController = new AbortController()
 						const { inputs, allExprs } = await glueCopilot(
@@ -944,7 +947,8 @@
 								value: RawScript | PathScript
 							},
 							isFirstInLoop,
-							abortController
+							abortController,
+							aiProvider
 						)
 
 						// create flow inputs used by AI for autocompletion
@@ -992,11 +996,7 @@
 							$shouldUpdatePropertyType[key] = 'javascript'
 						})
 					} else {
-						if (
-							isHubStep &&
-							pastModule !== undefined &&
-							!$copilotInfo.exists_openai_resource_path
-						) {
+						if (isHubStep && pastModule !== undefined && !$copilotInfo.exists_ai_resource) {
 							sendUserToast(
 								'For better input generation, enable Windmill AI in the workspace settings',
 								true

frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte

@@ -221,7 +221,7 @@
 		JOBS_DISAPPROVAL: 'jobs.disapproval',
 		JOBS_DELETE: 'jobs.delete',
 		ACCOUNT_DELETE: 'account.delete',
-		OPENAI_REQUEST: 'openai.request',
+		AI_REQUEST: 'ai.request',
 		RESOURCES_CREATE: 'resources.create',
 		RESOURCES_UPDATE: 'resources.update',
 		RESOURCES_DELETE: 'resources.delete',

frontend/src/lib/components/copilot/CodeCompletionStatus.svelte

@@ -12,7 +12,7 @@
 	}
 </script>
 
-{#if $copilotInfo.exists_openai_resource_path && $copilotInfo.code_completion_enabled}
+{#if $copilotInfo.exists_ai_resource && $copilotInfo.code_completion_enabled}
 	<Toggle
 		size="xs"
 		bind:checked={$codeCompletionSessionEnabled}

frontend/src/lib/components/copilot/CronGen.svelte

@@ -1,11 +1,13 @@
 <script lang="ts">
 	import { ExternalLink, Wand2 } from 'lucide-svelte'
 	import Button from '../common/button/Button.svelte'
-	import { getNonStreamingCompletion } from './lib'
+	import { getNonStreamingCompletion, type AiProviderTypes } from './lib'
 	import Popup from '../common/popup/Popup.svelte'
 	import { sendUserToast } from '$lib/toast'
 	import { copilotInfo } from '$lib/stores'
+
 	import { base } from '$lib/base'
+	import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs'
 
 	export let schedule: string
 
@@ -14,31 +16,35 @@
 	let genLoading = false
 	let abortController = new AbortController()
 	$: instructionsField && setTimeout(() => instructionsField?.focus(), 100)
+
 	const SYSTEM =
 		"You are a helpful assistant for creating CRON schedules using both standard and extended Croner patterns. The structure is 'second minute hour dayOfMonth month dayOfWeek'. Supported modifiers: ? (wildcard), L (last day/weekday), # (nth occurrence of a weekday), and W (closest weekday). Weekdays are Sunday (1 or 7), Monday (2), Tuesday (3), Wednesday (4), Thursday (5), Friday (6), Saturday (7). Ensure syntax is valid, including optional seconds and special modifiers. You only return either the CRON string without any leading/closing quotes or an error message prefixed with 'ERROR:'."
 	const USER = 'CRON schedule instructions: {instructions}'
 	async function generateCron() {
 		genLoading = true
 		abortController = new AbortController()
+		const aiProvider = $copilotInfo.ai_provider
 		try {
+			const messages: ChatCompletionMessageParam[] = [
+				{
+					role: 'system',
+					content: SYSTEM
+				},
+				{
+					role: 'user',
+					content: USER.replace('{instructions}', instructions)
+				}
+			]
+
 			const response = await getNonStreamingCompletion(
-				[
-					{
-						role: 'system',
-						content: SYSTEM
-					},
-					{
-						role: 'user',
-						content: USER.replace('{instructions}', instructions)
-					}
-				],
-				abortController
+				messages,
+				abortController,
+				aiProvider as AiProviderTypes
 			)
 
 			if (response.startsWith('ERROR:')) {
 				throw response.replace('ERROR:', '').trim()
 			}
-
 			schedule = response
 		} catch (err) {
 			if (!abortController.signal.aborted) {
@@ -69,7 +75,7 @@
 			on:click={genLoading ? () => abortController?.abort() : undefined}
 		/>
 	</svelte:fragment>
-	{#if $copilotInfo.exists_openai_resource_path}
+	{#if $copilotInfo.exists_ai_resource}
 		<div class="flex w-96">
 			<input
 				bind:this={instructionsField}
@@ -104,7 +110,7 @@
 		<div class="block text-primary">
 			<p class="text-sm"
 				>Enable Windmill AI in the <a
-					href="{base}/workspace_settings?tab=openai"
+					href="{base}/workspace_settings?tab=ai"
 					target="_blank"
 					class="inline-flex flex-row items-center gap-1"
 					>workspace settings <ExternalLink size={16} /></a

frontend/src/lib/components/copilot/FlowCopilotStatus.svelte

@@ -31,7 +31,7 @@
 	$: copilotStatus && copilotPopover?.refresh()
 </script>
 
-{#if $copilotInfo.exists_openai_resource_path}
+{#if $copilotInfo.exists_ai_resource}
 	<ManualPopover bind:this={copilotPopover}>
 		<Button
 			size="xs"
@@ -137,7 +137,7 @@
 		<div class="block text-primary">
 			<p class="text-sm"
 				>Enable Windmill AI in the <a
-					href="{base}/workspace_settings?tab=openai"
+					href="{base}/workspace_settings?tab=ai"
 					target="_blank"
 					class="inline-flex flex-row items-center gap-1"
 					>workspace settings <ExternalLink size={16} /></a

frontend/src/lib/components/copilot/IteratorGen.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
 	import { Check, Loader2, Wand2 } from 'lucide-svelte'
 	import Button from '../common/button/Button.svelte'
-	import { getNonStreamingCompletion } from './lib'
+	import { getNonStreamingCompletion, type AiProviderTypes } from './lib'
 	import { sendUserToast } from '$lib/toast'
 	import type { Flow, InputTransform } from '$lib/gen'
 	import ManualPopover from '../ManualPopover.svelte'
@@ -62,7 +62,7 @@ ${YAML.stringify(availableData)}</available>
 Reply with the most probable answer, do not explain or discuss.
 Use javascript object dot notation to access the properties.
 Only output the expression, do not explain or discuss.`
-
+			const aiProvider = $copilotInfo.ai_provider
 			generatedContent = await getNonStreamingCompletion(
 				[
 					{
@@ -70,7 +70,8 @@ Only output the expression, do not explain or discuss.`
 						content: user
 					}
 				],
-				abortController
+				abortController,
+				aiProvider as AiProviderTypes
 			)
 		} catch (err) {
 			if (!abortController.signal.aborted) {
@@ -82,7 +83,7 @@ Only output the expression, do not explain or discuss.`
 	}
 
 	export function onKeyUp(event: KeyboardEvent) {
-		if (!$copilotInfo.exists_openai_resource_path || !$stepInputCompletionEnabled) {
+		if (!$copilotInfo.exists_ai_resource || !$stepInputCompletionEnabled) {
 			return
 		}
 		if (event.key === 'Tab') {
@@ -117,7 +118,7 @@ Only output the expression, do not explain or discuss.`
 		cancelOnOutOfFocus()
 	}
 
-	$: if ($copilotInfo.exists_openai_resource_path && $stepInputCompletionEnabled && focused) {
+	$: if ($copilotInfo.exists_ai_resource && $stepInputCompletionEnabled && focused) {
 		automaticGeneration()
 	}
 
@@ -131,7 +132,7 @@ Only output the expression, do not explain or discuss.`
 	let out = true // hack to prevent regenerating answer when accepting the answer due to mouseenter on new icon
 </script>
 
-{#if $copilotInfo.exists_openai_resource_path && $stepInputCompletionEnabled}
+{#if $copilotInfo.exists_ai_resource && $stepInputCompletionEnabled}
 	<ManualPopover showTooltip={!empty && generatedContent.length > 0} placement="bottom" class="p-2">
 		<Button
 			size="xs"

frontend/src/lib/components/copilot/MetadataGen.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import { getCompletion } from './lib'
+	import { getCompletion, getResponseFromEvent, type AiProviderTypes } from './lib'
 	import { isInitialCode } from '$lib/script_helpers'
 	import { Check, Loader2, Wand2 } from 'lucide-svelte'
 	import { copilotInfo, metadataCompletionEnabled } from '$lib/stores'
@@ -118,11 +118,11 @@ Generate a description for the flow below:
 					content: config.user.replace(`{${config.placeholderName}}`, placeholderContent)
 				}
 			]
-			const response = await getCompletion(messages, abortController)
+			const aiProvider = $copilotInfo.ai_provider as AiProviderTypes
+			const response = await getCompletion(messages, abortController, aiProvider)
 			generatedContent = ''
 			for await (const chunk of response) {
-				const toks = chunk.choices[0]?.delta?.content || ''
-				generatedContent += toks
+				generatedContent += getResponseFromEvent(chunk, aiProvider)
 			}
 		} catch (err) {
 			if (!abortController.signal.aborted) {
@@ -138,7 +138,7 @@ Generate a description for the flow below:
 	}
 
 	if (
-		$copilotInfo.exists_openai_resource_path &&
+		$copilotInfo.exists_ai_resource &&
 		$metadataCompletionEnabled &&
 		generateOnAppear &&
 		!content &&
@@ -158,7 +158,7 @@ Generate a description for the flow below:
 	}
 
 	$: active =
-		$copilotInfo.exists_openai_resource_path &&
+		$copilotInfo.exists_ai_resource &&
 		$metadataCompletionEnabled &&
 		!content &&
 		(loading || focused || !!generatedContent) &&
@@ -192,7 +192,7 @@ Generate a description for the flow below:
 	class={twMerge('relative', $$props.class)}
 	bind:clientWidth={width}
 	on:keydown={(event) => {
-		if (!$copilotInfo.exists_openai_resource_path || !$metadataCompletionEnabled) {
+		if (!$copilotInfo.exists_ai_resource || !$metadataCompletionEnabled) {
 			return
 		}
 		if (event.key === 'Tab') {

frontend/src/lib/components/copilot/PredicateGen.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
 	import { Wand2 } from 'lucide-svelte'
 	import Button from '../common/button/Button.svelte'
-	import { getNonStreamingCompletion } from './lib'
+	import { getNonStreamingCompletion, type AiProviderTypes } from './lib'
 	import { sendUserToast } from '$lib/toast'
 	import { createEventDispatcher, getContext } from 'svelte'
 	import type { FlowEditorContext } from '../flows/types'
@@ -59,7 +59,7 @@ Here's a summary of the available data:
 ${YAML.stringify(availableData)}</available>
 If the branching is made inside a for-loop, the iterator value is accessible as flow_input.iter.value
 Only return the expression without any wrapper. Do not explain or discuss.`
-
+			const aiProvider = $copilotInfo.ai_provider
 			const result = await getNonStreamingCompletion(
 				[
 					{
@@ -67,7 +67,8 @@ Only return the expression without any wrapper. Do not explain or discuss.`
 						content: user
 					}
 				],
-				abortController
+				abortController,
+				aiProvider as AiProviderTypes
 			)
 
 			dispatch('setExpr', result)
@@ -82,7 +83,7 @@ Only return the expression without any wrapper. Do not explain or discuss.`
 	}
 </script>
 
-{#if $copilotInfo.exists_openai_resource_path && $stepInputCompletionEnabled}
+{#if $copilotInfo.exists_ai_resource && $stepInputCompletionEnabled}
 	<Popup
 		floatingConfig={{ strategy: 'absolute', placement: 'bottom-end' }}
 		containerClasses="border rounded-lg shadow-lg p-4 bg-surface"

frontend/src/lib/components/copilot/RegexGen.svelte

@@ -2,7 +2,7 @@
 	import { base } from '$lib/base'
 	import { Button } from '../common'
 
-	import { getNonStreamingCompletion } from './lib'
+	import { getNonStreamingCompletion, type AiProviderTypes } from './lib'
 	import { sendUserToast } from '$lib/toast'
 	import Popup from '../common/popup/Popup.svelte'
 	import { copilotInfo } from '$lib/stores'
@@ -25,6 +25,7 @@
 		savePrompt()
 		genLoading = true
 		abortController = new AbortController()
+		const aiProvider = $copilotInfo.ai_provider
 		try {
 			const res = await getNonStreamingCompletion(
 				[
@@ -38,7 +39,8 @@
 						content: funcDesc
 					}
 				],
-				abortController
+				abortController,
+				aiProvider as AiProviderTypes
 			)
 			dispatch('gen', { res: res, prompt: funcDesc })
 			funcDesc = ''
@@ -114,7 +116,7 @@
 		/>
 	</svelte:fragment>
 	<div class="block text-primary z-[2000]">
-		{#if $copilotInfo.exists_openai_resource_path}
+		{#if $copilotInfo.exists_ai_resource}
 			<div class="flex flex-col gap-4">
 				<div class="flex w-96">
 					<input
@@ -167,7 +169,7 @@
 		{:else}
 			<p class="text-sm">
 				Enable Windmill AI in the <a
-					href="{base}/workspace_settings?tab=openai"
+					href="{base}/workspace_settings?tab=ai"
 					target="_blank"
 					class="inline-flex flex-row items-center gap-1"
 				>

frontend/src/lib/components/copilot/ScriptFix.svelte

@@ -2,7 +2,7 @@
 	import { base } from '$lib/base'
 	import { Button } from '../common'
 
-	import { SUPPORTED_LANGUAGES, copilot } from './lib'
+	import { SUPPORTED_LANGUAGES, copilot, type AiProviderTypes } from './lib'
 	import type { SupportedLanguage } from '$lib/common'
 	import { sendUserToast } from '$lib/toast'
 	import type Editor from '../Editor.svelte'
@@ -40,6 +40,7 @@
 		try {
 			genLoading = true
 			abortController = new AbortController()
+			const aiProvider = $copilotInfo.ai_provider as AiProviderTypes
 			await copilot(
 				{
 					language: lang,
@@ -51,7 +52,8 @@
 				},
 				generatedCode,
 				abortController,
-				generatedExplanation
+				aiProvider,
+				generatedExplanation,
 			)
 			setupDiff()
 			diffEditor?.setModified($generatedCode)
@@ -186,12 +188,12 @@
 					</Button>
 				</svelte:fragment>
 				{@const fixAction = (_) => {
-					if ($copilotInfo.exists_openai_resource_path) {
+					if ($copilotInfo.exists_ai_resource) {
 						onFix(() => close(null))
 					}
 				}}
 				<div use:fixAction>
-					{#if $copilotInfo.exists_openai_resource_path}
+					{#if $copilotInfo.exists_ai_resource}
 						<div class="w-[42rem] min-h-[3rem] max-h-[34rem] overflow-y-scroll">
 							{#if $generatedCode.length > 0}
 								<div class="overflow-x-scroll">
@@ -211,7 +213,7 @@
 							<p class="text-sm"
 								>Enable Windmill AI in the <a
 									class="inline-flex flex-row items-center gap-1"
-									href="{base}/workspace_settings?tab=openai"
+									href="{base}/workspace_settings?tab=ai"
 									target="_blank">workspace settings</a
 								></p
 							></div

frontend/src/lib/components/copilot/ScriptGen.svelte

@@ -2,7 +2,13 @@
 	import { base } from '$lib/base'
 	import { Button } from '../common'
 
-	import { MAX_SCHEMA_LENGTH, SUPPORTED_LANGUAGES, addThousandsSeparator, copilot } from './lib'
+	import {
+		MAX_SCHEMA_LENGTH,
+		SUPPORTED_LANGUAGES,
+		addThousandsSeparator,
+		copilot,
+		type AiProviderTypes
+	} from './lib'
 	import type { SupportedLanguage } from '$lib/common'
 	import { sendUserToast } from '$lib/toast'
 	import type Editor from '../Editor.svelte'
@@ -68,6 +74,7 @@
 			return
 		}
 		savePrompt()
+		const aiProvider = $copilotInfo.ai_provider as AiProviderTypes
 		try {
 			genLoading = true
 			blockPopupOpen = true
@@ -84,7 +91,8 @@
 						workspace: $workspaceStore!
 					},
 					generatedCode,
-					abortController
+					abortController,
+					aiProvider
 				)
 			} else {
 				await copilot(
@@ -97,7 +105,8 @@
 						workspace: $workspaceStore!
 					},
 					generatedCode,
-					abortController
+					abortController,
+					aiProvider
 				)
 			}
 			setupDiff()
@@ -388,7 +397,7 @@
 						<LoadingIcon />
 					{/if}
 				</div>
-			{:else if $copilotInfo.exists_openai_resource_path}
+			{:else if $copilotInfo.exists_ai_resource}
 				<div class="flex flex-col gap-4">
 					<div class="flex flex-row justify-between items-center">
 						<ToggleButtonGroup class="w-auto shrink-0" bind:selected={mode}>
@@ -397,7 +406,14 @@
 						</ToggleButtonGroup>
 
 						<div class="text-[0.6rem] text-secondary opacity-60 flex flex-row items-center gap-0.5">
-							GPT-4o<Bot size={14} />
+							{#if $copilotInfo.ai_provider === 'openai'}
+								GPT-4o
+							{:else if $copilotInfo.ai_provider === 'anthropic'}
+								Claude-3.5
+							{:else}
+								Codestral
+							{/if}
+							<Bot size={14} />
 						</div>
 					</div>
 					<div class="flex w-96 items-start">
@@ -493,7 +509,7 @@
 			{:else}
 				<p class="text-sm">
 					Enable Windmill AI in the <a
-						href="{base}/workspace_settings?tab=openai"
+						href="{base}/workspace_settings?tab=ai"
 						target="_blank"
 						class="inline-flex flex-row items-center gap-1"
 					>

frontend/src/lib/components/copilot/StepGen.svelte

@@ -70,7 +70,7 @@
 	}
 
 	async function onGenerate() {
-		if (!selectedCompletion && !$copilotInfo.exists_openai_resource_path) {
+		if (!selectedCompletion && !$copilotInfo.exists_ai_resource) {
 			sendUserToast(
 				'Windmill AI is not enabled, you can activate it in the workspace settings',
 				true

frontend/src/lib/components/copilot/StepInputGen.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
 	import { Check, Loader2, Wand2 } from 'lucide-svelte'
 	import Button from '../common/button/Button.svelte'
-	import { getNonStreamingCompletion } from './lib'
+	import { getNonStreamingCompletion, type AiProviderTypes } from './lib'
 	import { sendUserToast } from '$lib/toast'
 	import type { Flow, InputTransform } from '$lib/gen'
 	import ManualPopover from '../ManualPopover.svelte'
@@ -107,7 +107,7 @@ If none of the available results are appropriate, are already used or are more a
 Reply with the most probable answer, do not explain or discuss.
 Use javascript object dot notation to access the properties.
 Only return the expression without any wrapper.`
-
+			const aiProvider = $copilotInfo.ai_provider
 			generatedContent = await getNonStreamingCompletion(
 				[
 					{
@@ -115,7 +115,8 @@ Only return the expression without any wrapper.`
 						content: user
 					}
 				],
-				abortController
+				abortController,
+				aiProvider as AiProviderTypes
 			)
 
 			if (
@@ -138,7 +139,7 @@ Only return the expression without any wrapper.`
 	}
 
 	export function onKeyUp(event: KeyboardEvent) {
-		if (!$copilotInfo.exists_openai_resource_path || !$stepInputCompletionEnabled) {
+		if (!$copilotInfo.exists_ai_resource || !$stepInputCompletionEnabled) {
 			return
 		}
 		if (event.key === 'Tab') {
@@ -181,7 +182,7 @@ Only return the expression without any wrapper.`
 		cancelOnOutOfFocus()
 	}
 
-	$: if ($copilotInfo.exists_openai_resource_path && $stepInputCompletionEnabled && focused) {
+	$: if ($copilotInfo.exists_ai_resource && $stepInputCompletionEnabled && focused) {
 		automaticGeneration()
 	}
 
@@ -193,7 +194,7 @@ Only return the expression without any wrapper.`
 	let openInputsModal = false
 </script>
 
-{#if $copilotInfo.exists_openai_resource_path && $stepInputCompletionEnabled}
+{#if $copilotInfo.exists_ai_resource && $stepInputCompletionEnabled}
 	<FlowCopilotInputsModal
 		on:confirmed={async () => {
 			createFlowInput()

frontend/src/lib/components/copilot/StepInputsGen.svelte

@@ -7,7 +7,7 @@
 	import type { FlowEditorContext } from '../flows/types'
 	import type { PickableProperties } from '../flows/previousResults'
 	import { getContext } from 'svelte'
-	import { getNonStreamingCompletion } from './lib'
+	import { getNonStreamingCompletion, type AiProviderTypes } from './lib'
 	import { sendUserToast } from '$lib/toast'
 	import Button from '../common/button/Button.svelte'
 	import type { FlowCopilotContext } from './flow'
@@ -83,7 +83,7 @@ Your answer has to be in the following format (one line per input):
 input_name1: expression1
 input_name2: expression2
 ...`
-
+			const aiProvider = $copilotInfo.ai_provider
 			generatedContent = await getNonStreamingCompletion(
 				[
 					{
@@ -91,7 +91,8 @@ input_name2: expression2
 						content: user
 					}
 				],
-				abortController
+				abortController,
+				aiProvider as AiProviderTypes
 			)
 
 			parsedInputs = generatedContent.split('\n').map((x) => x.split(': '))
@@ -168,7 +169,7 @@ input_name2: expression2
 </script>
 
 <div class="flex flex-row justify-end">
-	{#if $copilotInfo.exists_openai_resource_path && $stepInputCompletionEnabled}
+	{#if $copilotInfo.exists_ai_resource && $stepInputCompletionEnabled}
 		<FlowCopilotInputsModal
 			on:confirmed={async () => {
 				createFlowInputs()
@@ -236,10 +237,10 @@ input_name2: expression2
 				</Button>
 			</svelte:fragment>
 			<p class="text-sm">
-				{#if !$copilotInfo.exists_openai_resource_path}
+				{#if !$copilotInfo.exists_ai_resource}
 					Enable Windmill AI in the{' '}
 					<a
-						href="{base}/workspace_settings?tab=openai"
+						href="{base}/workspace_settings?tab=ai"
 						target="_blank"
 						class="inline-flex flex-row items-center gap-1"
 					>

frontend/src/lib/components/copilot/TestAiKey.svelte

@@ -1,9 +1,10 @@
 <script lang="ts">
 	import { sendUserToast } from '$lib/toast'
 	import Button from '../common/button/Button.svelte'
-	import { testKey } from './lib'
+	import { testKey, type AiProviderTypes } from './lib'
 	export let disabled = false
 	export let apiKey: string | undefined = undefined
+	export let aiProvider: AiProviderTypes
 	let loading = false
 </script>
 
@@ -28,7 +29,8 @@
 						content: "this is a test, simply reply with 'ok'"
 					}
 				],
-				abortController
+				abortController,
+				aiProvider
 			})
 			sendUserToast('Valid key')
 		} catch (err) {

frontend/src/lib/components/copilot/completion.ts

@@ -1,5 +1,5 @@
 import type { ChatCompletionMessageParam } from 'openai/resources/chat/index.mjs'
-import { getNonStreamingCompletion } from './lib'
+import { getNonStreamingCompletion, type AiProviderTypes } from './lib'
 import { codeCompletionLoading } from '$lib/stores'
 
 const systemPrompt = `You are a code completion assistant, return the code that should go instead of the <completion_tokens>.
@@ -73,7 +73,8 @@ export async function editorCodeCompletion(
 	before: string,
 	after: string,
 	lang: string,
-	abortController: AbortController
+	abortController: AbortController,
+	aiProvider: AiProviderTypes
 ) {
 	codeCompletionLoading.set(true)
 	const messages: ChatCompletionMessageParam[] = [
@@ -91,7 +92,7 @@ export async function editorCodeCompletion(
 	]
 
 	try {
-		const result = await getNonStreamingCompletion(messages, abortController)
+		const result = await getNonStreamingCompletion(messages, abortController, aiProvider)
 
 		const match = result.match(/```[a-zA-Z]+\n([\s\S]*?)\n```/)
 

frontend/src/lib/components/copilot/flow.ts

@@ -6,7 +6,12 @@ import {
 	type InputTransform,
 	type Script
 } from '$lib/gen'
-import { addResourceTypes, deltaCodeCompletion, getNonStreamingCompletion } from './lib'
+import {
+	addResourceTypes,
+	deltaCodeCompletion,
+	getNonStreamingCompletion,
+	type AiProviderTypes
+} from './lib'
 import type { Writable } from 'svelte/store'
 import type Editor from '../Editor.svelte'
 import type { Drawer } from '../common'
@@ -14,7 +19,7 @@ import { scriptLangToEditorLang } from '$lib/scripts'
 
 export type FlowCopilotModule = {
 	id: string
-	type: 'trigger' | 'script' 
+	type: 'trigger' | 'script'
 	description: string
 	code: string
 	source: 'hub' | 'custom' | undefined
@@ -106,7 +111,6 @@ To maintain state across runs, you can use get_state() and set_state(value) whic
 {additionalInformation}`
 }
 
-
 // const preprocessorPrompts: {
 // 	bun: string
 // 	python3: string
@@ -128,7 +132,7 @@ To maintain state across runs, you can use get_state() and set_state(value) whic
 // 			headers: Record<string, string>
 // 		}
 // 	},
-// 	/* your other args */ 
+// 	/* your other args */
 // ) {
 // 	return {
 // 		// return the args to be passed to the flow
@@ -168,7 +172,6 @@ To maintain state across runs, you can use get_state() and set_state(value) whic
 // {additionalInformation}`
 // }
 
-
 const firstActionPrompt = `I'm building a workflow which is a sequence of script steps. Write a script in {codeLang} which should {description}.
 Return the script's output.
 
@@ -255,7 +258,8 @@ export async function stepCopilot(
 		  })
 		| undefined,
 	isFirstInLoop: boolean,
-	abortController: AbortController
+	abortController: AbortController,
+	aiProvider: AiProviderTypes
 ) {
 	if (module.source !== 'custom') {
 		throw new Error('Not a custom module')
@@ -265,9 +269,9 @@ export async function stepCopilot(
 	let prompt =
 		module.type === 'trigger'
 			? triggerPrompts[lang]
-			// : module.type === 'preprocessor'
+			: // : module.type === 'preprocessor'
 			// 	? preprocessorPrompts[lang]
-			: pastModule === undefined
+			pastModule === undefined
 			? firstActionPrompt
 			: isFirstInLoop
 			? loopActionPrompt
@@ -310,7 +314,8 @@ export async function stepCopilot(
 			}
 		],
 		deltaCodeStore,
-		abortController
+		abortController,
+		aiProvider
 	)
 	return code
 }
@@ -322,7 +327,8 @@ export async function glueCopilot(
 		value: RawScript | PathScript
 	},
 	isFirstInLoop: boolean,
-	abortController: AbortController
+	abortController: AbortController,
+	aiProvider: AiProviderTypes
 ) {
 	const { prevCode, prevLang } = await getPreviousStepContent(pastModule, workspace)
 
@@ -357,7 +363,8 @@ export async function glueCopilot(
 					)
 			}
 		],
-		abortController
+		abortController,
+		aiProvider
 	)
 
 	const matches = response.matchAll(/([a-zA-Z_0-9.]+): (.+)/g)

frontend/src/lib/components/copilot/lib.ts

@@ -1,42 +1,147 @@
 import { OpenAI } from 'openai'
 import { OpenAPI, ResourceService, type Script } from '../../gen'
 import type { Writable } from 'svelte/store'
-
+import { Anthropic } from '@anthropic-ai/sdk'
 import type { DBSchema, GraphqlSchema, SQLSchema } from '$lib/stores'
 import { formatResourceTypes } from './utils'
-
 import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts'
-
+import { Mistral } from '@mistralai/mistralai'
 import { buildClientSchema, printSchema } from 'graphql'
 import type {
 	ChatCompletionCreateParamsStreaming,
 	ChatCompletionMessageParam
 } from 'openai/resources/index.mjs'
 
+import type { MessageCreateParams, MessageParam } from '@anthropic-ai/sdk/resources/messages.mjs'
+import type { ChatCompletionRequest } from '@mistralai/mistralai/models/components/chatcompletionrequest'
+import type {
+	SystemMessage,
+	UserMessage,
+	AssistantMessage,
+	ToolMessage,
+	CompletionEvent,
+	ContentChunk
+} from '@mistralai/mistralai/models/components'
+
 export const SUPPORTED_LANGUAGES = new Set(Object.keys(GEN_CONFIG.prompts))
 
-const openaiConfig: ChatCompletionCreateParamsStreaming = {
-	temperature: 0,
-	max_tokens: 16384,
-	model: 'gpt-4o-2024-08-06',
-	seed: 42,
-	stream: true,
-	messages: []
+export type AiProviderTypes = 'openai' | 'anthropic' | 'mistral'
+
+interface AiProvider {
+	init: (workspace: string, updateClient: boolean, token?: string) => void
 }
 
-class WorkspacedOpenai {
+class WorkspacedMistral implements AiProvider {
+	private client: Mistral | undefined
+
+	init(workspace: string, updateClient: boolean, token?: string) {
+		if (!this.client || updateClient) {
+			this.client = initWorkspaceAiProvider(workspace, 'mistral', token) as unknown as Mistral
+		}
+	}
+
+	getClient() {
+		if (!this.client) {
+			throw new Error('AnthropicAi not initialized')
+		}
+		return this.client
+	}
+}
+
+export namespace MistralAi {
+	export let workspace = new WorkspacedMistral()
+
+	export const mistralConfig: ChatCompletionRequest = {
+		temperature: 0,
+		model: null,
+		maxTokens: 32000,
+		messages: []
+	}
+
+	export type MistralParamsMessage =
+		| (SystemMessage & { role: 'system' })
+		| (UserMessage & { role: 'user' })
+		| (AssistantMessage & { role: 'assistant' })
+		| (ToolMessage & { role: 'tool' })
+
+	export function retrieveTextValue(chunks: string | ContentChunk[] | null | undefined): string {
+		let response = ''
+		if (Array.isArray(chunks)) {
+			for (const chunk of chunks) {
+				if (chunk.type === 'text') {
+					response += chunk.text
+				}
+			}
+			return response
+		}
+		return chunks as string
+	}
+}
+
+class WorkspacedAnthropic implements AiProvider {
+	private client: Anthropic | undefined
+
+	init(workspace: string, updateClient: boolean, token: string | undefined = undefined) {
+		if (!this.client || updateClient) {
+			this.client = initWorkspaceAiProvider(workspace, 'anthropic', token) as unknown as Anthropic
+		}
+	}
+
+	getClient() {
+		if (!this.client) {
+			throw new Error('AnthropicAi not initialized')
+		}
+		return this.client
+	}
+}
+
+export namespace AnthropicAi {
+	export let workspace = new WorkspacedAnthropic()
+
+	export const config: MessageCreateParams = {
+		temperature: 0,
+		max_tokens: 8192,
+		model: 'claude-3-5-sonnet-20241022',
+		messages: []
+	}
+
+	export function getSystemPromptAndArrayMessages(
+		messages: ChatCompletionMessageParam[]
+	): [string, MessageParam[]] {
+		let system: string | undefined = undefined
+		if (messages[0].role == 'system') {
+			system = messages[0].content as string
+			messages.shift()
+		}
+		const anthropicMessages: MessageParam[] = messages.map((message) => {
+			return {
+				role: message.role == 'user' ? 'user' : 'assistant',
+				content: message.content as string
+			}
+		})
+		return [system as string, anthropicMessages ?? []]
+	}
+
+	export function retrieveTextValue(part: Anthropic.Messages.RawMessageStreamEvent) {
+		let response = ''
+		if (part.type == 'content_block_delta') {
+			if (part.delta.type == 'text_delta') {
+				response = part.delta.text
+			} else {
+				response = part.delta.partial_json
+			}
+		}
+		return response
+	}
+}
+
+class WorkspacedOpenai implements AiProvider {
 	private client: OpenAI | undefined
 
-	init(workspace: string, token: string | undefined = undefined) {
-		const baseURL = `${location.origin}${OpenAPI.BASE}/w/${workspace}/openai/proxy`
-		this.client = new OpenAI({
-			baseURL,
-			apiKey: 'fakekey',
-			defaultHeaders: {
-				Authorization: token ? `Bearer ${token}` : ''
-			},
-			dangerouslyAllowBrowser: true
-		})
+	init(workspace: string, updateClient: boolean, token: string | undefined = undefined) {
+		if (!this.client || updateClient) {
+			this.client = initWorkspaceAiProvider(workspace, 'openai', token) as unknown as OpenAI
+		}
 	}
 
 	getClient() {
@@ -47,34 +152,141 @@ class WorkspacedOpenai {
 	}
 }
 
-export let workspacedOpenai = new WorkspacedOpenai()
+export namespace OpenAi {
+	export let workspace = new WorkspacedOpenai()
+
+	export const openaiConfig: ChatCompletionCreateParamsStreaming = {
+		temperature: 0,
+		max_tokens: 16384,
+		model: 'gpt-4o-2024-08-06',
+		seed: 42,
+		stream: true,
+		messages: []
+	}
+
+	export function retrieveTextValue(part: OpenAI.Chat.Completions.ChatCompletionChunk) {
+		return part.choices[0]?.delta?.content || ''
+	}
+}
+
+export function initAllAiWorkspace(workspace: string, updateClient: boolean = false) {
+	OpenAi.workspace.init(workspace, updateClient)
+	AnthropicAi.workspace.init(workspace, updateClient)
+	MistralAi.workspace.init(workspace, updateClient)
+}
+
+function initWorkspaceAiProvider(
+	workspace: string,
+	aiProvider: AiProviderTypes,
+	token: string | undefined = undefined
+): Anthropic | OpenAI | Mistral {
+	const baseURL = `${location.origin}${OpenAPI.BASE}/w/${workspace}/ai/proxy`
+	let client
+	switch (aiProvider) {
+		case 'openai': {
+			client = new OpenAI({
+				baseURL,
+				apiKey: 'fake-key',
+				defaultHeaders: {
+					Authorization: token ? `Bearer ${token}` : ''
+				},
+				dangerouslyAllowBrowser: true
+			})
+			break
+		}
+		case 'anthropic': {
+			client = new Anthropic({
+				baseURL,
+				apiKey: 'fake-key',
+				defaultHeaders: {
+					Authorization: token ? `Bearer ${token}` : ''
+				},
+				dangerouslyAllowBrowser: true
+			})
+			break
+		}
+		case 'mistral': {
+			client = new Mistral({
+				serverURL: baseURL
+			})
+		}
+	}
+	return client
+}
 
 export async function testKey({
 	apiKey,
 	abortController,
-	messages
+	messages,
+	aiProvider
 }: {
 	apiKey?: string
 	messages: ChatCompletionMessageParam[]
 	abortController: AbortController
+	aiProvider: AiProviderTypes
 }) {
 	if (apiKey) {
-		const openai = new OpenAI({
-			apiKey,
-			dangerouslyAllowBrowser: true
-		})
-		await openai.chat.completions.create(
-			{
-				...openaiConfig,
-				messages,
-				stream: false
-			},
-			{
-				signal: abortController.signal
+		switch (aiProvider) {
+			case 'openai': {
+				const openai = new OpenAI({
+					apiKey,
+					dangerouslyAllowBrowser: true
+				})
+				await openai.chat.completions.create(
+					{
+						...OpenAi.openaiConfig,
+						messages,
+						stream: false
+					},
+					{
+						signal: abortController.signal
+					}
+				)
+				break
 			}
-		)
+			case 'anthropic': {
+				const anthropic = new Anthropic({
+					apiKey,
+					dangerouslyAllowBrowser: true
+				})
+				const [, anthropicMessages] = AnthropicAi.getSystemPromptAndArrayMessages(messages)
+				await anthropic.messages.create(
+					{
+						...AnthropicAi.config,
+						messages: anthropicMessages,
+						stream: false
+					},
+					{
+						signal: abortController.signal
+					}
+				)
+				break
+			}
+			case 'mistral': {
+				const mistral = new Mistral({
+					apiKey
+				})
+				await mistral.chat.complete(
+					{
+						...MistralAi.mistralConfig,
+						model: 'codestral-latest',
+						stream: false,
+						messages: messages as MistralAi.MistralParamsMessage[]
+					},
+					{
+						fetchOptions: {
+							signal: abortController.signal,
+							headers: {
+								'content-type': 'application/json'
+							}
+						}
+					}
+				)
+				break
+			}
+		}
 	} else {
-		await getNonStreamingCompletion(messages, abortController, undefined, true)
+		await getNonStreamingCompletion(messages, abortController, aiProvider, undefined, true)
 	}
 }
 
@@ -255,57 +467,154 @@ const PROMPTS_CONFIGS = {
 export async function getNonStreamingCompletion(
 	messages: ChatCompletionMessageParam[],
 	abortController: AbortController,
-	model = openaiConfig.model,
+	aiProvider: AiProviderTypes,
+	model = OpenAi.openaiConfig.model,
 	noCache?: boolean
 ) {
-	const openaiClient = workspacedOpenai.getClient()
-	const completion = await openaiClient.chat.completions.create(
-		{
-			...openaiConfig,
-			messages,
-			stream: false,
-			model
+	let response: string | undefined = ''
+	const queryOptions = {
+		query: {
+			no_cache: noCache
 		},
-		{
-			query: {
-				no_cache: noCache
-			},
-			signal: abortController.signal
+		signal: abortController.signal
+	}
+	switch (aiProvider) {
+		case 'openai': {
+			const openaiClient = OpenAi.workspace.getClient()
+			const completion = await openaiClient.chat.completions.create(
+				{
+					...OpenAi.openaiConfig,
+					messages,
+					stream: false,
+					model
+				},
+				queryOptions
+			)
+			response = completion.choices[0]?.message.content || ''
+			break
 		}
-	)
-
-	// if (completion.usage) {
-	// 	const { prompt_tokens, completion_tokens } = completion.usage
-	// 	console.log('Cost: ', (prompt_tokens * 0.0015 + completion_tokens * 0.002) / 1000)
-	// }
-
-	return completion.choices[0]?.message.content || ''
+		case 'anthropic': {
+			const anthropicClient = AnthropicAi.workspace.getClient()
+			const [system, anthropicMessages] = AnthropicAi.getSystemPromptAndArrayMessages(messages)
+			const message = await anthropicClient.messages.create(
+				{
+					...AnthropicAi.config,
+					system,
+					messages: anthropicMessages,
+					stream: false
+				},
+				queryOptions
+			)
+			response = message.content[0].type === 'text' ? message.content[0].text : ''
+			break
+		}
+		case 'mistral': {
+			const mistralClient = MistralAi.workspace.getClient()
+			const message = await mistralClient.chat.complete(
+				{
+					...MistralAi.mistralConfig,
+					model: 'codestral-latest',
+					stream: false,
+					messages: messages as MistralAi.MistralParamsMessage[]
+				},
+				{
+					fetchOptions: {
+						signal: abortController.signal,
+						cache: 'no-store'
+					}
+				}
+			)
+			response = MistralAi.retrieveTextValue(message.choices && message.choices[0].message.content)
+			break
+		}
+	}
+	return response
 }
 
 export async function getCompletion(
 	messages: ChatCompletionMessageParam[],
 	abortController: AbortController,
-	model = openaiConfig.model
+	aiProvider: AiProviderTypes,
+	model = OpenAi.openaiConfig.model
 ) {
-	const openaiClient = workspacedOpenai.getClient()
-	const completion = await openaiClient.chat.completions.create(
-		{
-			...openaiConfig,
-			messages,
-			model
-		},
-		{
-			signal: abortController.signal
-		}
-	)
+	switch (aiProvider) {
+		case 'anthropic': {
+			const anthropicClient = AnthropicAi.workspace.getClient()
+			const [system, anthropicMessages] = AnthropicAi.getSystemPromptAndArrayMessages(messages)
 
-	return completion
+			const completion = await anthropicClient.messages.create(
+				{
+					...AnthropicAi.config,
+					system,
+					messages: anthropicMessages,
+					stream: true
+				},
+				{ signal: abortController.signal }
+			)
+			return completion
+		}
+		case 'openai': {
+			const openaiClient = OpenAi.workspace.getClient()
+			const completion = await openaiClient.chat.completions.create(
+				{
+					...OpenAi.openaiConfig,
+					messages,
+					model
+				},
+				{
+					signal: abortController.signal
+				}
+			)
+			return completion
+		}
+		case 'mistral': {
+			const mistralClient = MistralAi.workspace.getClient()
+			const message = await mistralClient.chat.stream(
+				{
+					...MistralAi.mistralConfig,
+					model: 'codestral-latest',
+					messages: messages as MistralAi.MistralParamsMessage[]
+				},
+				{
+					fetchOptions: {
+						signal: abortController.signal,
+						cache: 'no-store'
+					}
+				}
+			)
+			return message
+		}
+	}
+}
+
+export function getResponseFromEvent(
+	part:
+		| Anthropic.Messages.RawMessageStreamEvent
+		| OpenAI.Chat.Completions.ChatCompletionChunk
+		| CompletionEvent,
+	aiProvider: AiProviderTypes
+): string {
+	switch (aiProvider) {
+		case 'openai': {
+			const messages = part as OpenAI.Chat.Completions.ChatCompletionChunk
+			return OpenAi.retrieveTextValue(messages)
+		}
+		case 'anthropic': {
+			const messages = part as Anthropic.Messages.RawMessageStreamEvent
+			return AnthropicAi.retrieveTextValue(messages)
+		}
+		case 'mistral': {
+			const messages = part as CompletionEvent
+			return MistralAi.retrieveTextValue(messages.data.choices[0].delta.content)
+		}
+	}
 }
 
 export async function copilot(
 	scriptOptions: CopilotOptions,
 	generatedCode: Writable<string>,
 	abortController: AbortController,
+	aiProvider: AiProviderTypes,
 	generatedExplanation?: Writable<string>
 ) {
 	const { prompt, systemPrompt } = await getPrompts(scriptOptions)
@@ -321,13 +630,14 @@ export async function copilot(
 				content: prompt
 			}
 		],
-		abortController
+		abortController,
+		aiProvider
 	)
 
 	let response = ''
 	let code = ''
 	for await (const part of completion) {
-		response += part.choices[0]?.delta?.content || ''
+		response += getResponseFromEvent(part, aiProvider)
 		let match = response.match(/```[a-zA-Z]+\n([\s\S]*?)\n```/)
 
 		if (match) {
@@ -393,15 +703,16 @@ function getStringEndDelta(prev: string, now: string) {
 export async function deltaCodeCompletion(
 	messages: ChatCompletionMessageParam[],
 	generatedCodeDelta: Writable<string>,
-	abortController: AbortController
+	abortController: AbortController,
+	aiProvider: AiProviderTypes
 ) {
-	const completion = await getCompletion(messages, abortController)
+	const completion = await getCompletion(messages, abortController, aiProvider)
 
 	let response = ''
 	let code = ''
 	let delta = ''
 	for await (const part of completion) {
-		response += part.choices[0]?.delta?.content || ''
+		response += getResponseFromEvent(part, aiProvider)
 		let match = response.match(/```[a-zA-Z]+\n([\s\S]*?)\n```/)
 
 		if (match) {

frontend/src/lib/components/flows/content/FlowInputsQuick.svelte

@@ -87,7 +87,7 @@
 	}
 
 	async function onGenerate() {
-		if (!selectedCompletion && !$copilotInfo.exists_openai_resource_path) {
+		if (!selectedCompletion && !$copilotInfo.exists_ai_resource) {
 			sendUserToast(
 				'Windmill AI is not enabled, you can activate it in the workspace settings',
 				true

frontend/src/lib/components/flows/map/FlowCopilotButton.svelte

@@ -20,7 +20,7 @@
 			<Menu pointerDown noMinW placement="bottom-center" let:close bind:show={openNoCopilot}>
 				<button
 					title="AI Flow Builder"
-					on:pointerdown={$copilotInfo.exists_openai_resource_path
+					on:pointerdown={$copilotInfo.exists_ai_resource
 						? (ev) => {
 								ev.preventDefault()
 								ev.stopPropagation()
@@ -33,12 +33,12 @@
 				>
 					<Wand2 size={12} />
 				</button>
-				{#if !$copilotInfo.exists_openai_resource_path}
+				{#if !$copilotInfo.exists_ai_resource}
 					<div class="text-primary p-4">
 						<p class="text-sm w-80">
 							Enable Windmill AI in the
 							<a
-								href="{base}/workspace_settings?tab=openai"
+								href="{base}/workspace_settings?tab=ai"
 								target="_blank"
 								class="inline-flex flex-row items-center gap-1"
 								on:click={() => {

frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte

@@ -91,10 +91,7 @@
 				kind
 			)
 		} else if (kind == 'forloop') {
-			;[module, state] = await createLoop(
-				module.id,
-				!disableAi && $copilotInfo.exists_openai_resource_path
-			)
+			;[module, state] = await createLoop(module.id, !disableAi && $copilotInfo.exists_ai_resource)
 		} else if (kind == 'whileloop') {
 			;[module, state] = await createWhileLoop(module.id)
 		} else if (kind == 'branchone') {

frontend/src/lib/components/sidebar/WorkspaceMenu.svelte

@@ -19,6 +19,7 @@
 	import MenuButton from './MenuButton.svelte'
 	import { MenuItem } from '@rgossiaux/svelte-headlessui'
 	import { isCloudHosted } from '$lib/cloud'
+	import { initAllAiWorkspace } from '../copilot/lib'
 
 	export let isCollapsed: boolean = false
 
@@ -26,7 +27,7 @@
 		if ($workspaceStore === id) {
 			return
 		}
-
+		initAllAiWorkspace(id, true)
 		const editPages = [
 			'/scripts/edit/',
 			'/flows/edit/',

frontend/src/lib/stores.ts

@@ -70,10 +70,12 @@ export const userWorkspaces: Readable<
 	}
 })
 export const copilotInfo = writable<{
-	exists_openai_resource_path: boolean
+	ai_provider: string
+	exists_ai_resource: boolean
 	code_completion_enabled: boolean
 }>({
-	exists_openai_resource_path: false,
+	ai_provider: '',
+	exists_ai_resource: false,
 	code_completion_enabled: false
 })
 export const codeCompletionLoading = writable<boolean>(false)

frontend/src/routes/(root)/(logged)/+layout.svelte

@@ -43,7 +43,7 @@
 	import { syncTutorialsTodos } from '$lib/tutorialUtils'
 	import { ArrowLeft, Search } from 'lucide-svelte'
 	import { getUserExt } from '$lib/user'
-	import { workspacedOpenai } from '$lib/components/copilot/lib'
+	import { initAllAiWorkspace } from '$lib/components/copilot/lib'
 	import { twMerge } from 'tailwind-merge'
 	import OperatorMenu from '$lib/components/sidebar/OperatorMenu.svelte'
 	import GlobalSearchModal from '$lib/components/search/GlobalSearchModal.svelte'
@@ -230,12 +230,14 @@
 	let devOnly = $page.url.pathname.startsWith(base + '/scripts/dev')
 
 	async function loadCopilot(workspace: string) {
-		workspacedOpenai.init(workspace)
+		initAllAiWorkspace(workspace)
 		try {
 			copilotInfo.set(await WorkspaceService.getCopilotInfo({ workspace }))
 		} catch (err) {
+			console.log(err)
 			copilotInfo.set({
-				exists_openai_resource_path: false,
+				ai_provider: '',
+				exists_ai_resource: false,
 				code_completion_enabled: false
 			})
 			console.error('Could not get copilot info')

frontend/src/routes/(root)/(logged)/user/(user)/create_workspace/+page.svelte

@@ -18,11 +18,12 @@
 	import Tooltip from '$lib/components/Tooltip.svelte'
 	import { onMount } from 'svelte'
 	import { sendUserToast } from '$lib/toast'
-	import TestOpenaiKey from '$lib/components/copilot/TestOpenaiKey.svelte'
+	import TestAiKey from '$lib/components/copilot/TestAiKey.svelte'
 	import { switchWorkspace } from '$lib/storeUtils'
 	import { isCloudHosted } from '$lib/cloud'
 	import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte'
 	import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte'
+	import type { AiProviderTypes } from '$lib/components/copilot/lib'
 
 	const rd = $page.url.searchParams.get('rd')
 
@@ -32,7 +33,7 @@
 
 	let errorId = ''
 	let errorUser = ''
-	let openAiKey = ''
+	let aiKey = ''
 	let codeCompletionEnabled = true
 	let checking = false
 
@@ -68,7 +69,7 @@
 				requestBody: { operator: operatorOnly, invite_all: !isCloudHosted(), auto_add: true }
 			})
 		}
-		if (openAiKey != '') {
+		if (aiKey != '') {
 			let actualUsername = username
 			if (automateUsernameCreation) {
 				const user = await UserService.whoami({
@@ -76,14 +77,14 @@
 				})
 				actualUsername = user.username
 			}
-			let path = `u/${actualUsername}/openai_windmill_codegen`
+			let path = `u/${actualUsername}/${selected}_windmill_codegen`
 			await VariableService.createVariable({
 				workspace: id,
 				requestBody: {
 					path,
-					value: openAiKey,
+					value: aiKey,
 					is_secret: true,
-					description: 'Token for openai'
+					description: 'Ai token'
 				}
 			})
 			await ResourceService.createResource({
@@ -93,12 +94,15 @@
 					value: {
 						api_key: '$var:' + path
 					},
-					resource_type: 'openai'
+					resource_type: selected
 				}
 			})
 			await WorkspaceService.editCopilotConfig({
 				workspace: id,
-				requestBody: { openai_resource_path: path, code_completion_enabled: codeCompletionEnabled }
+				requestBody: {
+					ai_resource: { path, provider: selected },
+					code_completion_enabled: codeCompletionEnabled
+				}
 			})
 		}
 
@@ -164,6 +168,7 @@
 
 	let auto_invite = false
 	let operatorOnly = false
+	let selected: AiProviderTypes = 'openai'
 </script>
 
 <CenteredModal title="New Workspace">
@@ -191,30 +196,38 @@
 		</label>
 	{/if}
 	<label class="block pb-4">
-		<span class="text-secondary text-sm">
-			OpenAI key for Windmill AI
-			<Tooltip>
-				Find out how it can help you <a
-					href="https://www.windmill.dev/docs/core_concepts/ai_generation"
-					target="_blank"
-					rel="noopener noreferrer">in the docs</a
-				>
-			</Tooltip>
-			<span class="text-2xs text-tertiary ml-2">(optional but recommended)</span>
-		</span>
-		<div class="flex flex-row gap-1">
+		<div class="flex flex-col gap-1">
+			<span class="text-secondary text-sm">
+				AI key for Windmill AI
+				<Tooltip>
+					Find out how it can help you <a
+						href="https://www.windmill.dev/docs/core_concepts/ai_generation"
+						target="_blank"
+						rel="noopener noreferrer">in the docs</a
+					>
+				</Tooltip>
+				<span class="text-2xs text-tertiary ml-2">(optional but recommended)</span>
+			</span>
+			<div class="pb-2">
+				<ToggleButtonGroup bind:selected>
+					<ToggleButton value="openai" label="OpenAi" />
+					<ToggleButton value="anthropic" label="Anthropic" />
+					<ToggleButton value="mistral" label="Mistral" />
+				</ToggleButtonGroup>
+			</div>
+		</div>
+		<div class="flex flex-row gap-1 pb-4">
 			<input
 				type="password"
 				autocomplete="new-password"
-				bind:value={openAiKey}
+				bind:value={aiKey}
 				on:keyup={handleKeyUp}
 			/>
-			<TestOpenaiKey apiKey={openAiKey} disabled={!openAiKey} />
+			<TestAiKey apiKey={aiKey} disabled={!aiKey} aiProvider={selected} />
 		</div>
-		{#if openAiKey}
+		{#if aiKey}
 			<Toggle
-				disabled={!openAiKey}
-				size="xs"
+				disabled={!aiKey}
 				bind:checked={codeCompletionEnabled}
 				options={{ right: 'Enable code completion' }}
 			/>

frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte

@@ -43,7 +43,7 @@
 
 	import PremiumInfo from '$lib/components/settings/PremiumInfo.svelte'
 	import Toggle from '$lib/components/Toggle.svelte'
-	import TestOpenaiKey from '$lib/components/copilot/TestOpenaiKey.svelte'
+	import TestAiKey from '$lib/components/copilot/TestAiKey.svelte'
 	import Portal from '$lib/components/Portal.svelte'
 
 	import { fade } from 'svelte/transition'
@@ -56,6 +56,9 @@
 	} from '$lib/workspace_settings'
 	import { base } from '$lib/base'
 	import { hubPaths } from '$lib/hub'
+	import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte'
+	import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte'
+	import type { AiProviderTypes } from '$lib/components/copilot/lib'
 
 	type GitSyncTypeMap = {
 		scripts: boolean
@@ -99,7 +102,8 @@
 	let errorHandlerItemKind: 'flow' | 'script' = 'script'
 	let errorHandlerExtraArgs: Record<string, any> = {}
 	let errorHandlerMutedOnCancel: boolean | undefined = undefined
-	let openaiResourceInitialPath: string | undefined = undefined
+	let aiResourceInitialPath: string | undefined = undefined
+	let aiResourceInitialProvider: string | undefined = undefined
 
 	let s3ResourceSettings: S3ResourceSettings = {
 		resourceType: 's3',
@@ -128,6 +132,7 @@
 	let workspaceReencryptionInProgress: boolean = false
 	let encryptionKeyRegex = /^[a-zA-Z0-9]{64}$/
 	let codeCompletionEnabled: boolean = false
+	let selected: AiProviderTypes = 'openai'
 	let tab =
 		($page.url.searchParams.get('tab') as
 			| 'users'
@@ -175,31 +180,37 @@
 		}
 	}
 
-	async function editCopilotConfig(openaiResourcePath: string): Promise<void> {
+	async function editCopilotConfig(aiResourcePath: string, aiProvider: string): Promise<void> {
 		// in JS, an empty string is also falsy
-		openaiResourceInitialPath = openaiResourcePath
-		if (openaiResourcePath) {
+		aiResourceInitialPath = aiResourcePath
+		aiResourceInitialProvider = aiProvider
+		if (aiResourcePath) {
 			await WorkspaceService.editCopilotConfig({
 				workspace: $workspaceStore!,
 				requestBody: {
-					openai_resource_path: openaiResourcePath,
+					ai_resource: {
+						path: aiResourcePath,
+						provider: aiProvider
+					},
 					code_completion_enabled: codeCompletionEnabled
 				}
 			})
 			copilotInfo.set({
-				exists_openai_resource_path: true,
+				ai_provider: aiProvider,
+				exists_ai_resource: true,
 				code_completion_enabled: codeCompletionEnabled
 			})
 		} else {
 			await WorkspaceService.editCopilotConfig({
 				workspace: $workspaceStore!,
 				requestBody: {
-					openai_resource_path: undefined,
+					ai_resource: undefined,
 					code_completion_enabled: codeCompletionEnabled
 				}
 			})
 			copilotInfo.set({
-				exists_openai_resource_path: true,
+				ai_provider: '',
+				exists_ai_resource: false,
 				code_completion_enabled: codeCompletionEnabled
 			})
 		}
@@ -373,7 +384,9 @@
 		customer_id = settings.customer_id
 		workspaceToDeployTo = settings.deploy_to
 		webhook = settings.webhook
-		openaiResourceInitialPath = settings.openai_resource_path
+		aiResourceInitialPath = settings.ai_resource?.path
+		aiResourceInitialProvider = settings.ai_resource?.provider
+		selected = aiResourceInitialProvider as AiProviderTypes ?? 'openai'
 		errorHandlerItemKind = settings.error_handler?.split('/')[0] as 'flow' | 'script'
 		errorHandlerScriptPath = (settings.error_handler ?? '').split('/').slice(1).join('/')
 		errorHandlerInitialScriptPath = errorHandlerScriptPath
@@ -631,7 +644,7 @@
 				<Tab size="xs" value="error_handler">
 					<div class="flex gap-2 items-center my-1">Error Handler</div>
 				</Tab>
-				<Tab size="xs" value="openai">
+				<Tab size="xs" value="ai">
 					<div class="flex gap-2 items-center my-1">Windmill AI</div>
 				</Tab>
 				<Tab size="xs" value="windmill_lfs">
@@ -959,7 +972,7 @@
 					Save
 				</Button>
 			</div>
-		{:else if tab == 'openai'}
+		{:else if tab == 'ai'}
 			<div class="flex flex-col gap-4 my-8">
 				<div class="flex flex-col gap-1">
 					<div class=" text-primary text-lg font-semibold"> Windmill AI</div>
@@ -967,7 +980,7 @@
 						Select an OpenAI resource to unlock Windmill AI features.
 					</div>
 					<div class="text-tertiary text-xs">
-						Windmill AI uses OpenAI's GPT-4o for all AI features.
+						Windmill AI supports integration with your preferred AI provider for all AI features.
 						<a
 							href="https://www.windmill.dev/docs/core_concepts/ai_generation"
 							target="_blank"
@@ -976,19 +989,32 @@
 					</div>
 				</div>
 			</div>
+			<ToggleButtonGroup
+				bind:selected
+				on:selected={() => {
+					aiResourceInitialPath = ''
+				}}
+			>
+				<ToggleButton value="openai" label="OpenAI" />
+				<ToggleButton value="anthropic" label="Anthropic" />
+				<ToggleButton value="mistral" label="Mistral" />
+			</ToggleButtonGroup>
 			<div class="mt-5 flex gap-1">
-				{#key [openaiResourceInitialPath, usingOpenaiClientCredentialsOauth]}
+				{#key [aiResourceInitialPath, aiResourceInitialProvider, usingOpenaiClientCredentialsOauth, selected]}
 					<ResourcePicker
 						resourceType={usingOpenaiClientCredentialsOauth
 							? 'openai_client_credentials_oauth'
-							: 'openai'}
-						initialValue={openaiResourceInitialPath}
+							: selected}
+						initialValue={aiResourceInitialPath}
 						on:change={(ev) => {
-							editCopilotConfig(ev.detail)
+							editCopilotConfig(ev.detail, selected)
 						}}
 					/>
+					<TestAiKey
+						disabled={!aiResourceInitialPath || aiResourceInitialProvider != selected}
+						aiProvider={selected}
+					/>
 				{/key}
-				<TestOpenaiKey disabled={!openaiResourceInitialPath} />
 			</div>
 			<div class="mt-3">
 				<Toggle
@@ -996,7 +1022,7 @@
 					bind:checked={codeCompletionEnabled}
 					options={{ right: 'Enable code completion' }}
 					on:change={() => {
-						editCopilotConfig(openaiResourceInitialPath || '')
+						editCopilotConfig(aiResourceInitialPath || '', aiResourceInitialProvider || '')
 					}}
 				/>
 			</div>