feat(ai agent): allow multiple images input for ai agent + code cleaning (#6591)

* add in frontend

* draft openai handling

* upload to s3

* simpler output

* return s3 directly if any

* low quality

* implement for gemini

* handle imagen model

* handle image input

* cleaning

* remove base64 from output

* cleaning

* fix timeout

* handle openrouter

* remove log

* allow image input when creating image

* cleaning

* increase stack size

* inline everything

* revert stack size

* refactor: move AI executor types to separate module

- Created ai module structure with types.rs
- Moved all type definitions from ai_executor.rs to ai/types.rs
- No functional changes, just code organization

* refactor: add QueryBuilder trait and provider detection utilities

- Created QueryBuilder trait for abstracting provider-specific logic
- Added helper functions for provider detection (is_anthropic_provider)
- Implemented placeholder QueryBuilder for all providers
- Updated OpenAIRequest to use slices instead of Vec references
- All providers now have QueryBuilder implementations (using default for now)

* feat: implement OpenAI query builder with image support foundation

- Created proper OpenAI query builder implementation
- Added image_handler module for S3 upload/download utilities
- Separated text and image request building logic
- Added prepare_messages_for_api to handle S3Object conversion
- Foundation laid for supporting tools with image output

* refactor(ai): complete AI executor refactoring with query builder pattern

- Created modular structure under ai/ module
- Moved all types to ai/types.rs
- Created QueryBuilder trait for provider abstraction
- Implemented OpenAI query builder with image+tools support
- Added unified agent runner supporting both text and image outputs with tools
- Refactored run_agent to delegate to new unified implementation
- Added image handler utilities for S3 operations
- Improved code organization and maintainability

* cleaning

* feat(ai): implement remaining provider query builders

- Added Anthropic query builder with proper message conversion
- Added Google AI query builder with Gemini API support
- Added OpenRouter query builder delegating to OpenAI for compatibility
- Added missing Anthropic and Gemini types to types.rs
- Fixed type references and compilation errors
- All providers now support the unified query builder interface

* fixes

* fixes

* mime type + cleaning

* image to images

* handle mutlitple images

* fix

* remove agent_runner file

* clean query builder logic

* cleaning

* cleaning

* hide structured_output based on output type

* fix

* user images and not nested

* better descriptions

backend/Cargo.lock

@@ -15744,6 +15744,7 @@ dependencies = [
  "async-once-cell",
  "async-recursion",
  "async-stream",
+ "async-trait",
  "backon",
  "base64 0.22.1",
  "bit-vec 0.6.3",

backend/windmill-worker/Cargo.toml

@@ -67,6 +67,7 @@ serde.workspace = true
 serde_json.workspace = true
 futures.workspace = true
 async-recursion.workspace = true
+async-trait.workspace = true
 anyhow.workspace = true
 itertools.workspace = true
 regex.workspace = true

backend/windmill-worker/src/ai/image_handler.rs

@@ -0,0 +1,68 @@
+use base64::Engine;
+use futures;
+use ulid;
+use windmill_common::{client::AuthedClient, error::Error, s3_helpers::S3Object};
+use windmill_queue::MiniPulledJob;
+
+/// Upload image to S3 and return S3Object
+pub async fn upload_image_to_s3(
+    base64_image: &str,
+    job: &MiniPulledJob,
+    client: &AuthedClient,
+) -> Result<S3Object, Error> {
+    let image_bytes = base64::engine::general_purpose::STANDARD
+        .decode(base64_image)
+        .map_err(|e| Error::internal_err(format!("Failed to decode base64 image: {}", e)))?;
+
+    // Generate unique S3 key
+    let unique_id = ulid::Ulid::new().to_string();
+    let s3_key = format!("ai_images/{}/{}.png", job.id, unique_id);
+
+    // Create byte stream
+    let byte_stream = futures::stream::once(async move {
+        Ok::<_, std::convert::Infallible>(bytes::Bytes::from(image_bytes))
+    });
+
+    // Upload to S3
+    client
+        .upload_s3_file(
+            &job.workspace_id,
+            s3_key.clone(),
+            None, // storage - use default
+            byte_stream,
+        )
+        .await
+        .map_err(|e| Error::internal_err(format!("Failed to upload image to S3: {}", e)))?;
+
+    Ok(S3Object {
+        s3: s3_key,
+        storage: None,
+        filename: Some("generated_image.png".to_string()),
+        presigned: None,
+    })
+}
+
+/// Download an S3 image and convert it to a base64 data URL
+pub async fn download_and_encode_s3_image(
+    image: &S3Object,
+    client: &AuthedClient,
+    workspace_id: &str,
+) -> Result<(String, String), Error> {
+    // Download the image from S3
+    let image_bytes = client
+        .download_s3_file(workspace_id, &image.s3, image.storage.clone())
+        .await
+        .map_err(|e| Error::internal_err(format!("Failed to download S3 image: {}", e)))?;
+
+    // Encode as base64 data URL
+    let base64_data = base64::engine::general_purpose::STANDARD.encode(&image_bytes);
+
+    // Determine MIME type using mime_guess from file extension, with PNG as fallback
+    let mime_type = mime_guess::from_path(&image.s3).first();
+    let mime_type = mime_type
+        .as_ref()
+        .map(|mime| mime.essence_str())
+        .unwrap_or("image/png");
+
+    Ok((mime_type.to_string(), base64_data))
+}

backend/windmill-worker/src/ai/mod.rs

@@ -0,0 +1,7 @@
+// AI executor module structure
+// This module will contain all AI-related execution logic
+
+pub mod image_handler;
+pub mod providers;
+pub mod query_builder;
+pub mod types;

backend/windmill-worker/src/ai/providers/google_ai.rs

@@ -0,0 +1,275 @@
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use serde_json;
+use windmill_common::{client::AuthedClient, error::Error};
+
+use crate::ai::{
+    image_handler::download_and_encode_s3_image,
+    query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder},
+    types::*,
+};
+
+// Google AI/Gemini-specific types
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct GeminiInlineData {
+    #[serde(rename = "mimeType")]
+    pub mime_type: String,
+    pub data: String,
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+#[serde(untagged)]
+pub enum GeminiPart {
+    Text { text: String },
+    InlineData { inline_data: GeminiInlineData },
+    FunctionCall { function_call: GeminiFunctionCall },
+    FunctionResponse { function_response: GeminiFunctionResponse },
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+pub struct GeminiFunctionCall {
+    pub name: String,
+    pub args: serde_json::Value,
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+pub struct GeminiFunctionResponse {
+    pub name: String,
+    pub response: serde_json::Value,
+}
+
+#[derive(Serialize)]
+pub struct GeminiContent {
+    pub parts: Vec<GeminiPart>,
+}
+
+#[derive(Serialize)]
+pub struct GeminiImageRequest {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub contents: Option<Vec<GeminiContent>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub instances: Option<Vec<GeminiPredictContent>>,
+}
+
+#[derive(Serialize)]
+pub struct GeminiPredictContent {
+    pub prompt: String,
+}
+
+#[derive(Deserialize)]
+pub struct GeminiImageResponse {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub candidates: Option<Vec<GeminiCandidate>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub predictions: Option<Vec<GeminiPredictCandidate>>,
+}
+
+#[derive(Deserialize)]
+pub struct GeminiCandidate {
+    pub content: GeminiResponseContent,
+}
+
+#[derive(Deserialize)]
+pub struct GeminiPredictCandidate {
+    #[serde(rename = "bytesBase64Encoded")]
+    pub bytes_base64_encoded: String, // base64 encoded image
+}
+
+#[derive(Deserialize)]
+pub struct GeminiResponseContent {
+    pub parts: Vec<GeminiResponsePart>,
+}
+
+#[derive(Deserialize)]
+pub struct GeminiResponsePart {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[allow(dead_code)]
+    pub text: Option<String>,
+    #[serde(rename = "inlineData", skip_serializing_if = "Option::is_none")]
+    pub inline_data: Option<GeminiInlineData>,
+    #[serde(rename = "functionCall", skip_serializing_if = "Option::is_none")]
+    #[allow(dead_code)]
+    pub function_call: Option<GeminiFunctionCall>,
+}
+
+pub struct GoogleAIQueryBuilder;
+
+impl GoogleAIQueryBuilder {
+    pub fn new() -> Self {
+        Self
+    }
+
+    async fn build_image_request(
+        &self,
+        args: &BuildRequestArgs<'_>,
+        client: &AuthedClient,
+        workspace_id: &str,
+    ) -> Result<String, Error> {
+        let is_imagen = args.model.contains("imagen");
+
+        let request = if is_imagen {
+            // For Imagen models, use simple prompt format
+            GeminiImageRequest {
+                instances: Some(vec![GeminiPredictContent {
+                    prompt: args.user_message.trim().to_string(),
+                }]),
+                contents: None,
+            }
+        } else {
+            // For Gemini models with image generation, build parts
+            let mut parts = vec![GeminiPart::Text { text: args.user_message.trim().to_string() }];
+
+            if let Some(system_prompt) = args.system_prompt {
+                parts.insert(
+                    0,
+                    GeminiPart::Text { text: format!("SYSTEM PROMPT: {}", system_prompt.trim()) },
+                );
+            }
+
+            // Add input images if provided
+            if let Some(images) = args.images {
+                for image in images.iter() {
+                    if !image.s3.is_empty() {
+                        let (mime_type, image_bytes) =
+                            download_and_encode_s3_image(image, client, workspace_id).await?;
+                        parts.push(GeminiPart::InlineData {
+                            inline_data: GeminiInlineData {
+                                mime_type: mime_type,
+                                data: image_bytes,
+                            },
+                        });
+                    }
+                }
+            }
+
+            GeminiImageRequest { instances: None, contents: Some(vec![GeminiContent { parts }]) }
+        };
+
+        serde_json::to_string(&request)
+            .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e)))
+    }
+}
+
+#[async_trait]
+impl QueryBuilder for GoogleAIQueryBuilder {
+    fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool {
+        // Google AI supports tools only for text output
+        matches!(output_type, OutputType::Text)
+    }
+
+    async fn build_request(
+        &self,
+        args: &BuildRequestArgs<'_>,
+        client: &AuthedClient,
+        workspace_id: &str,
+    ) -> Result<String, Error> {
+        match args.output_type {
+            OutputType::Text => {
+                // For text output, use OpenAI-compatible format
+                let openai_builder = super::openai::OpenAIQueryBuilder::new();
+                openai_builder
+                    .build_request(args, client, workspace_id)
+                    .await
+            }
+            OutputType::Image => self.build_image_request(args, client, workspace_id).await,
+        }
+    }
+
+    async fn parse_response(&self, response: reqwest::Response) -> Result<ParsedResponse, Error> {
+        let url = response.url().path();
+
+        // For chat completions (text), use OpenAI parser
+        if url.contains("/chat/completions") {
+            let openai_builder = super::openai::OpenAIQueryBuilder::new();
+            return openai_builder.parse_response(response).await;
+        }
+
+        // Check if this is an image generation response
+        if url.contains(":predict") || url.contains(":generateContent") {
+            let response_text = response
+                .text()
+                .await
+                .map_err(|e| Error::internal_err(format!("Failed to read response text: {}", e)))?;
+
+            let gemini_response: GeminiImageResponse = serde_json::from_str(&response_text)
+                .map_err(|e| {
+                    Error::internal_err(format!(
+                        "Failed to parse Gemini response: {}. Raw response: {}",
+                        e, response_text
+                    ))
+                })?;
+
+            // Find image data in response
+            let image_data = gemini_response
+                .candidates
+                .as_ref()
+                .and_then(|candidates| {
+                    candidates.iter().find_map(|candidate| {
+                        candidate
+                            .content
+                            .parts
+                            .iter()
+                            .find_map(|part| part.inline_data.as_ref().map(|data| &data.data))
+                    })
+                })
+                .or_else(|| {
+                    gemini_response
+                        .predictions
+                        .as_ref()
+                        .and_then(|predictions| {
+                            predictions
+                                .iter()
+                                .find_map(|prediction| Some(&prediction.bytes_base64_encoded))
+                        })
+                });
+
+            if let Some(base64_image) = image_data {
+                Ok(ParsedResponse::Image { base64_data: base64_image.clone() })
+            } else {
+                Err(Error::internal_err(
+                    "No image data received from Gemini".to_string(),
+                ))
+            }
+        } else {
+            // This should not happen as we use OpenAI format for text
+            Err(Error::internal_err(
+                "Unexpected text response in Google AI parser".to_string(),
+            ))
+        }
+    }
+
+    fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String {
+        match output_type {
+            OutputType::Text => format!("{}/chat/completions", base_url), // Use OpenAI-compatible endpoint
+            OutputType::Image => {
+                // For image generation, build the full URL with model name
+                let url_suffix = if model.contains("imagen") {
+                    "predict"
+                } else {
+                    "generateContent"
+                };
+                format!(
+                    "https://generativelanguage.googleapis.com/v1beta/models/{}:{}",
+                    model, url_suffix
+                )
+            }
+        }
+    }
+
+    fn get_auth_headers(
+        &self,
+        api_key: &str,
+        output_type: &OutputType,
+    ) -> Vec<(&'static str, String)> {
+        match output_type {
+            OutputType::Text => {
+                // For text output, use Bearer token (OpenAI-compatible)
+                vec![("Authorization", format!("Bearer {}", api_key))]
+            }
+            OutputType::Image => {
+                // For image generation, use Google API key header
+                vec![("x-goog-api-key", api_key.to_string())]
+            }
+        }
+    }
+}

backend/windmill-worker/src/ai/providers/mod.rs

@@ -0,0 +1,3 @@
+pub mod google_ai;
+pub mod openai;
+pub mod openrouter;

backend/windmill-worker/src/ai/providers/openai.rs

@@ -0,0 +1,349 @@
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use serde_json;
+use windmill_common::{client::AuthedClient, error::Error};
+
+use crate::ai::{
+    image_handler::download_and_encode_s3_image,
+    query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder},
+    types::*,
+};
+
+// OpenAI-specific types
+#[derive(Deserialize, Serialize, Clone, Debug)]
+pub struct OpenAIFunction {
+    pub name: String,
+    pub arguments: String,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug)]
+pub struct OpenAIToolCall {
+    pub id: String,
+    pub function: OpenAIFunction,
+    pub r#type: String,
+}
+
+#[derive(Deserialize)]
+pub struct OpenAIChoice {
+    pub message: OpenAIMessage,
+}
+
+#[derive(Deserialize)]
+pub struct OpenAIResponse {
+    pub choices: Vec<OpenAIChoice>,
+}
+
+#[derive(Serialize)]
+pub struct ImageGenerationTool {
+    pub r#type: String,
+    pub quality: Option<String>,
+    pub background: Option<String>,
+}
+
+// Input content for image generation - supports both text and images
+#[derive(Serialize, Clone, Debug)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum ImageGenerationContent {
+    #[serde(rename = "input_text")]
+    InputText { text: String },
+    #[serde(rename = "input_image")]
+    InputImage { image_url: String },
+}
+
+#[derive(Serialize)]
+pub struct ImageGenerationMessage {
+    pub role: String,
+    pub content: Vec<ImageGenerationContent>,
+}
+
+#[derive(Serialize)]
+pub struct ImageGenerationRequest<'a> {
+    pub model: &'a str,
+    pub input: Vec<ImageGenerationMessage>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub instructions: Option<&'a str>,
+    pub tools: Vec<ImageGenerationTool>,
+}
+
+#[derive(Deserialize)]
+pub struct OpenAIImageResponse {
+    pub output: Vec<OpenAIImageOutput>,
+}
+
+#[derive(Deserialize)]
+pub struct OpenAIImageOutput {
+    pub r#type: String, // Expected to be "image_generation_call"
+    pub status: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub result: Option<String>, // Base64 encoded image, None if not completed
+}
+
+#[derive(Serialize)]
+pub struct OpenAIRequest<'a> {
+    pub model: &'a str,
+    pub messages: &'a [OpenAIMessage],
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tools: Option<&'a [ToolDef]>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub temperature: Option<f32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub max_completion_tokens: Option<u32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub response_format: Option<ResponseFormat>,
+}
+
+pub struct OpenAIQueryBuilder;
+
+impl OpenAIQueryBuilder {
+    pub fn new() -> Self {
+        Self
+    }
+
+    pub async fn prepare_messages_for_api(
+        &self,
+        messages: &[OpenAIMessage],
+        client: &AuthedClient,
+        workspace_id: &str,
+    ) -> Result<Vec<OpenAIMessage>, Error> {
+        let mut prepared_messages = Vec::new();
+
+        for message in messages {
+            let mut prepared_message = message.clone();
+
+            if let Some(content) = &message.content {
+                match content {
+                    OpenAIContent::Text(text) => {
+                        prepared_message.content = Some(OpenAIContent::Text(text.clone()));
+                    }
+                    OpenAIContent::Parts(parts) => {
+                        let mut prepared_content = Vec::new();
+
+                        for part in parts {
+                            match part {
+                                ContentPart::S3Object { s3_object } => {
+                                    // Convert S3Object to base64 image URL
+                                    let (mime_type, image_bytes) = download_and_encode_s3_image(
+                                        s3_object,
+                                        client,
+                                        workspace_id,
+                                    )
+                                    .await?;
+                                    prepared_content.push(ContentPart::ImageUrl {
+                                        image_url: ImageUrlData {
+                                            url: format!(
+                                                "data:{};base64,{}",
+                                                mime_type, image_bytes
+                                            ),
+                                        },
+                                    });
+                                }
+                                other => {
+                                    // Keep Text and ImageUrl as-is
+                                    prepared_content.push(other.clone());
+                                }
+                            }
+                        }
+
+                        prepared_message.content = Some(OpenAIContent::Parts(prepared_content));
+                    }
+                }
+            }
+
+            prepared_messages.push(prepared_message);
+        }
+
+        Ok(prepared_messages)
+    }
+
+    async fn build_text_request(
+        &self,
+        args: &BuildRequestArgs<'_>,
+        client: &AuthedClient,
+        workspace_id: &str,
+    ) -> Result<String, Error> {
+        let prepared_messages = self
+            .prepare_messages_for_api(args.messages, client, workspace_id)
+            .await?;
+
+        // Check if we need to add response_format for structured output
+        let has_output_properties = args
+            .output_schema
+            .and_then(|schema| schema.properties.as_ref())
+            .map(|props| !props.is_empty())
+            .unwrap_or(false);
+
+        let response_format = if has_output_properties && args.output_schema.is_some() {
+            let schema = args.output_schema.unwrap();
+            let strict_schema = schema.clone().make_strict();
+            Some(ResponseFormat {
+                r#type: "json_schema".to_string(),
+                json_schema: JsonSchemaFormat {
+                    name: "structured_output".to_string(),
+                    schema: strict_schema,
+                    strict: Some(true),
+                },
+            })
+        } else {
+            None
+        };
+
+        let request = OpenAIRequest {
+            model: args.model,
+            messages: &prepared_messages,
+            tools: args.tools,
+            temperature: args.temperature,
+            max_completion_tokens: args.max_tokens,
+            response_format,
+        };
+
+        serde_json::to_string(&request)
+            .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e)))
+    }
+
+    async fn build_image_request(
+        &self,
+        args: &BuildRequestArgs<'_>,
+        client: &AuthedClient,
+        workspace_id: &str,
+    ) -> Result<String, Error> {
+        // Build content array with text and optional image
+        let mut content =
+            vec![ImageGenerationContent::InputText { text: args.user_message.to_string() }];
+
+        // Add images if provided
+        if let Some(images) = args.images {
+            for image in images.iter() {
+                if !image.s3.is_empty() {
+                    let (mime_type, image_bytes) =
+                        download_and_encode_s3_image(image, client, workspace_id).await?;
+                    content.push(ImageGenerationContent::InputImage {
+                        image_url: format!("data:{};base64,{}", mime_type, image_bytes),
+                    });
+                }
+            }
+        }
+
+        // Build the request with tools if provided
+        let tools = vec![ImageGenerationTool {
+            r#type: "image_generation".to_string(),
+            quality: Some("low".to_string()),
+            background: None,
+        }];
+
+        // TODO: OpenAI's image generation API doesn't support custom tools in the same way as chat completions
+        // This would require a different approach, potentially using chat completions with image output
+        // For now, we'll use the standard image generation without custom tools
+
+        let image_request = ImageGenerationRequest {
+            model: args.model,
+            input: vec![ImageGenerationMessage { role: "user".to_string(), content }],
+            instructions: args.system_prompt,
+            tools,
+        };
+
+        serde_json::to_string(&image_request)
+            .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e)))
+    }
+}
+
+#[async_trait]
+impl QueryBuilder for OpenAIQueryBuilder {
+    fn supports_tools_with_output_type(&self, _output_type: &OutputType) -> bool {
+        // OpenAI supports tools for both text and image output
+        true
+    }
+
+    async fn build_request(
+        &self,
+        args: &BuildRequestArgs<'_>,
+        client: &AuthedClient,
+        workspace_id: &str,
+    ) -> Result<String, Error> {
+        match args.output_type {
+            OutputType::Text => self.build_text_request(args, client, workspace_id).await,
+            OutputType::Image => self.build_image_request(args, client, workspace_id).await,
+        }
+    }
+
+    async fn parse_response(&self, response: reqwest::Response) -> Result<ParsedResponse, Error> {
+        // Check if this is an image response
+        let url = response.url().path();
+        if url.contains("/responses") {
+            // Parse image generation response
+            let response_text = response
+                .text()
+                .await
+                .map_err(|e| Error::internal_err(format!("Failed to read response text: {}", e)))?;
+
+            let image_response: OpenAIImageResponse = serde_json::from_str(&response_text)
+                .map_err(|e| {
+                    Error::internal_err(format!(
+                        "Failed to parse OpenAI image response: {}. Raw response: {}",
+                        e, response_text
+                    ))
+                })?;
+
+            // Find the first completed image generation output
+            let image_generation_call = image_response
+                .output
+                .iter()
+                .find(|output| {
+                    output.r#type == "image_generation_call" && output.status == "completed"
+                })
+                .and_then(|output| output.result.as_ref());
+
+            if let Some(base64_image) = image_generation_call {
+                Ok(ParsedResponse::Image { base64_data: base64_image.clone() })
+            } else {
+                Err(Error::internal_err(
+                    "No completed image output received from OpenAI".to_string(),
+                ))
+            }
+        } else {
+            // Parse text/chat completion response
+            let openai_response: OpenAIResponse = response
+                .json()
+                .await
+                .map_err(|e| Error::internal_err(format!("Failed to parse response: {}", e)))?;
+
+            let first_choice = openai_response
+                .choices
+                .into_iter()
+                .next()
+                .ok_or_else(|| Error::internal_err("No response from API"))?;
+
+            Ok(ParsedResponse::Text {
+                content: first_choice.message.content.map(|c| match c {
+                    OpenAIContent::Text(text) => text,
+                    OpenAIContent::Parts(parts) => {
+                        // Extract text from parts
+                        parts
+                            .into_iter()
+                            .filter_map(|part| match part {
+                                ContentPart::Text { text } => Some(text),
+                                _ => None,
+                            })
+                            .collect::<Vec<_>>()
+                            .join(" ")
+                    }
+                }),
+                tool_calls: first_choice.message.tool_calls.unwrap_or_default(),
+            })
+        }
+    }
+
+    fn get_endpoint(&self, base_url: &str, _model: &str, output_type: &OutputType) -> String {
+        match output_type {
+            OutputType::Text => format!("{}/chat/completions", base_url),
+            OutputType::Image => format!("{}/responses", base_url),
+        }
+    }
+
+    fn get_auth_headers(
+        &self,
+        api_key: &str,
+        _output_type: &OutputType,
+    ) -> Vec<(&'static str, String)> {
+        vec![("Authorization", format!("Bearer {}", api_key))]
+    }
+}

backend/windmill-worker/src/ai/providers/openrouter.rs

@@ -0,0 +1,202 @@
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use serde_json;
+use windmill_common::{client::AuthedClient, error::Error};
+
+use crate::ai::{
+    providers::openai::{OpenAIQueryBuilder, OpenAIResponse},
+    query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder},
+    types::*,
+};
+
+// OpenRouter-specific types
+#[derive(Serialize)]
+pub struct OpenRouterChatRequest<'a> {
+    pub model: &'a str,
+    pub messages: &'a [OpenAIMessage],
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tools: Option<&'a [ToolDef]>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub temperature: Option<f32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub max_completion_tokens: Option<u32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub response_format: Option<ResponseFormat>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub modalities: Option<Vec<&'a str>>,
+}
+
+#[derive(Deserialize)]
+pub struct OpenRouterImageResponse {
+    pub choices: Vec<OpenRouterImageChoice>,
+}
+
+#[derive(Deserialize)]
+pub struct OpenRouterImageChoice {
+    pub message: OpenRouterImageResponseMessage,
+}
+
+#[derive(Deserialize)]
+pub struct OpenRouterImageResponseMessage {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub images: Option<Vec<OpenRouterImageData>>,
+}
+
+#[derive(Deserialize)]
+pub struct OpenRouterImageData {
+    pub image_url: OpenRouterImageUrl,
+}
+
+#[derive(Deserialize)]
+pub struct OpenRouterImageUrl {
+    pub url: String, // data:image/png;base64,... format
+}
+
+pub struct OpenRouterQueryBuilder {
+    // OpenRouter uses OpenAI-compatible API, so we delegate most work to OpenAI builder
+    openai_builder: OpenAIQueryBuilder,
+}
+
+impl OpenRouterQueryBuilder {
+    pub fn new() -> Self {
+        Self { openai_builder: OpenAIQueryBuilder::new() }
+    }
+}
+
+#[async_trait]
+impl QueryBuilder for OpenRouterQueryBuilder {
+    fn supports_tools_with_output_type(&self, _output_type: &OutputType) -> bool {
+        // OpenRouter supports tools for both text and image output (via OpenAI-compatible API)
+        true
+    }
+
+    async fn build_request(
+        &self,
+        args: &BuildRequestArgs<'_>,
+        client: &AuthedClient,
+        workspace_id: &str,
+    ) -> Result<String, Error> {
+        match args.output_type {
+            OutputType::Text => {
+                // For text, use standard OpenAI format without modalities
+                self.openai_builder
+                    .build_request(args, client, workspace_id)
+                    .await
+            }
+            OutputType::Image => {
+                // For image generation, we need to add modalities field
+                // First, prepare the messages using the OpenAI builder's logic
+                let openai_builder = &self.openai_builder;
+                let prepared_messages = openai_builder
+                    .prepare_messages_for_api(args.messages, client, workspace_id)
+                    .await?;
+
+                // Check if we need to add response_format for structured output
+                let has_output_properties = args
+                    .output_schema
+                    .and_then(|schema| schema.properties.as_ref())
+                    .map(|props| !props.is_empty())
+                    .unwrap_or(false);
+
+                let response_format = if has_output_properties && args.output_schema.is_some() {
+                    let schema = args.output_schema.unwrap();
+                    let strict_schema = schema.clone().make_strict();
+                    Some(ResponseFormat {
+                        r#type: "json_schema".to_string(),
+                        json_schema: JsonSchemaFormat {
+                            name: "structured_output".to_string(),
+                            schema: strict_schema,
+                            strict: Some(true),
+                        },
+                    })
+                } else {
+                    None
+                };
+
+                // Build OpenRouter-specific request with modalities
+                let request = OpenRouterChatRequest {
+                    model: args.model,
+                    messages: &prepared_messages,
+                    tools: args.tools,
+                    temperature: args.temperature,
+                    max_completion_tokens: args.max_tokens,
+                    response_format,
+                    modalities: Some(vec!["image", "text"]),
+                };
+
+                serde_json::to_string(&request)
+                    .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e)))
+            }
+        }
+    }
+
+    async fn parse_response(&self, response: reqwest::Response) -> Result<ParsedResponse, Error> {
+        let response_text = response
+            .text()
+            .await
+            .map_err(|e| Error::internal_err(format!("Failed to read response text: {}", e)))?;
+
+        // First try to parse as OpenRouter image response
+        if let Ok(image_response) = serde_json::from_str::<OpenRouterImageResponse>(&response_text)
+        {
+            // Extract base64 image from the first choice
+            let image_url = image_response
+                .choices
+                .get(0)
+                .and_then(|choice| choice.message.images.as_ref())
+                .and_then(|images| images.get(0))
+                .map(|image| &image.image_url.url);
+
+            if let Some(data_url) = image_url {
+                // Extract base64 data from data URL format: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
+                if let Some(base64_start) = data_url.find("base64,") {
+                    let base64_data = &data_url[base64_start + 7..]; // Skip "base64," prefix
+                    return Ok(ParsedResponse::Image { base64_data: base64_data.to_string() });
+                }
+            }
+        }
+
+        // If not an image response or parsing failed, try as regular OpenAI response
+        let openai_response: OpenAIResponse =
+            serde_json::from_str(&response_text).map_err(|e| {
+                Error::internal_err(format!(
+                    "Failed to parse response: {}. Raw response: {}",
+                    e, response_text
+                ))
+            })?;
+
+        let first_choice = openai_response
+            .choices
+            .into_iter()
+            .next()
+            .ok_or_else(|| Error::internal_err("No response from API"))?;
+
+        Ok(ParsedResponse::Text {
+            content: first_choice.message.content.map(|c| match c {
+                OpenAIContent::Text(text) => text,
+                OpenAIContent::Parts(parts) => parts
+                    .into_iter()
+                    .filter_map(|part| match part {
+                        ContentPart::Text { text } => Some(text),
+                        _ => None,
+                    })
+                    .collect::<Vec<_>>()
+                    .join(" "),
+            }),
+            tool_calls: first_choice.message.tool_calls.unwrap_or_default(),
+        })
+    }
+
+    fn get_endpoint(&self, base_url: &str, _model: &str, _output_type: &OutputType) -> String {
+        // OpenRouter uses the same endpoint for both text and image generation
+        format!("{}/chat/completions", base_url)
+    }
+
+    fn get_auth_headers(
+        &self,
+        api_key: &str,
+        _output_type: &OutputType,
+    ) -> Vec<(&'static str, String)> {
+        vec![("Authorization", format!("Bearer {}", api_key))]
+    }
+}

backend/windmill-worker/src/ai/query_builder.rs

@@ -0,0 +1,70 @@
+use async_trait::async_trait;
+use windmill_common::{client::AuthedClient, error::Error, s3_helpers::S3Object};
+
+use crate::ai::{
+    providers::{
+        google_ai::GoogleAIQueryBuilder,
+        openai::{OpenAIQueryBuilder, OpenAIToolCall},
+        openrouter::OpenRouterQueryBuilder,
+    },
+    types::*,
+};
+
+/// Arguments for building an AI request
+pub struct BuildRequestArgs<'a> {
+    pub messages: &'a [OpenAIMessage],
+    pub tools: Option<&'a [ToolDef]>,
+    pub model: &'a str,
+    pub temperature: Option<f32>,
+    pub max_tokens: Option<u32>,
+    pub output_schema: Option<&'a OpenAPISchema>,
+    pub output_type: &'a OutputType,
+    pub system_prompt: Option<&'a str>,
+    pub user_message: &'a str,
+    pub images: Option<&'a [S3Object]>,
+}
+
+/// Response from AI provider
+pub enum ParsedResponse {
+    Text { content: Option<String>, tool_calls: Vec<OpenAIToolCall> },
+    Image { base64_data: String },
+}
+
+/// Trait for building provider-specific AI requests
+#[async_trait]
+pub trait QueryBuilder: Send + Sync {
+    /// Check if this provider supports tools with the given output type
+    fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool;
+
+    /// Build the request body for the provider
+    async fn build_request(
+        &self,
+        args: &BuildRequestArgs<'_>,
+        client: &AuthedClient,
+        workspace_id: &str,
+    ) -> Result<String, Error>;
+
+    /// Parse the response from the provider
+    async fn parse_response(&self, response: reqwest::Response) -> Result<ParsedResponse, Error>;
+
+    /// Get the API endpoint for this provider
+    fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String;
+
+    /// Get the authentication headers for this provider
+    fn get_auth_headers(
+        &self,
+        api_key: &str,
+        output_type: &OutputType,
+    ) -> Vec<(&'static str, String)>;
+}
+
+/// Factory function to create the appropriate query builder for a provider
+pub fn create_query_builder(provider: &ProviderWithResource) -> Box<dyn QueryBuilder> {
+    use windmill_common::ai_providers::AIProvider;
+
+    match provider.kind {
+        AIProvider::GoogleAI => Box::new(GoogleAIQueryBuilder::new()),
+        AIProvider::OpenRouter => Box::new(OpenRouterQueryBuilder::new()),
+        _ => Box::new(OpenAIQueryBuilder::new()), // Use OpenAI as default for all other providers
+    }
+}

backend/windmill-worker/src/ai/types.rs

@@ -0,0 +1,368 @@
+use serde::{Deserialize, Serialize};
+use serde_json::value::RawValue;
+use std::collections::HashMap;
+use windmill_common::{
+    ai_providers::AIProvider, db::DB, error::Error, flow_status::AgentAction, flows::FlowModule,
+    s3_helpers::S3Object,
+};
+use windmill_parser::Typ;
+
+use crate::ai::providers::openai::OpenAIToolCall;
+
+// Shared types used across multiple providers
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum ContentPart {
+    Text {
+        text: String,
+    },
+    #[serde(rename = "image_url")]
+    ImageUrl {
+        image_url: ImageUrlData,
+    },
+    #[serde(rename = "s3_object")]
+    S3Object {
+        s3_object: S3Object,
+    },
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct ImageUrlData {
+    pub url: String, // data:image/png;base64,... or https://...
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(untagged)]
+pub enum OpenAIContent {
+    Text(String),
+    Parts(Vec<ContentPart>),
+}
+
+#[derive(Deserialize, Serialize, Clone, Default, Debug)]
+pub struct OpenAIMessage {
+    pub role: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub content: Option<OpenAIContent>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tool_calls: Option<Vec<OpenAIToolCall>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tool_call_id: Option<String>,
+    #[serde(skip_serializing)]
+    pub agent_action: Option<AgentAction>,
+}
+
+/// same as OpenAIMessage but with agent_action field included in the serialization
+#[derive(Serialize)]
+pub struct Message<'a> {
+    #[serde(flatten)]
+    pub message: &'a OpenAIMessage,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub agent_action: Option<&'a AgentAction>,
+}
+
+#[derive(Serialize, Clone, Debug)]
+pub struct ResponseFormat {
+    pub r#type: String,
+    pub json_schema: JsonSchemaFormat,
+}
+
+#[derive(Serialize, Clone, Debug)]
+pub struct JsonSchemaFormat {
+    pub name: String,
+    pub schema: OpenAPISchema,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub strict: Option<bool>,
+}
+
+#[derive(Serialize, Clone, Debug)]
+pub struct ToolDefFunction {
+    pub name: String,
+    pub description: Option<String>,
+    pub parameters: Box<RawValue>,
+}
+
+#[derive(Serialize, Clone, Debug)]
+pub struct ToolDef {
+    pub r#type: String,
+    pub function: ToolDefFunction,
+}
+
+pub struct Tool {
+    pub module: FlowModule,
+    pub def: ToolDef,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
+#[serde(rename_all = "lowercase")]
+pub enum OutputType {
+    Text,
+    Image,
+}
+
+impl Default for OutputType {
+    fn default() -> Self {
+        OutputType::Text
+    }
+}
+
+#[derive(Deserialize, Debug)]
+pub struct AIAgentArgs {
+    pub provider: ProviderWithResource,
+    pub system_prompt: Option<String>,
+    pub user_message: String,
+    pub temperature: Option<f32>,
+    pub max_completion_tokens: Option<u32>,
+    pub output_schema: Option<OpenAPISchema>,
+    pub output_type: Option<OutputType>,
+    pub user_images: Option<Vec<S3Object>>,
+}
+
+#[derive(Deserialize, Debug)]
+pub struct ProviderResource {
+    #[serde(alias = "apiKey")]
+    pub api_key: String,
+    #[serde(alias = "baseUrl")]
+    pub base_url: Option<String>,
+}
+
+#[derive(Deserialize, Debug)]
+pub struct ProviderWithResource {
+    pub kind: AIProvider,
+    pub resource: ProviderResource,
+    pub model: String,
+}
+
+impl ProviderWithResource {
+    pub fn get_api_key(&self) -> &str {
+        &self.resource.api_key
+    }
+
+    pub fn get_model(&self) -> &str {
+        &self.model
+    }
+
+    pub async fn get_base_url(&self, db: &DB) -> Result<String, Error> {
+        self.kind
+            .get_base_url(self.resource.base_url.clone(), db)
+            .await
+    }
+}
+
+#[derive(Serialize)]
+pub struct AIAgentResult<'a> {
+    pub output: Box<RawValue>,
+    pub messages: Vec<Message<'a>>,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(untagged)]
+pub enum SchemaType {
+    Single(String),
+    Multiple(Vec<String>),
+}
+
+impl Default for SchemaType {
+    fn default() -> Self {
+        SchemaType::Single("object".to_string())
+    }
+}
+
+#[derive(Serialize, Deserialize, Default, Clone, Debug)]
+pub struct OpenAPISchema {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub r#type: Option<SchemaType>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub items: Option<Box<OpenAPISchema>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub properties: Option<HashMap<String, Box<OpenAPISchema>>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub required: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none", rename = "oneOf")]
+    pub one_of: Option<Vec<Box<OpenAPISchema>>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub format: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub r#enum: Option<Vec<String>>,
+    #[serde(
+        skip_serializing_if = "Option::is_none",
+        rename = "additionalProperties"
+    )]
+    pub additional_properties: Option<bool>,
+}
+
+impl OpenAPISchema {
+    pub fn from_str(typ: &str) -> Self {
+        OpenAPISchema { r#type: Some(SchemaType::Single(typ.to_string())), ..Default::default() }
+    }
+
+    pub fn from_str_with_enum(typ: &str, enu: &Option<Vec<String>>) -> Self {
+        OpenAPISchema {
+            r#type: Some(SchemaType::Single(typ.to_string())),
+            r#enum: enu.clone(),
+            ..Default::default()
+        }
+    }
+
+    pub fn datetime() -> Self {
+        Self {
+            r#type: Some(SchemaType::Single("string".to_string())),
+            format: Some("date-time".to_string()),
+            ..Default::default()
+        }
+    }
+
+    pub fn from_typ(typ: &Typ) -> Self {
+        match typ {
+            Typ::Str(enu) => Self::from_str_with_enum("string", enu),
+            Typ::Int => Self::from_str("integer"),
+            Typ::Float => Self::from_str("number"),
+            Typ::Bool => Self::from_str("boolean"),
+            Typ::Bytes => Self::from_str("string"),
+            Typ::Datetime => Self::datetime(),
+            Typ::Resource(_) => Self::from_str("string"),
+            Typ::Email => Self::from_str("string"),
+            Typ::Sql => Self::from_str("string"),
+            Typ::DynSelect(_) => Self::from_str("string"),
+            Typ::DynMultiselect(_) => Self::from_str("string"),
+            Typ::List(typ) => OpenAPISchema {
+                r#type: Some(SchemaType::Single("array".to_string())),
+                items: Some(Box::new(Self::from_typ(typ))),
+                ..Default::default()
+            },
+            Typ::Object(typ) => OpenAPISchema {
+                r#type: Some(SchemaType::Single("object".to_string())),
+                items: None,
+                properties: typ.props.as_ref().map(|props| {
+                    props
+                        .iter()
+                        .map(|prop| (prop.key.clone(), Box::new(Self::from_typ(&prop.typ))))
+                        .collect()
+                }),
+                required: typ
+                    .props
+                    .as_ref()
+                    .map(|props| props.iter().map(|prop| prop.key.clone()).collect()),
+                ..Default::default()
+            },
+            Typ::OneOf(variants) => OpenAPISchema {
+                r#type: Some(SchemaType::Single("object".to_string())),
+                one_of: Some(
+                    variants
+                        .iter()
+                        .map(|variant| {
+                            let schema = OpenAPISchema {
+                                r#type: Some(SchemaType::Single("object".to_string())),
+                                properties: Some(
+                                    variant
+                                        .properties
+                                        .iter()
+                                        .map(|prop| {
+                                            (
+                                                prop.key.clone(),
+                                                Box::new(
+                                                    if prop.key == "label" || prop.key == "kind" {
+                                                        Self::from_str_with_enum(
+                                                            "string",
+                                                            &Some(vec![variant.label.clone()]),
+                                                        )
+                                                    } else {
+                                                        Self::from_typ(&prop.typ)
+                                                    },
+                                                ),
+                                            )
+                                        })
+                                        .collect(),
+                                ),
+                                required: Some(
+                                    variant
+                                        .properties
+                                        .iter()
+                                        .map(|prop| prop.key.clone())
+                                        .collect(),
+                                ),
+                                ..Default::default()
+                            };
+                            Box::new(schema)
+                        })
+                        .collect(),
+                ),
+                ..Default::default()
+            },
+            Typ::Unknown => Self::from_str("object"),
+        }
+    }
+
+    /// Makes this schema compatible with OpenAI's strict mode by:
+    /// - Adding additionalProperties: false to all object types
+    /// - Making non-required properties nullable
+    /// - Ensuring all properties are in the required array
+    pub fn make_strict(mut self) -> Self {
+        // Handle this schema if it's an object type
+        if let Some(SchemaType::Single(ref type_str)) = self.r#type {
+            if type_str == "object" {
+                // Set additionalProperties to false
+                self.additional_properties = Some(false);
+
+                if let Some(properties) = self.properties.as_mut() {
+                    // Get original required fields
+                    let original_required = self.required.as_ref();
+
+                    if let Some(required) = original_required {
+                        // Update properties to make non-required fields nullable
+                        for (key, prop) in properties.iter_mut() {
+                            let mut new_prop = (**prop).clone();
+                            // Make non-required fields nullable
+                            if !required.contains(key) {
+                                new_prop = new_prop.make_nullable();
+                            }
+                            // Recursively make nested schemas strict
+                            new_prop = new_prop.make_strict();
+                            *prop = Box::new(new_prop);
+                        }
+                    }
+
+                    // All properties must be in required array for strict mode
+                    self.required = Some(properties.keys().cloned().collect());
+                }
+            }
+        }
+
+        // Recursively process nested schemas
+        if let Some(ref mut items) = self.items {
+            **items = items.as_ref().clone().make_strict();
+        }
+
+        if let Some(ref mut one_of) = self.one_of {
+            *one_of = one_of
+                .iter()
+                .map(|schema| Box::new(schema.as_ref().clone().make_strict()))
+                .collect();
+        }
+
+        self
+    }
+
+    /// Makes this property nullable by converting its type to a union with null
+    pub fn make_nullable(mut self) -> Self {
+        match self.r#type.take() {
+            Some(SchemaType::Single(type_str)) => {
+                if type_str != "null" {
+                    self.r#type = Some(SchemaType::Multiple(vec![type_str, "null".into()]));
+                } else {
+                    self.r#type = Some(SchemaType::Single("null".into()));
+                }
+            }
+            Some(SchemaType::Multiple(mut types)) => {
+                if !types.iter().any(|t| t == "null") {
+                    types.push("null".into());
+                }
+                self.r#type = Some(SchemaType::Multiple(types));
+            }
+            None => {
+                self.r#type = Some(SchemaType::Single("null".into()));
+            }
+        }
+        self
+    }
+}

backend/windmill-worker/src/lib.rs

@@ -16,6 +16,7 @@ mod java_executor;
 #[cfg(feature = "ruby")]
 mod ruby_executor;
 
+mod ai;
 mod ai_executor;
 mod bun_executor;
 pub mod common;

frontend/src/lib/components/InputTransformForm.svelte

@@ -11,6 +11,7 @@
 	import type { Schema } from '$lib/common'
 	import type { InputCat } from '$lib/utils'
 	import { createEventDispatcher, getContext, untrack } from 'svelte'
+	import { computeShow } from '$lib/utils'
 
 	import ArgInput from './ArgInput.svelte'
 	import FieldHeader from './FieldHeader.svelte'
@@ -58,6 +59,7 @@
 		hideHelpButton?: boolean
 		class?: string
 		editor?: SimpleEditor | undefined
+		otherArgs?: Record<string, InputTransform>
 	}
 
 	let {
@@ -79,7 +81,8 @@
 		enableAi = false,
 		hideHelpButton = false,
 		class: className = '',
-		editor = $bindable(undefined)
+		editor = $bindable(undefined),
+		otherArgs = {}
 	}: Props = $props()
 
 	let monaco: SimpleEditor | undefined = $state(undefined)
@@ -87,6 +90,8 @@
 	let argInput: ArgInput | undefined = $state(undefined)
 	let focusedPrev = false
 
+	let hidden = $state(false)
+
 	const variableMatch = (value: string): RegExpMatchArray | null =>
 		value.match(/^variable\('([^']+)'\)$/)
 	const resourceMatch = (value: string): RegExpMatchArray | null =>
@@ -259,6 +264,47 @@
 		}
 	}
 
+	function handleFieldVisibility(
+		schema: Schema | any,
+		arg: InputTransform | any,
+		otherArgs: Record<string, any>
+	) {
+		const schemaProperty = schema?.properties?.[argName]
+		if (schemaProperty?.showExpr) {
+			// Build args object with current field value and other context
+			const currentValue = propertyType === 'static' ? arg?.value : arg?.expr
+
+			// Convert otherArgs from InputTransform objects to their actual values
+			const contextArgs = {
+				[argName]: currentValue
+			}
+
+			// Extract values from InputTransform objects in otherArgs
+			Object.keys(otherArgs ?? {}).forEach((key) => {
+				const otherArg = otherArgs[key]
+				const otherArgValue = otherArg.type === 'static' ? otherArg.value : otherArg.expr
+				contextArgs[key] = otherArgValue
+			})
+
+			const shouldShow = computeShow(argName, schemaProperty.showExpr, contextArgs)
+			if (shouldShow) {
+				hidden = false
+			} else if (!hidden) {
+				hidden = true
+				// Clear the arg value when hidden (following SchemaForm pattern)
+				if (arg) {
+					arg.value = undefined
+					arg.expr = undefined
+				}
+				// Make sure validation passes when hidden
+				inputCheck = true
+			}
+		} else {
+			// No showExpr, always show
+			hidden = false
+		}
+	}
+
 	function onFocus() {
 		focused = true
 		if (isStaticTemplate(inputCat)) {
@@ -359,12 +405,20 @@
 	$effect(() => {
 		schema?.properties?.[argName]?.default && untrack(() => setDefaultCode())
 	})
+	$effect.pre(() => {
+		// Monitor changes that affect field visibility
+		JSON.stringify(schema)
+		JSON.stringify(arg)
+		JSON.stringify(otherArgs)
+
+		untrack(() => handleFieldVisibility(schema, arg, otherArgs))
+	})
 	let connecting = $derived(
 		$propPickerConfig?.propName == argName && $propPickerConfig?.insertionMode == 'connect'
 	)
 </script>
 
-{#if arg != undefined}
+{#if arg != undefined && !hidden}
 	<div
 		class={twMerge(
 			'pl-2 pt-2 pb-2 ml-2 relative hover:bg-surface hover:shadow-md transition-all duration-200',

frontend/src/lib/components/InputTransformSchemaForm.svelte

@@ -115,6 +115,9 @@
 						{noDynamicToggle}
 						{pickableProperties}
 						{enableAi}
+						otherArgs={Object.fromEntries(
+							Object.entries(args ?? {}).filter(([key]) => key !== argName)
+						)}
 					/>
 				</div>
 			{/if}

frontend/src/lib/components/flows/flowInfers.ts

@@ -62,50 +62,58 @@ export async function loadSchemaFromModule(module: FlowModule): Promise<{
 					type: 'object',
 					format: 'ai-provider'
 				},
+				output_type: {
+					type: 'string',
+					description:
+						'The type of output the AI agent will generate (text or image). Image output requires a configured workspace S3 storage, will ignore tools, and only works with OpenAI, Google AI and OpenRouter gemini-image-preview model.',
+					enum: ['text', 'image'],
+					default: 'text'
+				},
 				user_message: {
-					type: 'string'
+					type: 'string',
+					description: 'The message to give as input to the AI agent.'
 				},
 				system_prompt: {
-					type: 'string'
+					type: 'string',
+					description: 'The system prompt to give as input to the AI agent.'
 				},
-				image: {
-					type: 'object',
-					description: 'Image to send to the AI agent (optional)',
-					format: 'resource-s3_object'
+				user_images: {
+					type: 'array',
+					description:
+						'Array of images to give as input to the AI agent. Requires a configured workspace S3 storage.',
+					items: {
+						type: 'object' as const,
+						resourceType: 's3object'
+					}
 				},
 				max_completion_tokens: {
-					type: 'number'
+					type: 'number',
+					description: 'The maximum number of output tokens.'
 				},
 				temperature: {
 					type: 'number',
 					description:
-						'Controls randomness in text generation. Range: 0.0 (deterministic) to 2.0 (random).'
-				},
-				output_type: {
-					type: 'string',
-					description:
-						'The type of output the AI agent will generate (text or image). Image output will ignore tools, and only works with OpenAI, Google AI and OpenRouter gemini-image-preview model.',
-					enum: ['text', 'image'],
-					default: 'text'
+						'Controls randomness in text generation. Range: 0.0 (deterministic) to 2.0 (random).',
+					showExpr: "fields.output_type === 'text'"
 				},
 				output_schema: {
 					type: 'object',
 					description:
 						'JSON schema that the AI agent will follow for its response format (only used if output_type is text)',
-					format: 'json-schema'
+					format: 'json-schema',
+					showExpr: "fields.output_type === 'text'"
 				}
 			},
-			required: ['provider', 'model', 'user_message'],
+			required: ['provider', 'user_message', 'output_type'],
 			type: 'object',
 			order: [
 				'provider',
-				'model',
+				'output_type',
 				'user_message',
 				'system_prompt',
-				'image',
+				'user_images',
 				'max_completion_tokens',
 				'temperature',
-				'output_type',
 				'output_schema'
 			]
 		}