From 9325f5636c2e957a724bb051d41e15d2966b2899 Mon Sep 17 00:00:00 2001 From: wendrul <53628737+wendrul@users.noreply.github.com> Date: Wed, 17 Sep 2025 18:47:27 +0200 Subject: [PATCH 01/17] fix: Don't reencrypt secrets on workspace forking (#6622) * Do not reencrypt variable when performing a workspace fork * Update SQLx metadata * Update SQLx metadata * Remove unused --------- Co-authored-by: GitHub Action --- ...7a40ef42ca940ca4ff7f9fb2e69e63d830d27.json | 70 ------------ ...e36dc8e96a63c4690721ae0bc169a3d83aff1.json | 22 ---- ...5bcf902d143eec8faec306c651e74ad14c68e.json | 15 +++ ...62dd86568e89f278b93320911f8ae5475f038.json | 15 +++ backend/windmill-api/src/variables.rs | 21 +++- backend/windmill-api/src/workspaces.rs | 102 ++++-------------- backend/windmill-api/src/workspaces_export.rs | 7 +- backend/windmill-common/src/variables.rs | 7 +- backend/windmill-common/src/workspaces.rs | 7 +- backend/windmill-worker/src/common.rs | 7 +- 10 files changed, 89 insertions(+), 184 deletions(-) delete mode 100644 backend/.sqlx/query-31869c5dba5cefd4ffaae7720617a40ef42ca940ca4ff7f9fb2e69e63d830d27.json delete mode 100644 backend/.sqlx/query-a6d1b80e1b407610987c98521f8e36dc8e96a63c4690721ae0bc169a3d83aff1.json create mode 100644 backend/.sqlx/query-bd5d39d1ef26ac0526a2ae834b45bcf902d143eec8faec306c651e74ad14c68e.json create mode 100644 backend/.sqlx/query-c4121e4d1de409f66f1984ad2a962dd86568e89f278b93320911f8ae5475f038.json diff --git a/backend/.sqlx/query-31869c5dba5cefd4ffaae7720617a40ef42ca940ca4ff7f9fb2e69e63d830d27.json b/backend/.sqlx/query-31869c5dba5cefd4ffaae7720617a40ef42ca940ca4ff7f9fb2e69e63d830d27.json deleted file mode 100644 index e1bf471dd5..0000000000 --- a/backend/.sqlx/query-31869c5dba5cefd4ffaae7720617a40ef42ca940ca4ff7f9fb2e69e63d830d27.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at\n FROM variable \n WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "value", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "is_secret", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "description", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 6, - "name": "account", - "type_info": "Int4" - }, - { - "ordinal": 7, - "name": "is_oauth", - "type_info": "Bool" - }, - { - "ordinal": 8, - "name": "expires_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - true, - false, - true - ] - }, - "hash": "31869c5dba5cefd4ffaae7720617a40ef42ca940ca4ff7f9fb2e69e63d830d27" -} diff --git a/backend/.sqlx/query-a6d1b80e1b407610987c98521f8e36dc8e96a63c4690721ae0bc169a3d83aff1.json b/backend/.sqlx/query-a6d1b80e1b407610987c98521f8e36dc8e96a63c4690721ae0bc169a3d83aff1.json deleted file mode 100644 index 63301d439c..0000000000 --- a/backend/.sqlx/query-a6d1b80e1b407610987c98521f8e36dc8e96a63c4690721ae0bc169a3d83aff1.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Jsonb", - "Int4", - "Bool", - "Timestamptz" - ] - }, - "nullable": [] - }, - "hash": "a6d1b80e1b407610987c98521f8e36dc8e96a63c4690721ae0bc169a3d83aff1" -} diff --git a/backend/.sqlx/query-bd5d39d1ef26ac0526a2ae834b45bcf902d143eec8faec306c651e74ad14c68e.json b/backend/.sqlx/query-bd5d39d1ef26ac0526a2ae834b45bcf902d143eec8faec306c651e74ad14c68e.json new file mode 100644 index 0000000000..9fb02ee9e8 --- /dev/null +++ b/backend/.sqlx/query-bd5d39d1ef26ac0526a2ae834b45bcf902d143eec8faec306c651e74ad14c68e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_key (workspace_id, kind, key)\n SELECT $2, kind, key FROM workspace_key WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "bd5d39d1ef26ac0526a2ae834b45bcf902d143eec8faec306c651e74ad14c68e" +} diff --git a/backend/.sqlx/query-c4121e4d1de409f66f1984ad2a962dd86568e89f278b93320911f8ae5475f038.json b/backend/.sqlx/query-c4121e4d1de409f66f1984ad2a962dd86568e89f278b93320911f8ae5475f038.json new file mode 100644 index 0000000000..a0d0a5dfcb --- /dev/null +++ b/backend/.sqlx/query-c4121e4d1de409f66f1984ad2a962dd86568e89f278b93320911f8ae5475f038.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at)\n SELECT $2, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at\n FROM variable\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "c4121e4d1de409f66f1984ad2a962dd86568e89f278b93320911f8ae5475f038" +} diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index cca13a7c25..30cc9c0df3 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -210,7 +210,12 @@ async fn get_variable( } else if !value.is_empty() && decrypt_secret { let _ = tx.commit().await; let mc = build_crypt(&db, &w_id).await?; - Some(decrypt(&mc, value)?) + Some(decrypt(&mc, value).map_err(|e| { + Error::internal_err(format!( + "Error decrypting variable {}: {}", + variable.path, e + )) + })?) } else if q.include_encrypted.unwrap_or(false) { Some(value) } else { @@ -837,7 +842,12 @@ pub async fn get_value_internal<'a, 'e, A: sqlx::Acquire<'e, Database = Postgres return Err(Error::internal_err("Require oauth2 feature".to_string())); } else if !value.is_empty() { let mc = build_crypt(&db, &w_id).await?; - decrypt(&mc, value)? + decrypt(&mc, value).map_err(|e| { + Error::internal_err(format!( + "Error decrypting variable {}: {}", + variable.path, e + )) + })? } else { "".to_string() } @@ -873,7 +883,12 @@ pub async fn get_variable_or_self(path: String, db: &DB, w_id: &str) -> Result, source_workspace_id: &str, target_workspace_id: &str, - db: &DB, ) -> Result<()> { // Clone workspace settings (merge with existing basic settings) update_workspace_settings(tx, source_workspace_id, target_workspace_id).await?; @@ -2363,7 +2361,7 @@ async fn clone_workspace_data( clone_resources(tx, source_workspace_id, target_workspace_id).await?; // Clone variables with re-encryption - clone_variables(tx, source_workspace_id, target_workspace_id, db).await?; + clone_variables(tx, source_workspace_id, target_workspace_id).await?; // Clone scripts with new hashes clone_scripts(tx, source_workspace_id, target_workspace_id).await?; @@ -2391,6 +2389,15 @@ async fn update_workspace_settings( source_workspace_id: &str, target_workspace_id: &str, ) -> Result<()> { + sqlx::query!( + "INSERT INTO workspace_key (workspace_id, kind, key) + SELECT $2, kind, key FROM workspace_key WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + sqlx::query!( r#" UPDATE workspace_settings @@ -2544,81 +2551,18 @@ async fn clone_variables( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, target_workspace_id: &str, - db: &DB, ) -> Result<()> { - // Get all variables from source workspace - let variables = sqlx::query_as!( - ExportableListableVariable, - "SELECT workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at - FROM variable + sqlx::query!( + "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at) + SELECT $2, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at + FROM variable WHERE workspace_id = $1", - source_workspace_id + source_workspace_id, + target_workspace_id, ) - .fetch_all(&mut **tx) + .execute(&mut **tx) .await?; - if variables.is_empty() { - return Ok(()); - } - - // Get workspace keys from within the transaction - let source_key = sqlx::query_scalar!( - "SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud'", - source_workspace_id - ) - .fetch_one(db) - .await?; - - let target_key = sqlx::query_scalar!( - "SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud'", - target_workspace_id - ) - .fetch_one(&mut **tx) - .await?; - - // Build encryption keys manually - use windmill_common::variables::SECRET_SALT; - let source_crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { - format!("{}{}", source_key, salt) - } else { - source_key - }; - let target_crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { - format!("{}{}", target_key, salt) - } else { - target_key - }; - - let source_mc = magic_crypt::new_magic_crypt!(source_crypt_key, 256); - let target_mc = magic_crypt::new_magic_crypt!(target_crypt_key, 256); - - // Process each variable - for var in variables { - let final_value = if var.is_secret && var.value.is_some() { - // Decrypt with source key and re-encrypt with target key - let decrypted_value = decrypt(&source_mc, var.value.unwrap())?; - Some(encrypt(&target_mc, &decrypted_value)) - } else { - var.value - }; - - sqlx::query!( - "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", - target_workspace_id, - var.path, - final_value, - var.is_secret, - var.description, - var.extra_perms, - var.account, - var.is_oauth, - var.expires_at, - ) - .execute(&mut **tx) - .await?; - } - Ok(()) } @@ -2984,16 +2928,6 @@ async fn create_workspace_fork( ) .execute(&mut *tx) .await?; - let key = rd_string(64); - sqlx::query!( - "INSERT INTO workspace_key - (workspace_id, kind, key) - VALUES ($1, 'cloud', $2)", - forked_id, - &key - ) - .execute(&mut *tx) - .await?; sqlx::query!( "INSERT INTO usr @@ -3008,7 +2942,7 @@ async fn create_workspace_fork( .await?; // Clone all data from the parent workspace using Rust implementation - clone_workspace_data(&mut tx, &nw.parent_workspace_id, &forked_id, &db).await?; + clone_workspace_data(&mut tx, &nw.parent_workspace_id, &forked_id).await?; sqlx::query!( "INSERT INTO workspace_invite (workspace_id, email, is_admin, operator) diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index fb7afaccaa..e0a883ef5a 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -508,7 +508,12 @@ pub(crate) async fn tarball_workspace( && var.value.is_some() && var.is_secret { - var.value = Some(decrypt(&mc, var.value.unwrap())?); + var.value = Some(decrypt(&mc, var.value.unwrap()).map_err(|e| { + Error::internal_err(format!( + "Error decrypting variable {}: {}", + var.path, e + )) + })?); } let var_str = &to_string_without_metadata(&var, false, None).unwrap(); archive diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index ae81512a0a..92edbcc348 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -164,7 +164,12 @@ pub async fn get_secret_value_as_admin( let value = variable.value; if !value.is_empty() { let mc = build_crypt(db, w_id).await?; - decrypt(&mc, value)? + decrypt(&mc, value).map_err(|e| { + crate::error::Error::internal_err(format!( + "Error decrypting variable {}: {}", + variable.path, e + )) + })? } else { "".to_string() } diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 70be40e28b..f1a0a35be3 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -268,7 +268,12 @@ async fn transform_json_unchecked( .await .map_err(to_anyhow)?; let mc = build_crypt(&db, &w_id).await?; - let variable = decrypt(&mc, variable)?; + let variable = decrypt(&mc, variable).map_err(|e| { + Error::internal_err(format!( + "Error decrypting variable {}: {}", + &s, e + )) + })?; serde_json::Value::String(variable) } s @ serde_json::Value::String(_) => s.clone(), diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 723d99e062..fbdac8909e 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -261,7 +261,11 @@ pub async fn transform_json_value( ) .await?; decrypt(&mc, encrypted.to_string()).and_then(|x| { - serde_json::from_str(&x).map_err(|e| Error::internal_err(e.to_string())) + serde_json::from_str(&x).map_err(|e| { + Error::internal_err(format!( + "Failed to decrypt '$encrypted:' value: {e}" + )) + }) }) } Connection::Http(_) => { @@ -1120,4 +1124,3 @@ pub fn s3_mode_args_to_worker_data( workspace_id: job.workspace_id.clone(), } } - From 426065efee5e5e775dde403f7e7f7c78d002909a Mon Sep 17 00:00:00 2001 From: dieriba Date: Wed, 17 Sep 2025 18:48:37 +0200 Subject: [PATCH 02/17] fix: add ack deadline gcp (#6625) * update ref * add ack deadline * update ref --- ...bef428fc4f0d52776d7a5fca64f17225ef30.json} | 7 +++-- ...a2ee318c72b365c0fe0b9fbec886a461f5e4.json} | 7 +++-- backend/ee-repo-ref.txt | 2 +- ...4_add_ack_deadline_to_gcp_trigger.down.sql | 2 ++ ...504_add_ack_deadline_to_gcp_trigger.up.sql | 2 ++ backend/windmill-api/openapi.yaml | 6 ++++ backend/windmill-api/src/capture.rs | 2 ++ .../gcp/GcpTriggerEditorConfigSection.svelte | 2 ++ .../triggers/gcp/GcpTriggerEditorInner.svelte | 29 ++++++++++++++++++- .../src/lib/components/triggers/gcp/utils.ts | 1 + 10 files changed, 52 insertions(+), 8 deletions(-) rename backend/.sqlx/{query-1a9ba16c90d3d65c4ff39aaddb3079009e03af711e7f6b53332537cf4cb0e8dd.json => query-17ca259e1c78e1317fdd19436e15bef428fc4f0d52776d7a5fca64f17225ef30.json} (83%) rename backend/.sqlx/{query-e619fa013528a6275f98e14ae1727c55b0d4f4a5e4ee87c29251042e2916f0a0.json => query-e7c61bbdcf882f6e1e9b11df03e7a2ee318c72b365c0fe0b9fbec886a461f5e4.json} (67%) create mode 100644 backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.down.sql create mode 100644 backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.up.sql diff --git a/backend/.sqlx/query-1a9ba16c90d3d65c4ff39aaddb3079009e03af711e7f6b53332537cf4cb0e8dd.json b/backend/.sqlx/query-17ca259e1c78e1317fdd19436e15bef428fc4f0d52776d7a5fca64f17225ef30.json similarity index 83% rename from backend/.sqlx/query-1a9ba16c90d3d65c4ff39aaddb3079009e03af711e7f6b53332537cf4cb0e8dd.json rename to backend/.sqlx/query-17ca259e1c78e1317fdd19436e15bef428fc4f0d52776d7a5fca64f17225ef30.json index 141f92e271..dd40aabbc1 100644 --- a/backend/.sqlx/query-1a9ba16c90d3d65c4ff39aaddb3079009e03af711e7f6b53332537cf4cb0e8dd.json +++ b/backend/.sqlx/query-17ca259e1c78e1317fdd19436e15bef428fc4f0d52776d7a5fca64f17225ef30.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE \n gcp_trigger \n SET \n gcp_resource_path = $1,\n subscription_id = $2,\n topic_id = $3,\n delivery_type = $4,\n delivery_config = $5,\n is_flow = $6, \n edited_by = $7, \n email = $8,\n script_path = $9,\n path = $10,\n enabled = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16,\n auto_acknowledge_msg = $17\n WHERE \n workspace_id = $12 AND \n path = $13\n ", + "query": "\n UPDATE \n gcp_trigger \n SET \n gcp_resource_path = $1,\n subscription_id = $2,\n topic_id = $3,\n delivery_type = $4,\n delivery_config = $5,\n is_flow = $6, \n edited_by = $7, \n email = $8,\n script_path = $9,\n path = $10,\n enabled = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16,\n auto_acknowledge_msg = $17,\n ack_deadline = $18\n WHERE \n workspace_id = $12 AND \n path = $13\n ", "describe": { "columns": [], "parameters": { @@ -31,10 +31,11 @@ "Varchar", "Jsonb", "Jsonb", - "Bool" + "Bool", + "Int4" ] }, "nullable": [] }, - "hash": "1a9ba16c90d3d65c4ff39aaddb3079009e03af711e7f6b53332537cf4cb0e8dd" + "hash": "17ca259e1c78e1317fdd19436e15bef428fc4f0d52776d7a5fca64f17225ef30" } diff --git a/backend/.sqlx/query-e619fa013528a6275f98e14ae1727c55b0d4f4a5e4ee87c29251042e2916f0a0.json b/backend/.sqlx/query-e7c61bbdcf882f6e1e9b11df03e7a2ee318c72b365c0fe0b9fbec886a461f5e4.json similarity index 67% rename from backend/.sqlx/query-e619fa013528a6275f98e14ae1727c55b0d4f4a5e4ee87c29251042e2916f0a0.json rename to backend/.sqlx/query-e7c61bbdcf882f6e1e9b11df03e7a2ee318c72b365c0fe0b9fbec886a461f5e4.json index 2874c70c1c..6d41284328 100644 --- a/backend/.sqlx/query-e619fa013528a6275f98e14ae1727c55b0d4f4a5e4ee87c29251042e2916f0a0.json +++ b/backend/.sqlx/query-e7c61bbdcf882f6e1e9b11df03e7a2ee318c72b365c0fe0b9fbec886a461f5e4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO gcp_trigger (\n gcp_resource_path,\n subscription_id,\n topic_id,\n delivery_type,\n delivery_config,\n workspace_id, \n path, \n script_path, \n is_flow, \n email, \n enabled, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry,\n auto_acknowledge_msg\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4,\n $5,\n $6, \n $7, \n $8, \n $9,\n $10,\n $11,\n $12,\n $13,\n $14,\n $15,\n $16\n )", + "query": "\n INSERT INTO gcp_trigger (\n gcp_resource_path,\n subscription_id,\n topic_id,\n delivery_type,\n delivery_config,\n workspace_id, \n path, \n script_path, \n is_flow, \n email, \n enabled, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry,\n auto_acknowledge_msg,\n ack_deadline\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4,\n $5,\n $6, \n $7, \n $8, \n $9,\n $10,\n $11,\n $12,\n $13,\n $14,\n $15,\n $16,\n $17\n )", "describe": { "columns": [], "parameters": { @@ -30,10 +30,11 @@ "Varchar", "Jsonb", "Jsonb", - "Bool" + "Bool", + "Int4" ] }, "nullable": [] }, - "hash": "e619fa013528a6275f98e14ae1727c55b0d4f4a5e4ee87c29251042e2916f0a0" + "hash": "e7c61bbdcf882f6e1e9b11df03e7a2ee318c72b365c0fe0b9fbec886a461f5e4" } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index d9e0f1c4b1..8e99fde120 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -08f610ecc5486ad2568e0abd22405baa90a93481 +89ecae7db8c46cfc571eb58bcd7f4ba6f78e8aba \ No newline at end of file diff --git a/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.down.sql b/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.down.sql new file mode 100644 index 0000000000..144aab482d --- /dev/null +++ b/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE gcp_trigger DROP COLUMN ack_deadline; \ No newline at end of file diff --git a/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.up.sql b/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.up.sql new file mode 100644 index 0000000000..7a85bd7c82 --- /dev/null +++ b/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE gcp_trigger ADD COLUMN ack_deadline INTEGER; \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0e05dfbe9b..c385a78363 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -16818,6 +16818,12 @@ components: type: boolean auto_acknowledge_msg: type: boolean + ack_deadline: + type: integer + format: int32 + minimum: 10 + maximum: 600 + description: "Time in seconds within which the message must be acknowledged. If not provided, defaults to the subscription's acknowledgment deadline (600 seconds)." error_handler_path: type: string error_handler_args: diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 603c040475..3283dea123 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -183,6 +183,7 @@ pub struct GcpTriggerConfig { pub create_update: Option, pub topic_id: String, pub auto_acknowledge_msg: Option, + pub ack_deadline: Option, } #[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] @@ -397,6 +398,7 @@ async fn set_gcp_trigger_config( gcp_config.create_update, false, capture_config.is_flow, + gcp_config.ack_deadline ) .await?; gcp_config.create_update = Some(config); diff --git a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorConfigSection.svelte b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorConfigSection.svelte index 5b6f695f43..76858f79fb 100644 --- a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorConfigSection.svelte +++ b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorConfigSection.svelte @@ -83,6 +83,7 @@ cloud_subscription_id?: string create_update_subscription_id?: string auto_acknowledge_msg: boolean + ack_deadline?: number } let { @@ -97,6 +98,7 @@ subscription_mode = $bindable('create_update'), base_endpoint = $bindable(getBaseUrl()), auto_acknowledge_msg = $bindable(true), + ack_deadline = $bindable(), path = '', showTestingBadge = false, cloud_subscription_id = $bindable(''), diff --git a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte index d6044e765a..edb2e2d644 100644 --- a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte @@ -55,6 +55,7 @@ let deploymentLoading = $state(false) let base_endpoint = $derived(`${window.location.origin}${base}`) let auto_acknowledge_msg = $state(true) + let ack_deadline: number | undefined = $state() let optionTabSelected: 'settings' | 'error_handler' | 'retries' = $state('error_handler') let errorHandlerSelected: ErrorHandler = $state('slack') let error_handler_path: string | undefined = $state() @@ -151,6 +152,7 @@ error_handler_args = defaultValues?.error_handler_args ?? {} retry = defaultValues?.retry ?? undefined auto_acknowledge_msg = defaultValues?.auto_acknowledge_msg ?? true + ack_deadline = defaultValues?.ack_deadline errorHandlerSelected = getHandlerType(error_handler_path ?? '') } finally { drawerLoading = false @@ -191,6 +193,7 @@ error_handler_args = cfg?.error_handler_args ?? {} retry = cfg?.retry auto_acknowledge_msg = cfg?.auto_acknowledge_msg ?? true + ack_deadline = cfg?.ack_deadline errorHandlerSelected = getHandlerType(error_handler_path ?? '') } @@ -230,7 +233,8 @@ error_handler_path, error_handler_args, retry, - auto_acknowledge_msg + auto_acknowledge_msg, + ack_deadline } } @@ -243,6 +247,7 @@ delivery_config, base_endpoint, auto_acknowledge_msg, + ack_deadline, topic_id, path } @@ -400,6 +405,7 @@ bind:topic_id bind:subscription_mode bind:auto_acknowledge_msg + bind:ack_deadline {path} cloud_subscription_id={subscription_id} create_update_subscription_id={subscription_id} @@ -439,6 +445,27 @@ {/if} + +
+ +
+
+ Leave empty to use subscription default (600 seconds). This affects how long + messages remain in flight before being redelivered. +
+
{:else}
No settings available for push delivery type diff --git a/frontend/src/lib/components/triggers/gcp/utils.ts b/frontend/src/lib/components/triggers/gcp/utils.ts index 684604260f..eb8430786f 100644 --- a/frontend/src/lib/components/triggers/gcp/utils.ts +++ b/frontend/src/lib/components/triggers/gcp/utils.ts @@ -31,6 +31,7 @@ export async function saveGcpTriggerFromCfg( enabled: cfg.enabled, is_flow: cfg.is_flow, auto_acknowledge_msg: cfg.auto_acknowledge_msg, + ack_deadline: cfg.ack_deadline, ...errorHandlerAndRetries } if (edit) { From d837badf2c70c483e260b099a663fdda3f1f509a Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Wed, 17 Sep 2025 18:55:30 +0200 Subject: [PATCH 03/17] feat(aichat): add max tokens settings (#6613) * add max tokens settings * higher max * fixes * save max tokens in workspace settings * cleaning * cleaning * cleaning * feat(ai): add collapsible sections to ModelTokenLimits component - Add collapsible/expandable sections for each AI provider - Display 'Modified' badge when providers have custom settings - Use ChevronDown/ChevronUp icons for visual feedback - Add smooth slide transitions for better UX - Reduce vertical space usage in workspace settings Co-authored-by: centdix * adjust * nit --------- Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> Co-authored-by: centdix --- backend/windmill-api/openapi.yaml | 6 + backend/windmill-api/src/ai.rs | 2 + backend/windmill-api/src/workspaces.rs | 1 + .../copilot/chat/AIChatDisplay.svelte | 4 +- .../copilot/chat/ProviderModelSelector.svelte | 2 +- frontend/src/lib/components/copilot/lib.ts | 8 +- .../workspaceSettings/AISettings.svelte | 10 +- .../workspaceSettings/ModelTokenLimits.svelte | 192 ++++++++++++++++++ frontend/src/lib/stores.ts | 11 +- .../(logged)/workspace_settings/+page.svelte | 3 + 10 files changed, 231 insertions(+), 8 deletions(-) create mode 100644 frontend/src/lib/components/workspaceSettings/ModelTokenLimits.svelte diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c385a78363..b6ca1b56c8 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -14514,6 +14514,12 @@ components: type: object additionalProperties: type: string + max_tokens_per_model: + type: object + additionalProperties: + type: integer + minimum: 1 + maximum: 2000000 Alert: type: object diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 9f13d32cf5..abde227214 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -272,6 +272,8 @@ pub struct AIConfig { pub code_completion_model: Option, #[serde(skip_serializing_if = "Option::is_none")] pub custom_prompts: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_tokens_per_model: Option>, } pub fn global_service() -> Router { diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index b19f47b6cb..1b9d928cf4 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -923,6 +923,7 @@ async fn get_copilot_info( default_model: None, code_completion_model: None, custom_prompts: None, + max_tokens_per_model: None, })) } } diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 75489ecfaa..b3f624a9d2 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -84,7 +84,9 @@ } }) - const isLastMessageTool = $derived(messages.length > 0 && messages[messages.length - 1].role === 'tool') + const isLastMessageTool = $derived( + messages.length > 0 && messages[messages.length - 1].role === 'tool' + )
diff --git a/frontend/src/lib/components/copilot/chat/ProviderModelSelector.svelte b/frontend/src/lib/components/copilot/chat/ProviderModelSelector.svelte index 6c7068a80c..ffd2b7075a 100644 --- a/frontend/src/lib/components/copilot/chat/ProviderModelSelector.svelte +++ b/frontend/src/lib/components/copilot/chat/ProviderModelSelector.svelte @@ -26,7 +26,7 @@
{providerModel.model} {#if multipleModels} -
+
{/if} diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index addcda69e2..0ccc7ab975 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -1,5 +1,6 @@ import type { AIProvider, AIProviderModel } from '$lib/gen' import { + copilotInfo, getCurrentModel, workspaceStore, type DBSchema, @@ -157,7 +158,7 @@ export async function fetchAvailableModels( return data?.data.map((m) => m.id) ?? [] } -function getModelMaxTokens(provider: AIProvider, model: string) { +export function getModelMaxTokens(provider: AIProvider, model: string) { if (model.startsWith('gpt-5')) { return 128000 } else if ((provider === 'azure_openai' || provider === 'openai') && model.startsWith('o')) { @@ -196,7 +197,10 @@ function getModelSpecificConfig( modelProvider: AIProviderModel, tools?: OpenAI.Chat.Completions.ChatCompletionTool[] ) { - const maxTokens = getModelMaxTokens(modelProvider.provider, modelProvider.model) + const defaultMaxTokens = getModelMaxTokens(modelProvider.provider, modelProvider.model) + const modelKey = `${modelProvider.provider}:${modelProvider.model}` + const customMaxTokensStore = get(copilotInfo)?.maxTokensPerModel + const maxTokens = customMaxTokensStore?.[modelKey] ?? defaultMaxTokens if ( (modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai') && (modelProvider.model.startsWith('o') || modelProvider.model.startsWith('gpt-5')) diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 0b34bf0b54..06afffee6f 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -18,6 +18,7 @@ import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' import autosize from '$lib/autosize' + import ModelTokenLimits from './ModelTokenLimits.svelte' const MAX_CUSTOM_PROMPT_LENGTH = 5000 @@ -26,12 +27,14 @@ codeCompletionModel = $bindable(), defaultModel = $bindable(), customPrompts = $bindable(), + maxTokensPerModel = $bindable(), usingOpenaiClientCredentialsOauth = $bindable() }: { aiProviders: Exclude codeCompletionModel: string | undefined defaultModel: string | undefined customPrompts: Record + maxTokensPerModel: Record usingOpenaiClientCredentialsOauth: boolean } = $props() @@ -101,7 +104,8 @@ providers: aiProviders, code_completion_model, default_model, - custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined + custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined, + max_tokens_per_model: Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined } await WorkspaceService.editCopilotConfig({ workspace: $workspaceStore!, @@ -317,6 +321,10 @@
{/if} + {#if Object.keys(aiProviders).length > 0} + + {/if} + {#if Object.keys(aiProviders).length > 0}

Custom system prompts

diff --git a/frontend/src/lib/components/workspaceSettings/ModelTokenLimits.svelte b/frontend/src/lib/components/workspaceSettings/ModelTokenLimits.svelte new file mode 100644 index 0000000000..1ecf775c08 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/ModelTokenLimits.svelte @@ -0,0 +1,192 @@ + + +{#if Object.keys(aiProviders).length > 0} +
+
+

Model Output Limits

+

+ Configure maximum token limits for each model. These limits apply to all AI chat + interactions in the workspace. +

+
+ +
+ {#each Object.entries(modelsByProvider).filter(([provider, models]) => models.length > 0) as [provider, models]} + {@const isExpanded = !collapsedProviders[provider]} + {@const hasCustom = hasCustomSettings(provider, models)} +
+ + + {#if isExpanded} +
+
+ {#each models as { model }} + {@const currentTokens = getCurrentTokensForModel(provider as AIProvider, model)} + {@const defaultTokens = getDefaultTokensForModel(provider as AIProvider, model)} + {@const isAtDefault = isModelAtDefault(provider as AIProvider, model)} +
+
+
+ {model} +
+
+ { + const value = parseInt(e.currentTarget.value) + if (!isNaN(value)) { + updateTokensForModel(provider as AIProvider, model, value) + } + }} + class="w-20 px-2 py-1 text-xs text-center border border-gray-200 dark:border-gray-700 rounded bg-surface focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent" + /> + tokens +
+
+ {#if !isAtDefault} +
+ Default: {defaultTokens} tokens + +
+ {#if errors[getModelKey(provider as AIProvider, model)]} +
{errors[getModelKey(provider as AIProvider, model)]}
+ {/if} + {/if} +
+ {/each} +
+
+ {/if} +
+ {/each} +
+
+{/if} diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index 61f22fc478..5c33b2cca6 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -102,12 +102,14 @@ export const copilotInfo = writable<{ defaultModel?: AIProviderModel aiModels: AIProviderModel[] customPrompts?: Record + maxTokensPerModel?: Record }>({ enabled: false, codeCompletionModel: undefined, defaultModel: undefined, aiModels: [], - customPrompts: {} + customPrompts: {}, + maxTokensPerModel: {} }) export async function loadCopilot(workspace: string) { @@ -143,7 +145,8 @@ export function setCopilotInfo(aiConfig: AIConfig) { codeCompletionModel: aiConfig.code_completion_model, defaultModel: aiConfig.default_model, aiModels: aiModels, - customPrompts: aiConfig.custom_prompts ?? {} + customPrompts: aiConfig.custom_prompts ?? {}, + maxTokensPerModel: aiConfig.max_tokens_per_model ?? {} }) } else { copilotSessionModel.set(undefined) @@ -153,7 +156,8 @@ export function setCopilotInfo(aiConfig: AIConfig) { codeCompletionModel: undefined, defaultModel: undefined, aiModels: [], - customPrompts: {} + customPrompts: {}, + maxTokensPerModel: {} }) } } @@ -197,6 +201,7 @@ export const copilotSessionModel = writable( } : undefined ) + export const usedTriggerKinds = writable([]) type SQLBaseSchema = { diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index f7251ec844..47fd6e33b7 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -81,6 +81,7 @@ let codeCompletionModel: string | undefined = $state(undefined) let defaultModel: string | undefined = $state(undefined) let customPrompts: Record = $state({}) + let maxTokensPerModel: Record = $state({}) let s3ResourceSettings: S3ResourceSettings = $state({ resourceType: 's3', @@ -246,6 +247,7 @@ defaultModel = settings.ai_config?.default_model?.model codeCompletionModel = settings.ai_config?.code_completion_model?.model customPrompts = settings.ai_config?.custom_prompts ?? {} + maxTokensPerModel = settings.ai_config?.max_tokens_per_model ?? {} for (const mode of Object.values(AIMode)) { if (!(mode in customPrompts)) { customPrompts[mode] = '' @@ -813,6 +815,7 @@ bind:codeCompletionModel bind:defaultModel bind:customPrompts + bind:maxTokensPerModel bind:usingOpenaiClientCredentialsOauth /> {:else if tab == 'windmill_lfs'} From 3199f9fffd36c84d7cbb4a512935f7eb19fa5049 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Wed, 17 Sep 2025 18:55:54 +0200 Subject: [PATCH 04/17] feat(ai agent): allow multiple images input for ai agent + code cleaning (#6591) * add in frontend * draft openai handling * upload to s3 * simpler output * return s3 directly if any * low quality * implement for gemini * handle imagen model * handle image input * cleaning * remove base64 from output * cleaning * fix timeout * handle openrouter * remove log * allow image input when creating image * cleaning * increase stack size * inline everything * revert stack size * refactor: move AI executor types to separate module - Created ai module structure with types.rs - Moved all type definitions from ai_executor.rs to ai/types.rs - No functional changes, just code organization * refactor: add QueryBuilder trait and provider detection utilities - Created QueryBuilder trait for abstracting provider-specific logic - Added helper functions for provider detection (is_anthropic_provider) - Implemented placeholder QueryBuilder for all providers - Updated OpenAIRequest to use slices instead of Vec references - All providers now have QueryBuilder implementations (using default for now) * feat: implement OpenAI query builder with image support foundation - Created proper OpenAI query builder implementation - Added image_handler module for S3 upload/download utilities - Separated text and image request building logic - Added prepare_messages_for_api to handle S3Object conversion - Foundation laid for supporting tools with image output * refactor(ai): complete AI executor refactoring with query builder pattern - Created modular structure under ai/ module - Moved all types to ai/types.rs - Created QueryBuilder trait for provider abstraction - Implemented OpenAI query builder with image+tools support - Added unified agent runner supporting both text and image outputs with tools - Refactored run_agent to delegate to new unified implementation - Added image handler utilities for S3 operations - Improved code organization and maintainability * cleaning * feat(ai): implement remaining provider query builders - Added Anthropic query builder with proper message conversion - Added Google AI query builder with Gemini API support - Added OpenRouter query builder delegating to OpenAI for compatibility - Added missing Anthropic and Gemini types to types.rs - Fixed type references and compilation errors - All providers now support the unified query builder interface * fixes * fixes * mime type + cleaning * image to images * handle mutlitple images * fix * remove agent_runner file * clean query builder logic * cleaning * cleaning * hide structured_output based on output type * fix * user images and not nested * better descriptions --- backend/Cargo.lock | 1 + backend/windmill-worker/Cargo.toml | 1 + .../windmill-worker/src/ai/image_handler.rs | 68 + backend/windmill-worker/src/ai/mod.rs | 7 + .../src/ai/providers/google_ai.rs | 275 ++ .../windmill-worker/src/ai/providers/mod.rs | 3 + .../src/ai/providers/openai.rs | 349 +++ .../src/ai/providers/openrouter.rs | 202 ++ .../windmill-worker/src/ai/query_builder.rs | 70 + backend/windmill-worker/src/ai/types.rs | 368 +++ backend/windmill-worker/src/ai_executor.rs | 2276 +++++------------ backend/windmill-worker/src/lib.rs | 1 + .../lib/components/InputTransformForm.svelte | 58 +- .../InputTransformSchemaForm.svelte | 3 + .../src/lib/components/flows/flowInfers.ts | 48 +- 15 files changed, 2076 insertions(+), 1654 deletions(-) create mode 100644 backend/windmill-worker/src/ai/image_handler.rs create mode 100644 backend/windmill-worker/src/ai/mod.rs create mode 100644 backend/windmill-worker/src/ai/providers/google_ai.rs create mode 100644 backend/windmill-worker/src/ai/providers/mod.rs create mode 100644 backend/windmill-worker/src/ai/providers/openai.rs create mode 100644 backend/windmill-worker/src/ai/providers/openrouter.rs create mode 100644 backend/windmill-worker/src/ai/query_builder.rs create mode 100644 backend/windmill-worker/src/ai/types.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 005bec2f26..0a427653f9 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15744,6 +15744,7 @@ dependencies = [ "async-once-cell", "async-recursion", "async-stream", + "async-trait", "backon", "base64 0.22.1", "bit-vec 0.6.3", diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index de7261a44e..19c8c0d31f 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -67,6 +67,7 @@ serde.workspace = true serde_json.workspace = true futures.workspace = true async-recursion.workspace = true +async-trait.workspace = true anyhow.workspace = true itertools.workspace = true regex.workspace = true diff --git a/backend/windmill-worker/src/ai/image_handler.rs b/backend/windmill-worker/src/ai/image_handler.rs new file mode 100644 index 0000000000..c668ded17f --- /dev/null +++ b/backend/windmill-worker/src/ai/image_handler.rs @@ -0,0 +1,68 @@ +use base64::Engine; +use futures; +use ulid; +use windmill_common::{client::AuthedClient, error::Error, s3_helpers::S3Object}; +use windmill_queue::MiniPulledJob; + +/// Upload image to S3 and return S3Object +pub async fn upload_image_to_s3( + base64_image: &str, + job: &MiniPulledJob, + client: &AuthedClient, +) -> Result { + let image_bytes = base64::engine::general_purpose::STANDARD + .decode(base64_image) + .map_err(|e| Error::internal_err(format!("Failed to decode base64 image: {}", e)))?; + + // Generate unique S3 key + let unique_id = ulid::Ulid::new().to_string(); + let s3_key = format!("ai_images/{}/{}.png", job.id, unique_id); + + // Create byte stream + let byte_stream = futures::stream::once(async move { + Ok::<_, std::convert::Infallible>(bytes::Bytes::from(image_bytes)) + }); + + // Upload to S3 + client + .upload_s3_file( + &job.workspace_id, + s3_key.clone(), + None, // storage - use default + byte_stream, + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to upload image to S3: {}", e)))?; + + Ok(S3Object { + s3: s3_key, + storage: None, + filename: Some("generated_image.png".to_string()), + presigned: None, + }) +} + +/// Download an S3 image and convert it to a base64 data URL +pub async fn download_and_encode_s3_image( + image: &S3Object, + client: &AuthedClient, + workspace_id: &str, +) -> Result<(String, String), Error> { + // Download the image from S3 + let image_bytes = client + .download_s3_file(workspace_id, &image.s3, image.storage.clone()) + .await + .map_err(|e| Error::internal_err(format!("Failed to download S3 image: {}", e)))?; + + // Encode as base64 data URL + let base64_data = base64::engine::general_purpose::STANDARD.encode(&image_bytes); + + // Determine MIME type using mime_guess from file extension, with PNG as fallback + let mime_type = mime_guess::from_path(&image.s3).first(); + let mime_type = mime_type + .as_ref() + .map(|mime| mime.essence_str()) + .unwrap_or("image/png"); + + Ok((mime_type.to_string(), base64_data)) +} diff --git a/backend/windmill-worker/src/ai/mod.rs b/backend/windmill-worker/src/ai/mod.rs new file mode 100644 index 0000000000..b381e51693 --- /dev/null +++ b/backend/windmill-worker/src/ai/mod.rs @@ -0,0 +1,7 @@ +// AI executor module structure +// This module will contain all AI-related execution logic + +pub mod image_handler; +pub mod providers; +pub mod query_builder; +pub mod types; diff --git a/backend/windmill-worker/src/ai/providers/google_ai.rs b/backend/windmill-worker/src/ai/providers/google_ai.rs new file mode 100644 index 0000000000..9b08d40e46 --- /dev/null +++ b/backend/windmill-worker/src/ai/providers/google_ai.rs @@ -0,0 +1,275 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json; +use windmill_common::{client::AuthedClient, error::Error}; + +use crate::ai::{ + image_handler::download_and_encode_s3_image, + query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder}, + types::*, +}; + +// Google AI/Gemini-specific types +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiInlineData { + #[serde(rename = "mimeType")] + pub mime_type: String, + pub data: String, +} + +#[derive(Serialize, Deserialize, Clone)] +#[serde(untagged)] +pub enum GeminiPart { + Text { text: String }, + InlineData { inline_data: GeminiInlineData }, + FunctionCall { function_call: GeminiFunctionCall }, + FunctionResponse { function_response: GeminiFunctionResponse }, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct GeminiFunctionCall { + pub name: String, + pub args: serde_json::Value, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct GeminiFunctionResponse { + pub name: String, + pub response: serde_json::Value, +} + +#[derive(Serialize)] +pub struct GeminiContent { + pub parts: Vec, +} + +#[derive(Serialize)] +pub struct GeminiImageRequest { + #[serde(skip_serializing_if = "Option::is_none")] + pub contents: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub instances: Option>, +} + +#[derive(Serialize)] +pub struct GeminiPredictContent { + pub prompt: String, +} + +#[derive(Deserialize)] +pub struct GeminiImageResponse { + #[serde(skip_serializing_if = "Option::is_none")] + pub candidates: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub predictions: Option>, +} + +#[derive(Deserialize)] +pub struct GeminiCandidate { + pub content: GeminiResponseContent, +} + +#[derive(Deserialize)] +pub struct GeminiPredictCandidate { + #[serde(rename = "bytesBase64Encoded")] + pub bytes_base64_encoded: String, // base64 encoded image +} + +#[derive(Deserialize)] +pub struct GeminiResponseContent { + pub parts: Vec, +} + +#[derive(Deserialize)] +pub struct GeminiResponsePart { + #[serde(skip_serializing_if = "Option::is_none")] + #[allow(dead_code)] + pub text: Option, + #[serde(rename = "inlineData", skip_serializing_if = "Option::is_none")] + pub inline_data: Option, + #[serde(rename = "functionCall", skip_serializing_if = "Option::is_none")] + #[allow(dead_code)] + pub function_call: Option, +} + +pub struct GoogleAIQueryBuilder; + +impl GoogleAIQueryBuilder { + pub fn new() -> Self { + Self + } + + async fn build_image_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result { + let is_imagen = args.model.contains("imagen"); + + let request = if is_imagen { + // For Imagen models, use simple prompt format + GeminiImageRequest { + instances: Some(vec![GeminiPredictContent { + prompt: args.user_message.trim().to_string(), + }]), + contents: None, + } + } else { + // For Gemini models with image generation, build parts + let mut parts = vec![GeminiPart::Text { text: args.user_message.trim().to_string() }]; + + if let Some(system_prompt) = args.system_prompt { + parts.insert( + 0, + GeminiPart::Text { text: format!("SYSTEM PROMPT: {}", system_prompt.trim()) }, + ); + } + + // Add input images if provided + if let Some(images) = args.images { + for image in images.iter() { + if !image.s3.is_empty() { + let (mime_type, image_bytes) = + download_and_encode_s3_image(image, client, workspace_id).await?; + parts.push(GeminiPart::InlineData { + inline_data: GeminiInlineData { + mime_type: mime_type, + data: image_bytes, + }, + }); + } + } + } + + GeminiImageRequest { instances: None, contents: Some(vec![GeminiContent { parts }]) } + }; + + serde_json::to_string(&request) + .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) + } +} + +#[async_trait] +impl QueryBuilder for GoogleAIQueryBuilder { + fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool { + // Google AI supports tools only for text output + matches!(output_type, OutputType::Text) + } + + async fn build_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result { + match args.output_type { + OutputType::Text => { + // For text output, use OpenAI-compatible format + let openai_builder = super::openai::OpenAIQueryBuilder::new(); + openai_builder + .build_request(args, client, workspace_id) + .await + } + OutputType::Image => self.build_image_request(args, client, workspace_id).await, + } + } + + async fn parse_response(&self, response: reqwest::Response) -> Result { + let url = response.url().path(); + + // For chat completions (text), use OpenAI parser + if url.contains("/chat/completions") { + let openai_builder = super::openai::OpenAIQueryBuilder::new(); + return openai_builder.parse_response(response).await; + } + + // Check if this is an image generation response + if url.contains(":predict") || url.contains(":generateContent") { + let response_text = response + .text() + .await + .map_err(|e| Error::internal_err(format!("Failed to read response text: {}", e)))?; + + let gemini_response: GeminiImageResponse = serde_json::from_str(&response_text) + .map_err(|e| { + Error::internal_err(format!( + "Failed to parse Gemini response: {}. Raw response: {}", + e, response_text + )) + })?; + + // Find image data in response + let image_data = gemini_response + .candidates + .as_ref() + .and_then(|candidates| { + candidates.iter().find_map(|candidate| { + candidate + .content + .parts + .iter() + .find_map(|part| part.inline_data.as_ref().map(|data| &data.data)) + }) + }) + .or_else(|| { + gemini_response + .predictions + .as_ref() + .and_then(|predictions| { + predictions + .iter() + .find_map(|prediction| Some(&prediction.bytes_base64_encoded)) + }) + }); + + if let Some(base64_image) = image_data { + Ok(ParsedResponse::Image { base64_data: base64_image.clone() }) + } else { + Err(Error::internal_err( + "No image data received from Gemini".to_string(), + )) + } + } else { + // This should not happen as we use OpenAI format for text + Err(Error::internal_err( + "Unexpected text response in Google AI parser".to_string(), + )) + } + } + + fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String { + match output_type { + OutputType::Text => format!("{}/chat/completions", base_url), // Use OpenAI-compatible endpoint + OutputType::Image => { + // For image generation, build the full URL with model name + let url_suffix = if model.contains("imagen") { + "predict" + } else { + "generateContent" + }; + format!( + "https://generativelanguage.googleapis.com/v1beta/models/{}:{}", + model, url_suffix + ) + } + } + } + + fn get_auth_headers( + &self, + api_key: &str, + output_type: &OutputType, + ) -> Vec<(&'static str, String)> { + match output_type { + OutputType::Text => { + // For text output, use Bearer token (OpenAI-compatible) + vec![("Authorization", format!("Bearer {}", api_key))] + } + OutputType::Image => { + // For image generation, use Google API key header + vec![("x-goog-api-key", api_key.to_string())] + } + } + } +} diff --git a/backend/windmill-worker/src/ai/providers/mod.rs b/backend/windmill-worker/src/ai/providers/mod.rs new file mode 100644 index 0000000000..13cf766e28 --- /dev/null +++ b/backend/windmill-worker/src/ai/providers/mod.rs @@ -0,0 +1,3 @@ +pub mod google_ai; +pub mod openai; +pub mod openrouter; diff --git a/backend/windmill-worker/src/ai/providers/openai.rs b/backend/windmill-worker/src/ai/providers/openai.rs new file mode 100644 index 0000000000..26bc804aed --- /dev/null +++ b/backend/windmill-worker/src/ai/providers/openai.rs @@ -0,0 +1,349 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json; +use windmill_common::{client::AuthedClient, error::Error}; + +use crate::ai::{ + image_handler::download_and_encode_s3_image, + query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder}, + types::*, +}; + +// OpenAI-specific types +#[derive(Deserialize, Serialize, Clone, Debug)] +pub struct OpenAIFunction { + pub name: String, + pub arguments: String, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +pub struct OpenAIToolCall { + pub id: String, + pub function: OpenAIFunction, + pub r#type: String, +} + +#[derive(Deserialize)] +pub struct OpenAIChoice { + pub message: OpenAIMessage, +} + +#[derive(Deserialize)] +pub struct OpenAIResponse { + pub choices: Vec, +} + +#[derive(Serialize)] +pub struct ImageGenerationTool { + pub r#type: String, + pub quality: Option, + pub background: Option, +} + +// Input content for image generation - supports both text and images +#[derive(Serialize, Clone, Debug)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ImageGenerationContent { + #[serde(rename = "input_text")] + InputText { text: String }, + #[serde(rename = "input_image")] + InputImage { image_url: String }, +} + +#[derive(Serialize)] +pub struct ImageGenerationMessage { + pub role: String, + pub content: Vec, +} + +#[derive(Serialize)] +pub struct ImageGenerationRequest<'a> { + pub model: &'a str, + pub input: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub instructions: Option<&'a str>, + pub tools: Vec, +} + +#[derive(Deserialize)] +pub struct OpenAIImageResponse { + pub output: Vec, +} + +#[derive(Deserialize)] +pub struct OpenAIImageOutput { + pub r#type: String, // Expected to be "image_generation_call" + pub status: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub result: Option, // Base64 encoded image, None if not completed +} + +#[derive(Serialize)] +pub struct OpenAIRequest<'a> { + pub model: &'a str, + pub messages: &'a [OpenAIMessage], + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option<&'a [ToolDef]>, + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_completion_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub response_format: Option, +} + +pub struct OpenAIQueryBuilder; + +impl OpenAIQueryBuilder { + pub fn new() -> Self { + Self + } + + pub async fn prepare_messages_for_api( + &self, + messages: &[OpenAIMessage], + client: &AuthedClient, + workspace_id: &str, + ) -> Result, Error> { + let mut prepared_messages = Vec::new(); + + for message in messages { + let mut prepared_message = message.clone(); + + if let Some(content) = &message.content { + match content { + OpenAIContent::Text(text) => { + prepared_message.content = Some(OpenAIContent::Text(text.clone())); + } + OpenAIContent::Parts(parts) => { + let mut prepared_content = Vec::new(); + + for part in parts { + match part { + ContentPart::S3Object { s3_object } => { + // Convert S3Object to base64 image URL + let (mime_type, image_bytes) = download_and_encode_s3_image( + s3_object, + client, + workspace_id, + ) + .await?; + prepared_content.push(ContentPart::ImageUrl { + image_url: ImageUrlData { + url: format!( + "data:{};base64,{}", + mime_type, image_bytes + ), + }, + }); + } + other => { + // Keep Text and ImageUrl as-is + prepared_content.push(other.clone()); + } + } + } + + prepared_message.content = Some(OpenAIContent::Parts(prepared_content)); + } + } + } + + prepared_messages.push(prepared_message); + } + + Ok(prepared_messages) + } + + async fn build_text_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result { + let prepared_messages = self + .prepare_messages_for_api(args.messages, client, workspace_id) + .await?; + + // Check if we need to add response_format for structured output + let has_output_properties = args + .output_schema + .and_then(|schema| schema.properties.as_ref()) + .map(|props| !props.is_empty()) + .unwrap_or(false); + + let response_format = if has_output_properties && args.output_schema.is_some() { + let schema = args.output_schema.unwrap(); + let strict_schema = schema.clone().make_strict(); + Some(ResponseFormat { + r#type: "json_schema".to_string(), + json_schema: JsonSchemaFormat { + name: "structured_output".to_string(), + schema: strict_schema, + strict: Some(true), + }, + }) + } else { + None + }; + + let request = OpenAIRequest { + model: args.model, + messages: &prepared_messages, + tools: args.tools, + temperature: args.temperature, + max_completion_tokens: args.max_tokens, + response_format, + }; + + serde_json::to_string(&request) + .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) + } + + async fn build_image_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result { + // Build content array with text and optional image + let mut content = + vec![ImageGenerationContent::InputText { text: args.user_message.to_string() }]; + + // Add images if provided + if let Some(images) = args.images { + for image in images.iter() { + if !image.s3.is_empty() { + let (mime_type, image_bytes) = + download_and_encode_s3_image(image, client, workspace_id).await?; + content.push(ImageGenerationContent::InputImage { + image_url: format!("data:{};base64,{}", mime_type, image_bytes), + }); + } + } + } + + // Build the request with tools if provided + let tools = vec![ImageGenerationTool { + r#type: "image_generation".to_string(), + quality: Some("low".to_string()), + background: None, + }]; + + // TODO: OpenAI's image generation API doesn't support custom tools in the same way as chat completions + // This would require a different approach, potentially using chat completions with image output + // For now, we'll use the standard image generation without custom tools + + let image_request = ImageGenerationRequest { + model: args.model, + input: vec![ImageGenerationMessage { role: "user".to_string(), content }], + instructions: args.system_prompt, + tools, + }; + + serde_json::to_string(&image_request) + .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) + } +} + +#[async_trait] +impl QueryBuilder for OpenAIQueryBuilder { + fn supports_tools_with_output_type(&self, _output_type: &OutputType) -> bool { + // OpenAI supports tools for both text and image output + true + } + + async fn build_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result { + match args.output_type { + OutputType::Text => self.build_text_request(args, client, workspace_id).await, + OutputType::Image => self.build_image_request(args, client, workspace_id).await, + } + } + + async fn parse_response(&self, response: reqwest::Response) -> Result { + // Check if this is an image response + let url = response.url().path(); + if url.contains("/responses") { + // Parse image generation response + let response_text = response + .text() + .await + .map_err(|e| Error::internal_err(format!("Failed to read response text: {}", e)))?; + + let image_response: OpenAIImageResponse = serde_json::from_str(&response_text) + .map_err(|e| { + Error::internal_err(format!( + "Failed to parse OpenAI image response: {}. Raw response: {}", + e, response_text + )) + })?; + + // Find the first completed image generation output + let image_generation_call = image_response + .output + .iter() + .find(|output| { + output.r#type == "image_generation_call" && output.status == "completed" + }) + .and_then(|output| output.result.as_ref()); + + if let Some(base64_image) = image_generation_call { + Ok(ParsedResponse::Image { base64_data: base64_image.clone() }) + } else { + Err(Error::internal_err( + "No completed image output received from OpenAI".to_string(), + )) + } + } else { + // Parse text/chat completion response + let openai_response: OpenAIResponse = response + .json() + .await + .map_err(|e| Error::internal_err(format!("Failed to parse response: {}", e)))?; + + let first_choice = openai_response + .choices + .into_iter() + .next() + .ok_or_else(|| Error::internal_err("No response from API"))?; + + Ok(ParsedResponse::Text { + content: first_choice.message.content.map(|c| match c { + OpenAIContent::Text(text) => text, + OpenAIContent::Parts(parts) => { + // Extract text from parts + parts + .into_iter() + .filter_map(|part| match part { + ContentPart::Text { text } => Some(text), + _ => None, + }) + .collect::>() + .join(" ") + } + }), + tool_calls: first_choice.message.tool_calls.unwrap_or_default(), + }) + } + } + + fn get_endpoint(&self, base_url: &str, _model: &str, output_type: &OutputType) -> String { + match output_type { + OutputType::Text => format!("{}/chat/completions", base_url), + OutputType::Image => format!("{}/responses", base_url), + } + } + + fn get_auth_headers( + &self, + api_key: &str, + _output_type: &OutputType, + ) -> Vec<(&'static str, String)> { + vec![("Authorization", format!("Bearer {}", api_key))] + } +} diff --git a/backend/windmill-worker/src/ai/providers/openrouter.rs b/backend/windmill-worker/src/ai/providers/openrouter.rs new file mode 100644 index 0000000000..9fc6c6f051 --- /dev/null +++ b/backend/windmill-worker/src/ai/providers/openrouter.rs @@ -0,0 +1,202 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json; +use windmill_common::{client::AuthedClient, error::Error}; + +use crate::ai::{ + providers::openai::{OpenAIQueryBuilder, OpenAIResponse}, + query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder}, + types::*, +}; + +// OpenRouter-specific types +#[derive(Serialize)] +pub struct OpenRouterChatRequest<'a> { + pub model: &'a str, + pub messages: &'a [OpenAIMessage], + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option<&'a [ToolDef]>, + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_completion_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub response_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub modalities: Option>, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageResponse { + pub choices: Vec, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageChoice { + pub message: OpenRouterImageResponseMessage, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageResponseMessage { + #[serde(skip_serializing_if = "Option::is_none")] + pub images: Option>, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageData { + pub image_url: OpenRouterImageUrl, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageUrl { + pub url: String, // data:image/png;base64,... format +} + +pub struct OpenRouterQueryBuilder { + // OpenRouter uses OpenAI-compatible API, so we delegate most work to OpenAI builder + openai_builder: OpenAIQueryBuilder, +} + +impl OpenRouterQueryBuilder { + pub fn new() -> Self { + Self { openai_builder: OpenAIQueryBuilder::new() } + } +} + +#[async_trait] +impl QueryBuilder for OpenRouterQueryBuilder { + fn supports_tools_with_output_type(&self, _output_type: &OutputType) -> bool { + // OpenRouter supports tools for both text and image output (via OpenAI-compatible API) + true + } + + async fn build_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result { + match args.output_type { + OutputType::Text => { + // For text, use standard OpenAI format without modalities + self.openai_builder + .build_request(args, client, workspace_id) + .await + } + OutputType::Image => { + // For image generation, we need to add modalities field + // First, prepare the messages using the OpenAI builder's logic + let openai_builder = &self.openai_builder; + let prepared_messages = openai_builder + .prepare_messages_for_api(args.messages, client, workspace_id) + .await?; + + // Check if we need to add response_format for structured output + let has_output_properties = args + .output_schema + .and_then(|schema| schema.properties.as_ref()) + .map(|props| !props.is_empty()) + .unwrap_or(false); + + let response_format = if has_output_properties && args.output_schema.is_some() { + let schema = args.output_schema.unwrap(); + let strict_schema = schema.clone().make_strict(); + Some(ResponseFormat { + r#type: "json_schema".to_string(), + json_schema: JsonSchemaFormat { + name: "structured_output".to_string(), + schema: strict_schema, + strict: Some(true), + }, + }) + } else { + None + }; + + // Build OpenRouter-specific request with modalities + let request = OpenRouterChatRequest { + model: args.model, + messages: &prepared_messages, + tools: args.tools, + temperature: args.temperature, + max_completion_tokens: args.max_tokens, + response_format, + modalities: Some(vec!["image", "text"]), + }; + + serde_json::to_string(&request) + .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) + } + } + } + + async fn parse_response(&self, response: reqwest::Response) -> Result { + let response_text = response + .text() + .await + .map_err(|e| Error::internal_err(format!("Failed to read response text: {}", e)))?; + + // First try to parse as OpenRouter image response + if let Ok(image_response) = serde_json::from_str::(&response_text) + { + // Extract base64 image from the first choice + let image_url = image_response + .choices + .get(0) + .and_then(|choice| choice.message.images.as_ref()) + .and_then(|images| images.get(0)) + .map(|image| &image.image_url.url); + + if let Some(data_url) = image_url { + // Extract base64 data from data URL format: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... + if let Some(base64_start) = data_url.find("base64,") { + let base64_data = &data_url[base64_start + 7..]; // Skip "base64," prefix + return Ok(ParsedResponse::Image { base64_data: base64_data.to_string() }); + } + } + } + + // If not an image response or parsing failed, try as regular OpenAI response + let openai_response: OpenAIResponse = + serde_json::from_str(&response_text).map_err(|e| { + Error::internal_err(format!( + "Failed to parse response: {}. Raw response: {}", + e, response_text + )) + })?; + + let first_choice = openai_response + .choices + .into_iter() + .next() + .ok_or_else(|| Error::internal_err("No response from API"))?; + + Ok(ParsedResponse::Text { + content: first_choice.message.content.map(|c| match c { + OpenAIContent::Text(text) => text, + OpenAIContent::Parts(parts) => parts + .into_iter() + .filter_map(|part| match part { + ContentPart::Text { text } => Some(text), + _ => None, + }) + .collect::>() + .join(" "), + }), + tool_calls: first_choice.message.tool_calls.unwrap_or_default(), + }) + } + + fn get_endpoint(&self, base_url: &str, _model: &str, _output_type: &OutputType) -> String { + // OpenRouter uses the same endpoint for both text and image generation + format!("{}/chat/completions", base_url) + } + + fn get_auth_headers( + &self, + api_key: &str, + _output_type: &OutputType, + ) -> Vec<(&'static str, String)> { + vec![("Authorization", format!("Bearer {}", api_key))] + } +} diff --git a/backend/windmill-worker/src/ai/query_builder.rs b/backend/windmill-worker/src/ai/query_builder.rs new file mode 100644 index 0000000000..07b9c96e4e --- /dev/null +++ b/backend/windmill-worker/src/ai/query_builder.rs @@ -0,0 +1,70 @@ +use async_trait::async_trait; +use windmill_common::{client::AuthedClient, error::Error, s3_helpers::S3Object}; + +use crate::ai::{ + providers::{ + google_ai::GoogleAIQueryBuilder, + openai::{OpenAIQueryBuilder, OpenAIToolCall}, + openrouter::OpenRouterQueryBuilder, + }, + types::*, +}; + +/// Arguments for building an AI request +pub struct BuildRequestArgs<'a> { + pub messages: &'a [OpenAIMessage], + pub tools: Option<&'a [ToolDef]>, + pub model: &'a str, + pub temperature: Option, + pub max_tokens: Option, + pub output_schema: Option<&'a OpenAPISchema>, + pub output_type: &'a OutputType, + pub system_prompt: Option<&'a str>, + pub user_message: &'a str, + pub images: Option<&'a [S3Object]>, +} + +/// Response from AI provider +pub enum ParsedResponse { + Text { content: Option, tool_calls: Vec }, + Image { base64_data: String }, +} + +/// Trait for building provider-specific AI requests +#[async_trait] +pub trait QueryBuilder: Send + Sync { + /// Check if this provider supports tools with the given output type + fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool; + + /// Build the request body for the provider + async fn build_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result; + + /// Parse the response from the provider + async fn parse_response(&self, response: reqwest::Response) -> Result; + + /// Get the API endpoint for this provider + fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String; + + /// Get the authentication headers for this provider + fn get_auth_headers( + &self, + api_key: &str, + output_type: &OutputType, + ) -> Vec<(&'static str, String)>; +} + +/// Factory function to create the appropriate query builder for a provider +pub fn create_query_builder(provider: &ProviderWithResource) -> Box { + use windmill_common::ai_providers::AIProvider; + + match provider.kind { + AIProvider::GoogleAI => Box::new(GoogleAIQueryBuilder::new()), + AIProvider::OpenRouter => Box::new(OpenRouterQueryBuilder::new()), + _ => Box::new(OpenAIQueryBuilder::new()), // Use OpenAI as default for all other providers + } +} diff --git a/backend/windmill-worker/src/ai/types.rs b/backend/windmill-worker/src/ai/types.rs new file mode 100644 index 0000000000..cc1417a92d --- /dev/null +++ b/backend/windmill-worker/src/ai/types.rs @@ -0,0 +1,368 @@ +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use std::collections::HashMap; +use windmill_common::{ + ai_providers::AIProvider, db::DB, error::Error, flow_status::AgentAction, flows::FlowModule, + s3_helpers::S3Object, +}; +use windmill_parser::Typ; + +use crate::ai::providers::openai::OpenAIToolCall; + +// Shared types used across multiple providers + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ContentPart { + Text { + text: String, + }, + #[serde(rename = "image_url")] + ImageUrl { + image_url: ImageUrlData, + }, + #[serde(rename = "s3_object")] + S3Object { + s3_object: S3Object, + }, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ImageUrlData { + pub url: String, // data:image/png;base64,... or https://... +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(untagged)] +pub enum OpenAIContent { + Text(String), + Parts(Vec), +} + +#[derive(Deserialize, Serialize, Clone, Default, Debug)] +pub struct OpenAIMessage { + pub role: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_call_id: Option, + #[serde(skip_serializing)] + pub agent_action: Option, +} + +/// same as OpenAIMessage but with agent_action field included in the serialization +#[derive(Serialize)] +pub struct Message<'a> { + #[serde(flatten)] + pub message: &'a OpenAIMessage, + #[serde(skip_serializing_if = "Option::is_none")] + pub agent_action: Option<&'a AgentAction>, +} + +#[derive(Serialize, Clone, Debug)] +pub struct ResponseFormat { + pub r#type: String, + pub json_schema: JsonSchemaFormat, +} + +#[derive(Serialize, Clone, Debug)] +pub struct JsonSchemaFormat { + pub name: String, + pub schema: OpenAPISchema, + #[serde(skip_serializing_if = "Option::is_none")] + pub strict: Option, +} + +#[derive(Serialize, Clone, Debug)] +pub struct ToolDefFunction { + pub name: String, + pub description: Option, + pub parameters: Box, +} + +#[derive(Serialize, Clone, Debug)] +pub struct ToolDef { + pub r#type: String, + pub function: ToolDefFunction, +} + +pub struct Tool { + pub module: FlowModule, + pub def: ToolDef, +} + +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum OutputType { + Text, + Image, +} + +impl Default for OutputType { + fn default() -> Self { + OutputType::Text + } +} + +#[derive(Deserialize, Debug)] +pub struct AIAgentArgs { + pub provider: ProviderWithResource, + pub system_prompt: Option, + pub user_message: String, + pub temperature: Option, + pub max_completion_tokens: Option, + pub output_schema: Option, + pub output_type: Option, + pub user_images: Option>, +} + +#[derive(Deserialize, Debug)] +pub struct ProviderResource { + #[serde(alias = "apiKey")] + pub api_key: String, + #[serde(alias = "baseUrl")] + pub base_url: Option, +} + +#[derive(Deserialize, Debug)] +pub struct ProviderWithResource { + pub kind: AIProvider, + pub resource: ProviderResource, + pub model: String, +} + +impl ProviderWithResource { + pub fn get_api_key(&self) -> &str { + &self.resource.api_key + } + + pub fn get_model(&self) -> &str { + &self.model + } + + pub async fn get_base_url(&self, db: &DB) -> Result { + self.kind + .get_base_url(self.resource.base_url.clone(), db) + .await + } +} + +#[derive(Serialize)] +pub struct AIAgentResult<'a> { + pub output: Box, + pub messages: Vec>, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(untagged)] +pub enum SchemaType { + Single(String), + Multiple(Vec), +} + +impl Default for SchemaType { + fn default() -> Self { + SchemaType::Single("object".to_string()) + } +} + +#[derive(Serialize, Deserialize, Default, Clone, Debug)] +pub struct OpenAPISchema { + #[serde(skip_serializing_if = "Option::is_none")] + pub r#type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub items: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub properties: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub required: Option>, + #[serde(skip_serializing_if = "Option::is_none", rename = "oneOf")] + pub one_of: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub r#enum: Option>, + #[serde( + skip_serializing_if = "Option::is_none", + rename = "additionalProperties" + )] + pub additional_properties: Option, +} + +impl OpenAPISchema { + pub fn from_str(typ: &str) -> Self { + OpenAPISchema { r#type: Some(SchemaType::Single(typ.to_string())), ..Default::default() } + } + + pub fn from_str_with_enum(typ: &str, enu: &Option>) -> Self { + OpenAPISchema { + r#type: Some(SchemaType::Single(typ.to_string())), + r#enum: enu.clone(), + ..Default::default() + } + } + + pub fn datetime() -> Self { + Self { + r#type: Some(SchemaType::Single("string".to_string())), + format: Some("date-time".to_string()), + ..Default::default() + } + } + + pub fn from_typ(typ: &Typ) -> Self { + match typ { + Typ::Str(enu) => Self::from_str_with_enum("string", enu), + Typ::Int => Self::from_str("integer"), + Typ::Float => Self::from_str("number"), + Typ::Bool => Self::from_str("boolean"), + Typ::Bytes => Self::from_str("string"), + Typ::Datetime => Self::datetime(), + Typ::Resource(_) => Self::from_str("string"), + Typ::Email => Self::from_str("string"), + Typ::Sql => Self::from_str("string"), + Typ::DynSelect(_) => Self::from_str("string"), + Typ::DynMultiselect(_) => Self::from_str("string"), + Typ::List(typ) => OpenAPISchema { + r#type: Some(SchemaType::Single("array".to_string())), + items: Some(Box::new(Self::from_typ(typ))), + ..Default::default() + }, + Typ::Object(typ) => OpenAPISchema { + r#type: Some(SchemaType::Single("object".to_string())), + items: None, + properties: typ.props.as_ref().map(|props| { + props + .iter() + .map(|prop| (prop.key.clone(), Box::new(Self::from_typ(&prop.typ)))) + .collect() + }), + required: typ + .props + .as_ref() + .map(|props| props.iter().map(|prop| prop.key.clone()).collect()), + ..Default::default() + }, + Typ::OneOf(variants) => OpenAPISchema { + r#type: Some(SchemaType::Single("object".to_string())), + one_of: Some( + variants + .iter() + .map(|variant| { + let schema = OpenAPISchema { + r#type: Some(SchemaType::Single("object".to_string())), + properties: Some( + variant + .properties + .iter() + .map(|prop| { + ( + prop.key.clone(), + Box::new( + if prop.key == "label" || prop.key == "kind" { + Self::from_str_with_enum( + "string", + &Some(vec![variant.label.clone()]), + ) + } else { + Self::from_typ(&prop.typ) + }, + ), + ) + }) + .collect(), + ), + required: Some( + variant + .properties + .iter() + .map(|prop| prop.key.clone()) + .collect(), + ), + ..Default::default() + }; + Box::new(schema) + }) + .collect(), + ), + ..Default::default() + }, + Typ::Unknown => Self::from_str("object"), + } + } + + /// Makes this schema compatible with OpenAI's strict mode by: + /// - Adding additionalProperties: false to all object types + /// - Making non-required properties nullable + /// - Ensuring all properties are in the required array + pub fn make_strict(mut self) -> Self { + // Handle this schema if it's an object type + if let Some(SchemaType::Single(ref type_str)) = self.r#type { + if type_str == "object" { + // Set additionalProperties to false + self.additional_properties = Some(false); + + if let Some(properties) = self.properties.as_mut() { + // Get original required fields + let original_required = self.required.as_ref(); + + if let Some(required) = original_required { + // Update properties to make non-required fields nullable + for (key, prop) in properties.iter_mut() { + let mut new_prop = (**prop).clone(); + // Make non-required fields nullable + if !required.contains(key) { + new_prop = new_prop.make_nullable(); + } + // Recursively make nested schemas strict + new_prop = new_prop.make_strict(); + *prop = Box::new(new_prop); + } + } + + // All properties must be in required array for strict mode + self.required = Some(properties.keys().cloned().collect()); + } + } + } + + // Recursively process nested schemas + if let Some(ref mut items) = self.items { + **items = items.as_ref().clone().make_strict(); + } + + if let Some(ref mut one_of) = self.one_of { + *one_of = one_of + .iter() + .map(|schema| Box::new(schema.as_ref().clone().make_strict())) + .collect(); + } + + self + } + + /// Makes this property nullable by converting its type to a union with null + pub fn make_nullable(mut self) -> Self { + match self.r#type.take() { + Some(SchemaType::Single(type_str)) => { + if type_str != "null" { + self.r#type = Some(SchemaType::Multiple(vec![type_str, "null".into()])); + } else { + self.r#type = Some(SchemaType::Single("null".into())); + } + } + Some(SchemaType::Multiple(mut types)) => { + if !types.iter().any(|t| t == "null") { + types.push("null".into()); + } + self.r#type = Some(SchemaType::Multiple(types)); + } + None => { + self.r#type = Some(SchemaType::Single("null".into())); + } + } + self + } +} diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index 7f5df68bc8..ddc2cc480f 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -1,35 +1,34 @@ use async_recursion::async_recursion; -use base64::Engine; -use mime_guess; use regex::Regex; -use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::{collections::HashMap, sync::Arc}; -#[cfg(feature = "benchmark")] -use windmill_common::bench::BenchmarkIter; +use ulid; +use uuid::Uuid; use windmill_common::{ ai_providers::AIProvider, - auth::get_job_perms, cache, client::AuthedClient, db::DB, error::{self, to_anyhow, Error}, flow_status::AgentAction, - flows::{FlowModule, FlowModuleValue, Step}, + flows::{FlowModuleValue, Step}, get_latest_hash_for_path, jobs::JobKind, - s3_helpers::S3Object, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, utils::{StripPath, HTTP_CLIENT}, worker::{to_raw_value, Connection}, }; -use windmill_parser::Typ; use windmill_queue::{ flow_status::get_step_of_flow_status, get_mini_pulled_job, push, CanceledBy, JobCompleted, MiniPulledJob, PushArgs, PushIsolationLevel, }; use crate::{ + ai::{ + image_handler::upload_image_to_s3, + query_builder::{create_query_builder, BuildRequestArgs, ParsedResponse}, + types::*, + }, common::{build_args_map, error_to_value, OccupancyMetrics}, create_job_dir, handle_child::run_future_with_polling_update_job_poller, @@ -39,1093 +38,12 @@ use crate::{ JobCompletedSender, SendResult, SendResultPayload, }; -const MAX_AGENT_ITERATIONS: usize = 10; -const REQUEST_TIMEOUT: u64 = 120; - lazy_static::lazy_static! { static ref TOOL_NAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]+$").unwrap(); } -#[derive(Deserialize, Serialize, Clone, Debug)] -struct OpenAIFunction { - name: String, - arguments: String, -} - -#[derive(Deserialize, Serialize, Clone, Debug)] -struct OpenAIToolCall { - id: String, - function: OpenAIFunction, - r#type: String, -} - -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(tag = "type", rename_all = "snake_case")] -enum ContentPart { - Text { - text: String, - }, - #[serde(rename = "image_url")] - ImageUrl { - image_url: ImageUrlData, - }, - #[serde(rename = "s3_object")] - S3Object { - s3_object: S3Object, - }, -} - -#[derive(Serialize, Deserialize, Clone, Debug)] -struct ImageUrlData { - url: String, // data:image/png;base64,... or https://... -} - -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(untagged)] -enum OpenAIContent { - Text(String), - Parts(Vec), -} - -#[derive(Deserialize, Serialize, Clone, Default, Debug)] -struct OpenAIMessage { - role: String, - #[serde(skip_serializing_if = "Option::is_none")] - content: Option, - #[serde(skip_serializing_if = "Option::is_none")] - tool_calls: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - tool_call_id: Option, - #[serde(skip_serializing)] - agent_action: Option, -} - -/// same as OpenAIMessage but with agent_action field included in the serialization -#[derive(Serialize)] -struct Message<'a> { - #[serde(flatten)] - message: &'a OpenAIMessage, - #[serde(skip_serializing_if = "Option::is_none")] - agent_action: Option<&'a AgentAction>, -} - -#[derive(Deserialize)] -struct OpenAIChoice { - message: OpenAIMessage, -} - -#[derive(Deserialize)] -struct OpenAIResponse { - choices: Vec, -} - -#[derive(Serialize)] -struct ImageGenerationTool { - r#type: String, - quality: Option, - background: Option, -} - -// Input content for image generation - supports both text and images -#[derive(Serialize, Clone, Debug)] -#[serde(tag = "type", rename_all = "snake_case")] -enum ImageGenerationContent { - #[serde(rename = "input_text")] - InputText { text: String }, - #[serde(rename = "input_image")] - InputImage { image_url: String }, -} - -#[derive(Serialize)] -struct ImageGenerationMessage { - role: String, - content: Vec, -} - -#[derive(Serialize)] -struct ImageGenerationRequest<'a> { - model: &'a str, - input: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - instructions: Option<&'a str>, - tools: Vec, -} - -#[derive(Deserialize)] -struct OpenAIImageResponse { - output: Vec, -} - -#[derive(Deserialize)] -struct OpenAIImageOutput { - r#type: String, // Expected to be "image_generation_call" - #[serde(default)] - result: Option, // Base64 encoded image -} - -// Gemini API structures -#[derive(Serialize, Deserialize, Clone, Debug)] -struct GeminiInlineData { - #[serde(rename = "mimeType")] - mime_type: String, - data: String, -} - -#[derive(Serialize)] -#[serde(untagged)] -enum GeminiPart { - Text { text: String }, - InlineData { inline_data: GeminiInlineData }, -} - -#[derive(Serialize)] -struct GeminiContent { - parts: Vec, -} - -#[derive(Serialize)] -struct GeminiImageRequest { - #[serde(skip_serializing_if = "Option::is_none")] - contents: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - instances: Option>, -} - -#[derive(Serialize)] -struct GeminiPredictContent { - prompt: String, -} - -#[derive(Deserialize)] -struct GeminiImageResponse { - #[serde(skip_serializing_if = "Option::is_none")] - candidates: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - predictions: Option>, -} - -#[derive(Deserialize)] -struct GeminiCandidate { - content: GeminiResponseContent, -} - -#[derive(Deserialize)] -struct GeminiPredictCandidate { - #[serde(rename = "bytesBase64Encoded")] - bytes_base64_encoded: String, // base64 encoded image -} - -#[derive(Deserialize)] -struct GeminiResponseContent { - parts: Vec, -} - -#[derive(Deserialize)] -struct GeminiResponsePart { - #[serde(rename = "inlineData")] - inline_data: Option, -} - -// OpenRouter image generation structures -#[derive(Serialize)] -struct OpenRouterImageRequest<'a> { - model: &'a str, - messages: Vec, - modalities: Vec<&'a str>, -} - -#[derive(Serialize)] -struct OpenRouterImageMessage { - role: String, - content: String, -} - -#[derive(Deserialize)] -struct OpenRouterImageResponse { - choices: Vec, -} - -#[derive(Deserialize)] -struct OpenRouterImageChoice { - message: OpenRouterImageResponseMessage, -} - -#[derive(Deserialize)] -struct OpenRouterImageResponseMessage { - #[serde(skip_serializing_if = "Option::is_none")] - images: Option>, -} - -#[derive(Deserialize)] -struct OpenRouterImageData { - image_url: OpenRouterImageUrl, -} - -#[derive(Deserialize)] -struct OpenRouterImageUrl { - url: String, // data:image/png;base64,... format -} - -#[derive(Serialize)] -struct OpenAIRequest<'a> { - model: &'a str, - messages: &'a Vec, - #[serde(skip_serializing_if = "Option::is_none")] - tools: Option<&'a Vec>, - #[serde(skip_serializing_if = "Option::is_none")] - temperature: Option, - #[serde(skip_serializing_if = "Option::is_none")] - max_completion_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - response_format: Option, -} - -#[derive(Serialize, Clone, Debug)] -struct ResponseFormat { - r#type: String, - json_schema: JsonSchemaFormat, -} - -#[derive(Serialize, Clone, Debug)] -struct JsonSchemaFormat { - name: String, - schema: OpenAPISchema, - #[serde(skip_serializing_if = "Option::is_none")] - strict: Option, -} - -#[derive(Serialize, Clone, Debug)] -struct ToolDefFunction { - name: String, - description: Option, - parameters: Box, -} - -#[derive(Serialize, Clone, Debug)] -struct ToolDef { - r#type: String, - function: ToolDefFunction, -} - -struct Tool { - module: FlowModule, - def: ToolDef, -} - -#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] -#[serde(rename_all = "lowercase")] -enum OutputType { - Text, - Image, -} - -impl Default for OutputType { - fn default() -> Self { - OutputType::Text - } -} - -#[derive(Deserialize, Debug)] -struct AIAgentArgs { - provider: ProviderWithResource, - system_prompt: Option, - user_message: String, - temperature: Option, - max_completion_tokens: Option, - output_schema: Option, - output_type: Option, - image: Option, -} - -#[derive(Deserialize, Debug)] -struct ProviderResource { - #[serde(alias = "apiKey")] - api_key: String, - #[serde(alias = "baseUrl")] - base_url: Option, -} - -#[derive(Deserialize, Debug)] -struct ProviderWithResource { - kind: AIProvider, - resource: ProviderResource, - model: String, -} - -impl ProviderWithResource { - fn get_api_key(&self) -> &str { - &self.resource.api_key - } - - fn get_model(&self) -> &str { - &self.model - } - - async fn get_base_url(&self, db: &DB) -> Result { - self.kind - .get_base_url(self.resource.base_url.clone(), db) - .await - } -} - -#[derive(Serialize)] -struct AIAgentResult<'a> { - output: Box, - messages: Vec>, -} - -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(untagged)] -enum SchemaType { - Single(String), - Multiple(Vec), -} - -impl Default for SchemaType { - fn default() -> Self { - SchemaType::Single("object".to_string()) - } -} - -#[derive(Serialize, Deserialize, Default, Clone, Debug)] -struct OpenAPISchema { - #[serde(skip_serializing_if = "Option::is_none")] - r#type: Option, - #[serde(skip_serializing_if = "Option::is_none")] - items: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - properties: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - required: Option>, - #[serde(skip_serializing_if = "Option::is_none", rename = "oneOf")] - one_of: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - format: Option, - #[serde(skip_serializing_if = "Option::is_none")] - r#enum: Option>, - #[serde( - skip_serializing_if = "Option::is_none", - rename = "additionalProperties" - )] - additional_properties: Option, -} - -impl OpenAPISchema { - fn from_str(typ: &str) -> Self { - OpenAPISchema { r#type: Some(SchemaType::Single(typ.to_string())), ..Default::default() } - } - - fn from_str_with_enum(typ: &str, enu: &Option>) -> Self { - OpenAPISchema { - r#type: Some(SchemaType::Single(typ.to_string())), - r#enum: enu.clone(), - ..Default::default() - } - } - - fn datetime() -> Self { - Self { - r#type: Some(SchemaType::Single("string".to_string())), - format: Some("date-time".to_string()), - ..Default::default() - } - } - - fn from_typ(typ: &Typ) -> Self { - match typ { - Typ::Str(enu) => Self::from_str_with_enum("string", enu), - Typ::Int => Self::from_str("integer"), - Typ::Float => Self::from_str("number"), - Typ::Bool => Self::from_str("boolean"), - Typ::Bytes => Self::from_str("string"), - Typ::Datetime => Self::datetime(), - Typ::Resource(_) => Self::from_str("string"), - Typ::Email => Self::from_str("string"), - Typ::Sql => Self::from_str("string"), - Typ::DynSelect(_) => Self::from_str("string"), - Typ::DynMultiselect(_) => Self::from_str("string"), - Typ::List(typ) => OpenAPISchema { - r#type: Some(SchemaType::Single("array".to_string())), - items: Some(Box::new(Self::from_typ(typ))), - ..Default::default() - }, - Typ::Object(typ) => OpenAPISchema { - r#type: Some(SchemaType::Single("object".to_string())), - items: None, - properties: typ.props.as_ref().map(|props| { - props - .iter() - .map(|prop| (prop.key.clone(), Box::new(Self::from_typ(&prop.typ)))) - .collect() - }), - required: typ - .props - .as_ref() - .map(|props| props.iter().map(|prop| prop.key.clone()).collect()), - ..Default::default() - }, - Typ::OneOf(variants) => OpenAPISchema { - r#type: Some(SchemaType::Single("object".to_string())), - one_of: Some( - variants - .iter() - .map(|variant| { - let schema = OpenAPISchema { - r#type: Some(SchemaType::Single("object".to_string())), - properties: Some( - variant - .properties - .iter() - .map(|prop| { - ( - prop.key.clone(), - Box::new( - if prop.key == "label" || prop.key == "kind" { - Self::from_str_with_enum( - "string", - &Some(vec![variant.label.clone()]), - ) - } else { - Self::from_typ(&prop.typ) - }, - ), - ) - }) - .collect(), - ), - required: Some( - variant - .properties - .iter() - .map(|prop| prop.key.clone()) - .collect(), - ), - ..Default::default() - }; - Box::new(schema) - }) - .collect(), - ), - ..Default::default() - }, - Typ::Unknown => Self::from_str("object"), - } - } - - /// Makes this schema compatible with OpenAI's strict mode by: - /// - Adding additionalProperties: false to all object types - /// - Making non-required properties nullable - /// - Ensuring all properties are in the required array - fn make_strict(mut self) -> Self { - // Handle this schema if it's an object type - if let Some(SchemaType::Single(ref type_str)) = self.r#type { - if type_str == "object" { - // Set additionalProperties to false - self.additional_properties = Some(false); - - if let Some(properties) = self.properties.as_mut() { - // Get original required fields - let original_required = self.required.as_ref(); - - if let Some(required) = original_required { - // Update properties to make non-required fields nullable - for (key, prop) in properties.iter_mut() { - let mut new_prop = (**prop).clone(); - // Make non-required fields nullable - if !required.contains(key) { - new_prop = new_prop.make_nullable(); - } - // Recursively make nested schemas strict - new_prop = new_prop.make_strict(); - *prop = Box::new(new_prop); - } - } - - // All properties must be in required array for strict mode - self.required = Some(properties.keys().cloned().collect()); - } - } - } - - // Recursively process nested schemas - if let Some(ref mut items) = self.items { - **items = items.as_ref().clone().make_strict(); - } - - if let Some(ref mut one_of) = self.one_of { - *one_of = one_of - .iter() - .map(|schema| Box::new(schema.as_ref().clone().make_strict())) - .collect(); - } - - self - } - - /// Makes this property nullable by converting its type to a union with null - fn make_nullable(mut self) -> Self { - match self.r#type.take() { - Some(SchemaType::Single(type_str)) => { - if type_str != "null" { - self.r#type = Some(SchemaType::Multiple(vec![type_str, "null".into()])); - } else { - self.r#type = Some(SchemaType::Single("null".into())); - } - } - Some(SchemaType::Multiple(mut types)) => { - if !types.iter().any(|t| t == "null") { - types.push("null".into()); - } - self.r#type = Some(SchemaType::Multiple(types)); - } - None => { - self.r#type = Some(SchemaType::Single("null".into())); - } - } - self - } -} - -/// Find a unique tool name to avoid collisions with user-provided tools -fn find_unique_tool_name(base_name: &str, existing_tools: Option<&[ToolDef]>) -> String { - let Some(tools) = existing_tools else { - return base_name.to_string(); - }; - - if !tools.iter().any(|t| t.function.name == base_name) { - return base_name.to_string(); - } - - for i in 1..100 { - let candidate = format!("{}_{}", base_name, i); - if !tools.iter().any(|t| t.function.name == candidate) { - return candidate; - } - } - - // Fallback with process id if somehow we can't find a unique name - format!("{}_{}_fallback", base_name, std::process::id()) -} - -/// Helper function to download an S3 image and convert it to a base64 data URL -async fn download_and_encode_s3_image( - image: &S3Object, - client: &AuthedClient, - workspace_id: &str, -) -> error::Result<(String, String)> { - // Download the image from S3 - let image_bytes = client - .download_s3_file(workspace_id, &image.s3, image.storage.clone()) - .await - .map_err(|e| Error::internal_err(format!("Failed to download S3 image: {}", e)))?; - - // Encode as base64 data URL - let base64_data = base64::engine::general_purpose::STANDARD.encode(&image_bytes); - - // Determine MIME type using mime_guess from file extension, with PNG as fallback - let mime_type = mime_guess::from_path(&image.s3).first(); - let mime_type = mime_type - .as_ref() - .map(|mime| mime.essence_str()) - .unwrap_or("image/png"); - - Ok((mime_type.to_string(), base64_data)) -} - -/// Convert messages with S3Objects to messages with base64 image URLs for API calls -async fn prepare_messages_for_api( - messages: &[OpenAIMessage], - client: &AuthedClient, - workspace_id: &str, -) -> error::Result> { - let mut prepared_messages = Vec::new(); - - for message in messages { - let mut prepared_message = message.clone(); - - if let Some(content) = &message.content { - match content { - OpenAIContent::Text(text) => { - prepared_message.content = Some(OpenAIContent::Text(text.clone())); - } - OpenAIContent::Parts(parts) => { - let mut prepared_content = Vec::new(); - - for part in parts { - match part { - ContentPart::S3Object { s3_object } => { - // Convert S3Object to base64 image URL - let (mime_type, image_data_url) = - download_and_encode_s3_image(s3_object, client, workspace_id) - .await?; - prepared_content.push(ContentPart::ImageUrl { - image_url: ImageUrlData { - url: format!( - "data:{};base64,{}", - mime_type, image_data_url - ), - }, - }); - } - other => { - // Keep Text and ImageUrl as-is - prepared_content.push(other.clone()); - } - } - } - - prepared_message.content = Some(OpenAIContent::Parts(prepared_content)); - } - } - } - - prepared_messages.push(prepared_message); - } - - Ok(prepared_messages) -} - -/// Generate image from provider and extract base64 data -async fn generate_image_from_provider( - provider: &ProviderWithResource, - user_message: &str, - system_prompt: Option<&str>, - base_url: &str, - api_key: &str, - image: Option<&S3Object>, - client: &AuthedClient, - workspace_id: &str, -) -> error::Result { - match provider.kind { - AIProvider::OpenAI => { - // Build content array with text and optional image - let mut content = - vec![ImageGenerationContent::InputText { text: user_message.to_string() }]; - - // Add image if provided - if let Some(image) = image { - if !image.s3.is_empty() { - // Download and encode S3 image to base64 - let (mime_type, bytes64) = - download_and_encode_s3_image(image, client, workspace_id).await?; - content.push(ImageGenerationContent::InputImage { - image_url: format!("data:{};base64,{}", mime_type, bytes64), - }); - } - } - - let image_request = ImageGenerationRequest { - model: provider.get_model(), - input: vec![ImageGenerationMessage { role: "user".to_string(), content }], - instructions: system_prompt, - tools: vec![ImageGenerationTool { - r#type: "image_generation".to_string(), - quality: Some("low".to_string()), - background: None, - }], - }; - - let resp = HTTP_CLIENT - .post(format!("{}/responses", base_url)) - .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT)) - .bearer_auth(api_key) - .json(&image_request) - .send() - .await - .map_err(|e| Error::internal_err(format!("Failed to call OpenAI API: {}", e)))?; - - match resp.error_for_status_ref() { - Ok(_) => { - let image_response = resp.json::().await.map_err(|e| { - Error::internal_err(format!("Failed to parse OpenAI response: {}", e)) - })?; - - // Find the first image generation output - let image_generation_call = image_response - .output - .iter() - .find(|output| output.r#type == "image_generation_call") - .and_then(|output| output.result.as_ref()); - - if let Some(base64_image) = image_generation_call { - Ok(base64_image.to_string()) - } else { - Err(Error::internal_err( - "No image output received from OpenAI".to_string(), - )) - } - } - Err(e) => { - let _status = resp.status(); - let text = resp - .text() - .await - .unwrap_or_else(|_| "".to_string()); - Err(Error::internal_err(format!( - "OpenAI API error: {} - {}", - e, text - ))) - } - } - } - AIProvider::GoogleAI => { - let is_imagen = provider.get_model().contains("imagen"); - - let gemini_request = if is_imagen { - // For Imagen models, we keep the simple prompt format (no image support) - GeminiImageRequest { - instances: Some(vec![GeminiPredictContent { - prompt: user_message.trim().to_string(), - }]), - contents: None, - } - } else { - // For Gemini models, build parts array with text and optional image - let mut parts = vec![GeminiPart::Text { text: user_message.trim().to_string() }]; - - if let Some(system_prompt) = system_prompt { - parts.insert( - 0, - GeminiPart::Text { - text: format!("SYSTEM PROMPT: {}", system_prompt.trim().to_string()), - }, - ); - } - - // Add image if provided - if let Some(image) = image { - if !image.s3.is_empty() { - // Download and encode S3 image to base64 - let (mime_type, bytes64) = - download_and_encode_s3_image(image, client, workspace_id).await?; - - parts.push(GeminiPart::InlineData { - inline_data: GeminiInlineData { mime_type, data: bytes64 }, - }); - } - } - - GeminiImageRequest { - instances: None, - contents: Some(vec![GeminiContent { parts }]), - } - }; - - let url_suffix = if is_imagen { - "predict" - } else { - "generateContent" - }; - let gemini_url = format!( - "https://generativelanguage.googleapis.com/v1beta/models/{}:{}", - provider.get_model(), - url_suffix - ); - - let resp = HTTP_CLIENT - .post(&gemini_url) - .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT)) - .header("x-goog-api-key", api_key) - .header("Content-Type", "application/json") - .json(&gemini_request) - .send() - .await - .map_err(|e| Error::internal_err(format!("Failed to call Gemini API: {}", e)))?; - - match resp.error_for_status_ref() { - Ok(_) => { - let response_text = resp.text().await.map_err(|e| { - Error::internal_err(format!("Failed to read response text: {}", e)) - })?; - - let gemini_response: GeminiImageResponse = serde_json::from_str(&response_text) - .map_err(|e| { - Error::internal_err(format!( - "Failed to parse Gemini response: {}. Raw response: {}", - e, response_text - )) - })?; - - // Find the first candidate with inline image data - let mut image_data = - gemini_response.candidates.as_ref().and_then(|candidates| { - candidates.iter().find_map(|candidate| { - candidate.content.parts.iter().find_map(|part| { - part.inline_data.as_ref().map(|data| &data.data) - }) - }) - }); - - if image_data.is_none() { - image_data = gemini_response - .predictions - .as_ref() - .and_then(|predictions| { - predictions - .iter() - .find_map(|prediction| Some(&prediction.bytes_base64_encoded)) - }); - } - - if let Some(base64_image) = image_data { - Ok(base64_image.clone()) - } else { - Err(Error::internal_err( - "No image data received from Gemini".to_string(), - )) - } - } - Err(e) => { - let _status = resp.status(); - let text = resp - .text() - .await - .unwrap_or_else(|_| "".to_string()); - Err(Error::internal_err(format!( - "Gemini API error: {} - {}", - e, text - ))) - } - } - } - AIProvider::OpenRouter => { - let mut messages = Vec::new(); - - // Add system message if provided - if let Some(system_prompt) = system_prompt { - messages.push(OpenRouterImageMessage { - role: "system".to_string(), - content: system_prompt.to_string(), - }); - } - - // Add user message - messages.push(OpenRouterImageMessage { - role: "user".to_string(), - content: user_message.to_string(), - }); - - let openrouter_request = OpenRouterImageRequest { - model: provider.get_model(), - messages, - modalities: vec!["image", "text"], - }; - - let resp = HTTP_CLIENT - .post(format!("{}/chat/completions", base_url)) - .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT)) - .bearer_auth(api_key) - .json(&openrouter_request) - .send() - .await - .map_err(|e| { - Error::internal_err(format!("Failed to call OpenRouter API: {}", e)) - })?; - - match resp.error_for_status_ref() { - Ok(_) => { - let openrouter_response = - resp.json::().await.map_err(|e| { - Error::internal_err(format!( - "Failed to parse OpenRouter response: {}", - e - )) - })?; - - // Extract base64 image from the first choice - let image_url = openrouter_response - .choices - .get(0) - .and_then(|choice| choice.message.images.as_ref()) - .and_then(|images| images.get(0)) - .map(|image| &image.image_url.url); - - if let Some(data_url) = image_url { - // Extract base64 data from data URL format: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... - if let Some(base64_start) = data_url.find("base64,") { - let base64_data = &data_url[base64_start + 7..]; // Skip "base64," prefix - Ok(base64_data.to_string()) - } else { - Err(Error::internal_err( - "Invalid data URL format received from OpenRouter".to_string(), - )) - } - } else { - Err(Error::internal_err( - "No image data received from OpenRouter".to_string(), - )) - } - } - Err(e) => { - let _status = resp.status(); - let text = resp - .text() - .await - .unwrap_or_else(|_| "".to_string()); - Err(Error::internal_err(format!( - "OpenRouter API error: {} - {}", - e, text - ))) - } - } - } - _ => Err(Error::BadRequest(format!( - "Image generation is not supported for provider: {:?}", - provider.kind - ))), - } -} - -/// Upload image to S3 and return S3Object -async fn upload_image_to_s3( - base64_image: &str, - job: &MiniPulledJob, - client: &AuthedClient, -) -> error::Result { - let image_bytes = base64::engine::general_purpose::STANDARD - .decode(base64_image) - .map_err(|e| Error::internal_err(format!("Failed to decode base64 image: {}", e)))?; - - // Generate unique S3 key - let unique_id = ulid::Ulid::new().to_string(); - let s3_key = format!("ai_images/{}/{}.png", job.id, unique_id); - - // Create byte stream - let byte_stream = futures::stream::once(async move { - Ok::<_, std::convert::Infallible>(bytes::Bytes::from(image_bytes)) - }); - - // Upload to S3 - client - .upload_s3_file( - &job.workspace_id, - s3_key.clone(), - None, // storage - use default - byte_stream, - ) - .await - .map_err(|e| Error::internal_err(format!("Failed to upload image to S3: {}", e)))?; - - Ok(S3Object { - s3: s3_key, - storage: None, - filename: Some("generated_image.png".to_string()), - presigned: None, - }) -} - -/// Handle image output generation and return S3 object and messages -async fn handle_image_output( - args: &AIAgentArgs, - job: &MiniPulledJob, - client: &AuthedClient, - db: &DB, -) -> error::Result<(Option, Vec)> { - let base_url = args.provider.get_base_url(db).await?; - let api_key = args.provider.get_api_key(); - - let mut messages = - if let Some(system_prompt) = args.system_prompt.clone().filter(|s| !s.is_empty()) { - vec![OpenAIMessage { - role: "system".to_string(), - content: Some(OpenAIContent::Text(system_prompt)), - ..Default::default() - }] - } else { - vec![] - }; - - // Generate image from provider - let base64_image = generate_image_from_provider( - &args.provider, - &args.user_message, - args.system_prompt.as_deref(), - &base_url, - api_key, - args.image.as_ref(), - client, - &job.workspace_id, - ) - .await?; - - // Add assistant success message - messages.push(OpenAIMessage { - role: "assistant".to_string(), - content: Some(OpenAIContent::Text( - "Image created successfully".to_string(), - )), - ..Default::default() - }); - - // Upload to S3 - let s3_object = upload_image_to_s3(&base64_image, job, client).await?; - - Ok((Some(s3_object), messages)) -} - -async fn update_flow_status_module_with_actions( - db: &DB, - parent_job: &uuid::Uuid, - actions: &[AgentAction], -) -> Result<(), Error> { - let step = get_step_of_flow_status(db, parent_job.to_owned()).await?; - match step { - Step::Step(step) => { - sqlx::query!( - r#" - UPDATE v2_job_status SET - flow_status = jsonb_set( - flow_status, - array['modules', $3::TEXT, 'agent_actions'], - $2 - ) - WHERE id = $1 - "#, - parent_job, - sqlx::types::Json(actions) as _, - step as i32 - ) - .execute(db) - .await?; - } - _ => {} - } - Ok(()) -} - -async fn update_flow_status_module_with_actions_success( - db: &DB, - parent_job: &uuid::Uuid, - action_success: bool, -) -> Result<(), Error> { - let step = get_step_of_flow_status(db, parent_job.to_owned()).await?; - match step { - Step::Step(step) => { - // Append the new bool to the existing array, or create a new array if it doesn't exist - sqlx::query!( - r#" - UPDATE v2_job_status SET - flow_status = jsonb_set( - flow_status, - array['modules', $2::TEXT, 'agent_actions_success'], - COALESCE( - flow_status->'modules'->$2->'agent_actions_success', - to_jsonb(ARRAY[]::bool[]) - ) || to_jsonb(ARRAY[$3::bool]) - ) - WHERE id = $1 - "#, - parent_job, - step as i32, - action_success - ) - .execute(db) - .await?; - } - _ => {} - } - Ok(()) -} +const MAX_AGENT_ITERATIONS: usize = 10; +const REQUEST_TIMEOUT_SECONDS: u64 = 120; fn parse_raw_script_schema(content: &str, language: &ScriptLang) -> Result, Error> { let main_arg_signature = parse_sig_of_lang(content, Some(&language), None)?.unwrap(); // safe to unwrap as langauge is some @@ -1156,546 +74,6 @@ fn parse_raw_script_schema(content: &str, language: &ScriptLang) -> Result, -) -> error::Result> { - let output_type = args.output_type.as_ref().unwrap_or(&OutputType::Text); - - match *output_type { - OutputType::Image => { - let (s3_result, messages) = handle_image_output(&args, job, client, db).await?; - - let final_messages: Vec = messages - .iter() - .map(|m| Message { message: m, agent_action: m.agent_action.as_ref() }) - .collect(); - - if let Some(s3_output) = s3_result { - Ok(to_raw_value(&s3_output)) - } else { - Ok(to_raw_value(&AIAgentResult { - output: to_raw_value(&None::), - messages: final_messages, - })) - } - } - OutputType::Text => { - let base_url = args.provider.get_base_url(db).await?; - let api_key = args.provider.get_api_key(); - - let mut messages = - if let Some(system_prompt) = args.system_prompt.clone().filter(|s| !s.is_empty()) { - vec![OpenAIMessage { - role: "system".to_string(), - content: Some(OpenAIContent::Text(system_prompt)), - ..Default::default() - }] - } else { - vec![] - }; - - // Create user message with optional image - let user_content = if let Some(image) = &args.image { - if !image.s3.is_empty() { - OpenAIContent::Parts(vec![ - ContentPart::Text { text: args.user_message.clone() }, - ContentPart::S3Object { s3_object: image.clone() }, - ]) - } else { - OpenAIContent::Text(args.user_message.clone()) - } - } else { - OpenAIContent::Text(args.user_message.clone()) - }; - - messages.push(OpenAIMessage { - role: "user".to_string(), - content: Some(user_content), - ..Default::default() - }); - - let mut actions = vec![]; - let mut content = None; - - let mut tool_defs: Option> = if tools.is_empty() { - None - } else { - Some(tools.iter().map(|t| t.def.clone()).collect()) - }; - - let has_output_properties = args - .output_schema - .as_ref() - .and_then(|schema| schema.properties.as_ref()) - .map(|props| !props.is_empty()) - .unwrap_or(false); - let provider_is_anthropic = args.provider.kind.is_anthropic(); - let is_openrouter_anthropic = args.provider.kind == AIProvider::OpenRouter - && args.provider.model.starts_with("anthropic/"); - let is_anthropic = provider_is_anthropic || is_openrouter_anthropic; - let mut response_format: Option = None; - let mut used_structured_output_tool = false; - let mut structured_output_tool_name: Option = None; - - if has_output_properties { - let schema = args.output_schema.as_ref().unwrap(); // we know it's some because of the check above - if is_anthropic { - // if output schema is provided, and provider is anthropic, add a structured_output tool in the list of tools - let unique_tool_name = - find_unique_tool_name("structured_output", tool_defs.as_deref()); - structured_output_tool_name = Some(unique_tool_name.clone()); - - let output_tool = ToolDef { - r#type: "function".to_string(), - function: ToolDefFunction { - name: unique_tool_name, - description: Some( - "This tool MUST be used last to return a structured JSON object as the final output." - .to_string(), - ), - parameters: to_raw_value(&schema), - }, - }; - if let Some(ref mut existing_tools) = tool_defs { - existing_tools.push(output_tool); - } else { - tool_defs = Some(vec![output_tool]); - } - } else { - // if output schema is provided, and provider is openai, add a response_format with json_schema - let strict_schema = schema.clone().make_strict(); - response_format = Some(ResponseFormat { - r#type: "json_schema".to_string(), - json_schema: JsonSchemaFormat { - name: "structured_output".to_string(), - schema: strict_schema, - strict: Some(true), - }, - }); - } - } - - for i in 0..MAX_AGENT_ITERATIONS { - if used_structured_output_tool { - break; - } - - let response = { - // Convert messages with S3Objects to base64 image URLs for API request - let prepared_messages = - prepare_messages_for_api(&messages, client, &job.workspace_id).await?; - - let resp = HTTP_CLIENT - .post(format!("{}/chat/completions", base_url)) - .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT)) - .bearer_auth(api_key) - .json(&OpenAIRequest { - model: args.provider.get_model(), - messages: &prepared_messages, - tools: tool_defs.as_ref(), - temperature: args.temperature, - max_completion_tokens: args.max_completion_tokens, - response_format: if has_output_properties && !is_anthropic { - response_format.clone() - } else { - None - }, - }) - .send() - .await - .map_err(|e| Error::internal_err(format!("Failed to call API: {}", e)))?; - - match resp.error_for_status_ref() { - Ok(_) => resp, - Err(e) => { - let status = resp.status(); - let text = resp - .text() - .await - .unwrap_or_else(|_| "".to_string()); - tracing::error!( - "Non 200 response from API: status: {}, body: {}", - status, - text - ); - return Err(Error::internal_err(format!( - "Non 200 response from API: {} - {}", - e, text - ))); - } - } - }; - - let mut response = response.json::().await.map_err(|e| { - Error::internal_err(format!("Failed to parse API response: {}", e)) - })?; - - let first_choice = response - .choices - .pop() - .ok_or_else(|| Error::internal_err("No response from API"))?; - - content = first_choice.message.content; - let tool_calls = first_choice.message.tool_calls.unwrap_or_default(); - - if let Some(ref response_content) = content { - actions.push(AgentAction::Message {}); - messages.push(OpenAIMessage { - role: "assistant".to_string(), - content: Some(response_content.clone()), - agent_action: Some(AgentAction::Message {}), - ..Default::default() - }); - - update_flow_status_module_with_actions(db, parent_job, &actions).await?; - update_flow_status_module_with_actions_success(db, parent_job, true).await?; - } - - if tool_calls.is_empty() { - break; - } else if i == MAX_AGENT_ITERATIONS - 1 { - return Err(Error::internal_err( - "AI agent reached max iterations, but there are still tool calls" - .to_string(), - )); - } - - messages.push(OpenAIMessage { - role: "assistant".to_string(), - tool_calls: Some(tool_calls.clone()), - ..Default::default() - }); - - for tool_call in tool_calls.iter() { - // Structured output tool is used, we stop here as this will be the final output - if structured_output_tool_name - .as_ref() - .map_or(false, |name| tool_call.function.name == *name) - { - used_structured_output_tool = true; - messages.push(OpenAIMessage { - role: "tool".to_string(), - content: Some(OpenAIContent::Text( - "Successfully ran structured_output tool".to_string(), - )), - tool_call_id: Some(tool_call.id.clone()), - ..Default::default() - }); - messages.push(OpenAIMessage { - role: "assistant".to_string(), - content: Some(OpenAIContent::Text( - tool_call.function.arguments.clone(), - )), - agent_action: Some(AgentAction::Message {}), - ..Default::default() - }); - content = Some(OpenAIContent::Text(tool_call.function.arguments.clone())); - break; - } - - let tool = tools - .iter() - .find(|t| t.def.function.name == tool_call.function.name); - if let Some(tool) = tool { - let job_id = ulid::Ulid::new().into(); - actions.push(AgentAction::ToolCall { - job_id, - function_name: tool_call.function.name.clone(), - module_id: tool.module.id.clone(), - }); - - update_flow_status_module_with_actions(db, parent_job, &actions).await?; - - let tool_call_args = serde_json::from_str::>>( - &tool_call.function.arguments, - )?; - - let job_payload = match tool.module.get_value()? { - FlowModuleValue::Script { - path: script_path, - hash: script_hash, - tag_override, - .. - } => { - let payload = script_to_payload( - script_hash, - script_path, - db, - job, - &tool.module, - tag_override, - tool.module.apply_preprocessor, - ) - .await?; - payload - } - FlowModuleValue::RawScript { - path, - content, - language, - lock, - tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - .. - } => { - let path = path.unwrap_or_else(|| { - format!("{}/tools/{}", job.runnable_path(), tool.module.id) - }); - - let payload = raw_script_to_payload( - path, - content, - language, - lock, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - &tool.module, - tag, - tool.module.delete_after_use.unwrap_or(false), - ); - payload - } - _ => { - return Err(Error::internal_err(format!( - "Unsupported tool: {}", - tool_call.function.name - ))); - } - }; - - let mut tx = db.begin().await?; - - let job_perms = get_job_perms(&mut *tx, &job.id, &job.workspace_id) - .await? - .map(|x| x.into()); - - let (email, permissioned_as) = - if let Some(on_behalf_of) = job_payload.on_behalf_of.as_ref() { - (&on_behalf_of.email, on_behalf_of.permissioned_as.clone()) - } else { - (&job.permissioned_as_email, job.permissioned_as.to_owned()) - }; - - let job_priority = tool.module.priority.or(job.priority); - - let tx = PushIsolationLevel::Transaction(tx); - let (uuid, tx) = push( - db, - tx, - &job.workspace_id, - job_payload.payload, - PushArgs { args: &tool_call_args, extra: None }, - &job.created_by, - email, - permissioned_as, - Some(&format!("job-span-{}", job.id)), - None, - job.schedule_path(), - Some(job.id), - None, - None, - Some(job_id), - false, - false, - None, - job.visible_to_owner, - Some(job.tag.clone()), // we reuse the same tag as the agent job because it's run on the same worker - job_payload.timeout, - None, - job_priority, - job_perms.as_ref(), - true, - ) - .await?; - - tx.commit().await?; - - let tool_job = get_mini_pulled_job(db, &uuid).await?; - - let Some(tool_job) = tool_job else { - return Err(Error::internal_err("Tool job not found".to_string())); - }; - - let tool_job = Arc::new(tool_job); - - let job_dir = create_job_dir(&worker_dir, job.id).await; - - let (inner_job_completed_tx, inner_job_completed_rx) = - JobCompletedSender::new(&conn, 1); - - let inner_job_completed_rx = inner_job_completed_rx.expect( - "inner_job_completed_tx should be set as agent jobs are not supported on agent workers", - ); - - #[cfg(feature = "benchmark")] - let mut bench = BenchmarkIter::new(); - - match handle_queued_job( - tool_job.clone(), - None, - None, - None, - None, - conn, - client, - hostname, - worker_name, - worker_dir, - &job_dir, - None, - base_internal_url, - inner_job_completed_tx, - occupancy_metrics, - killpill_rx, - None, - #[cfg(feature = "benchmark")] - &mut bench, - ) - .await - { - Err(err) => { - let err_string = format!("{}: {}", err.name(), err.to_string()); - let err_json = error_to_value(&err); - let _ = handle_non_flow_job_error( - db, - &tool_job, - 0, - None, - err_string.clone(), - err_json, - worker_name, - ) - .await; - messages.push(OpenAIMessage { - role: "tool".to_string(), - content: Some(OpenAIContent::Text(format!( - "Error running tool: {}", - err_string - ))), - tool_call_id: Some(tool_call.id.clone()), - agent_action: Some(AgentAction::ToolCall { - job_id, - function_name: tool_call.function.name.clone(), - module_id: tool.module.id.clone(), - }), - ..Default::default() - }); - update_flow_status_module_with_actions_success( - db, parent_job, false, - ) - .await?; - } - Ok(success) => { - let send_result = inner_job_completed_rx.bounded_rx.try_recv().ok(); - - let result = if let Some(SendResult { - result: - SendResultPayload::JobCompleted(JobCompleted { result, .. }), - .. - }) = send_result.as_ref() - { - job_completed_tx - .send(send_result.as_ref().unwrap().result.clone(), true) - .await - .map_err(to_anyhow)?; - result - } else { - if let Some(send_result) = send_result { - job_completed_tx - .send(send_result.result, true) - .await - .map_err(to_anyhow)?; - } - return Err(Error::internal_err( - "Tool job completed but no result".to_string(), - )); - }; - - messages.push(OpenAIMessage { - role: "tool".to_string(), - content: Some(OpenAIContent::Text(result.get().to_string())), - tool_call_id: Some(tool_call.id.clone()), - agent_action: Some(AgentAction::ToolCall { - job_id, - function_name: tool_call.function.name.clone(), - module_id: tool.module.id.clone(), - }), - ..Default::default() - }); - update_flow_status_module_with_actions_success( - db, parent_job, success, - ) - .await?; - } - } - } else { - return Err(Error::internal_err(format!( - "Tool not found: {}", - tool_call.function.name - ))); - } - } - } - - let final_messages: Vec = messages - .iter() - .map(|m| Message { message: m, agent_action: m.agent_action.as_ref() }) - .collect(); - - // Parse content as JSON, fallback to string if it fails - let output_value = match content { - Some(content_str) => match has_output_properties { - true => match content_str { - OpenAIContent::Text(text) => serde_json::from_str::>(&text) - .map_err(|_e| { - Error::internal_err(format!( - "Failed to parse structured output: {}", - text - )) - }), - // No need to handle this, it will always be a text string - OpenAIContent::Parts(_parts) => Err(Error::internal_err( - "Failed to parse structured output".to_string(), - )), - }, - false => Ok(match content_str { - OpenAIContent::Text(text) => to_raw_value(&text), - OpenAIContent::Parts(parts) => to_raw_value(&parts), - }), - }?, - None => to_raw_value(&""), - }; - - Ok(to_raw_value(&AIAgentResult { - output: output_value, - messages: final_messages, - })) - } - } -} - pub struct FlowJobRunnableIdAndRawFlow { pub runnable_id: Option, pub raw_flow: Option>>, @@ -1918,3 +296,637 @@ pub async fn handle_ai_agent_job( Ok(result) } + +/// Find a unique tool name to avoid collisions with user-provided tools +fn find_unique_tool_name(base_name: &str, existing_tools: Option<&[ToolDef]>) -> String { + let Some(tools) = existing_tools else { + return base_name.to_string(); + }; + + if !tools.iter().any(|t| t.function.name == base_name) { + return base_name.to_string(); + } + + for i in 1..100 { + let candidate = format!("{}_{}", base_name, i); + if !tools.iter().any(|t| t.function.name == candidate) { + return candidate; + } + } + + // Fallback with process id if somehow we can't find a unique name + format!("{}_{}_fallback", base_name, std::process::id()) +} + +async fn update_flow_status_module_with_actions( + db: &DB, + parent_job: &Uuid, + actions: &[AgentAction], +) -> Result<(), Error> { + let step = get_step_of_flow_status(db, parent_job.to_owned()).await?; + match step { + Step::Step(step) => { + sqlx::query!( + r#" + UPDATE v2_job_status SET + flow_status = jsonb_set( + flow_status, + array['modules', $3::TEXT, 'agent_actions'], + $2 + ) + WHERE id = $1 + "#, + parent_job, + sqlx::types::Json(actions) as _, + step as i32 + ) + .execute(db) + .await?; + } + _ => {} + } + Ok(()) +} + +async fn update_flow_status_module_with_actions_success( + db: &DB, + parent_job: &Uuid, + action_success: bool, +) -> Result<(), Error> { + let step = get_step_of_flow_status(db, parent_job.to_owned()).await?; + match step { + Step::Step(step) => { + // Append the new bool to the existing array, or create a new array if it doesn't exist + sqlx::query!( + r#" + UPDATE v2_job_status SET + flow_status = jsonb_set( + flow_status, + array['modules', $2::TEXT, 'agent_actions_success'], + COALESCE( + flow_status->'modules'->$2->'agent_actions_success', + to_jsonb(ARRAY[]::bool[]) + ) || to_jsonb(ARRAY[$3::bool]) + ) + WHERE id = $1 + "#, + parent_job, + step as i32, + action_success + ) + .execute(db) + .await?; + } + _ => {} + } + Ok(()) +} + +/// Check if the provider is Anthropic (either direct or through OpenRouter) +fn is_anthropic_provider(provider: &ProviderWithResource) -> bool { + let provider_is_anthropic = provider.kind.is_anthropic(); + let is_openrouter_anthropic = + provider.kind == AIProvider::OpenRouter && provider.model.starts_with("anthropic/"); + provider_is_anthropic || is_openrouter_anthropic +} + +#[async_recursion] +pub async fn run_agent( + // connection + db: &DB, + conn: &Connection, + + // agent job and flow data + job: &MiniPulledJob, + parent_job: &Uuid, + args: &AIAgentArgs, + tools: &[Tool], + + // job execution context + client: &AuthedClient, + occupancy_metrics: &mut OccupancyMetrics, + job_completed_tx: &JobCompletedSender, + worker_dir: &str, + base_internal_url: &str, + worker_name: &str, + hostname: &str, + killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, +) -> error::Result> { + let output_type = args.output_type.as_ref().unwrap_or(&OutputType::Text); + let base_url = args.provider.get_base_url(db).await?; + let api_key = args.provider.get_api_key(); + + // Create the query builder for the provider + let query_builder = create_query_builder(&args.provider); + + // Initialize messages + let mut messages = + if let Some(system_prompt) = args.system_prompt.clone().filter(|s| !s.is_empty()) { + vec![OpenAIMessage { + role: "system".to_string(), + content: Some(OpenAIContent::Text(system_prompt)), + ..Default::default() + }] + } else { + vec![] + }; + + // Create user message with optional images + let mut parts = vec![ContentPart::Text { text: args.user_message.clone() }]; + if let Some(images) = &args.user_images { + for image in images.iter() { + if !image.s3.is_empty() { + parts.push(ContentPart::S3Object { s3_object: image.clone() }); + } + } + } + let user_content = OpenAIContent::Parts(parts); + + messages.push(OpenAIMessage { + role: "user".to_string(), + content: Some(user_content), + ..Default::default() + }); + + let mut actions = vec![]; + let mut content = None; + + // Check if this provider supports tools with the current output type + let supports_tools = query_builder.supports_tools_with_output_type(output_type); + + let mut tool_defs: Option> = if tools.is_empty() || !supports_tools { + None + } else { + Some(tools.iter().map(|t| t.def.clone()).collect()) + }; + + // Handle structured output schema + let has_output_properties = args + .output_schema + .as_ref() + .and_then(|schema| schema.properties.as_ref()) + .map(|props| !props.is_empty()) + .unwrap_or(false); + + let is_anthropic = is_anthropic_provider(&args.provider); + let mut used_structured_output_tool = false; + let mut structured_output_tool_name: Option = None; + + // For text output with schema, handle structured output + if has_output_properties && output_type == &OutputType::Text { + let schema = args.output_schema.as_ref().unwrap(); + if is_anthropic { + // Anthropic uses a tool for structured output + let unique_tool_name = find_unique_tool_name("structured_output", tool_defs.as_deref()); + structured_output_tool_name = Some(unique_tool_name.clone()); + + let output_tool = ToolDef { + r#type: "function".to_string(), + function: ToolDefFunction { + name: unique_tool_name, + description: Some( + "This tool MUST be used last to return a structured JSON object as the final output." + .to_string(), + ), + parameters: to_raw_value(&schema), + }, + }; + if let Some(ref mut existing_tools) = tool_defs { + existing_tools.push(output_tool); + } else { + tool_defs = Some(vec![output_tool]); + } + } + // For non-Anthropic providers, response_format is handled by the query builder + } + + // Main agent loop + for i in 0..MAX_AGENT_ITERATIONS { + if used_structured_output_tool { + break; + } + + // For text output or image output with tools + let build_args = BuildRequestArgs { + messages: &messages, + tools: tool_defs.as_deref(), + model: args.provider.get_model(), + temperature: args.temperature, + max_tokens: args.max_completion_tokens, + output_schema: args.output_schema.as_ref(), + output_type, + system_prompt: args.system_prompt.as_deref(), + user_message: &args.user_message, + images: args.user_images.as_deref(), + }; + + let request_body = query_builder + .build_request(&build_args, client, &job.workspace_id) + .await?; + + let endpoint = + query_builder.get_endpoint(&base_url, args.provider.get_model(), output_type); + let auth_headers = query_builder.get_auth_headers(api_key, output_type); + + let mut request = HTTP_CLIENT + .post(&endpoint) + .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT_SECONDS)) + .header("Content-Type", "application/json"); + + // Apply authentication headers + for (header_name, header_value) in auth_headers { + request = request.header(header_name, header_value); + } + + let resp = request + .body(request_body) + .send() + .await + .map_err(|e| Error::internal_err(format!("Failed to call API: {}", e)))?; + + match resp.error_for_status_ref() { + Ok(_) => { + let parsed = query_builder.parse_response(resp).await?; + + match parsed { + ParsedResponse::Text { content: response_content, tool_calls } => { + if let Some(ref response_content) = response_content { + actions.push(AgentAction::Message {}); + messages.push(OpenAIMessage { + role: "assistant".to_string(), + content: Some(OpenAIContent::Text(response_content.clone())), + agent_action: Some(AgentAction::Message {}), + ..Default::default() + }); + + update_flow_status_module_with_actions(db, parent_job, &actions) + .await?; + update_flow_status_module_with_actions_success(db, parent_job, true) + .await?; + + content = Some(OpenAIContent::Text(response_content.clone())); + } + + if tool_calls.is_empty() { + break; + } else if i == MAX_AGENT_ITERATIONS - 1 { + return Err(Error::internal_err( + "AI agent reached max iterations, but there are still tool calls" + .to_string(), + )); + } + + messages.push(OpenAIMessage { + role: "assistant".to_string(), + tool_calls: Some(tool_calls.clone()), + ..Default::default() + }); + + // Handle tool calls (keeping existing tool execution logic) + for tool_call in tool_calls.iter() { + // Check if this is the structured output tool + if structured_output_tool_name + .as_ref() + .map_or(false, |name| tool_call.function.name == *name) + { + used_structured_output_tool = true; + messages.push(OpenAIMessage { + role: "tool".to_string(), + content: Some(OpenAIContent::Text( + "Successfully ran structured_output tool".to_string(), + )), + tool_call_id: Some(tool_call.id.clone()), + ..Default::default() + }); + messages.push(OpenAIMessage { + role: "assistant".to_string(), + content: Some(OpenAIContent::Text( + tool_call.function.arguments.clone(), + )), + agent_action: Some(AgentAction::Message {}), + ..Default::default() + }); + content = + Some(OpenAIContent::Text(tool_call.function.arguments.clone())); + break; + } + + // Execute regular tool + let tool = tools + .iter() + .find(|t| t.def.function.name == tool_call.function.name); + if let Some(tool) = tool { + let job_id = ulid::Ulid::new().into(); + actions.push(AgentAction::ToolCall { + job_id, + function_name: tool_call.function.name.clone(), + module_id: tool.module.id.clone(), + }); + + update_flow_status_module_with_actions(db, parent_job, &actions) + .await?; + + let tool_call_args = + serde_json::from_str::>>( + &tool_call.function.arguments, + )?; + + let job_payload = match tool.module.get_value()? { + FlowModuleValue::Script { + path: script_path, + hash: script_hash, + tag_override, + .. + } => { + let payload = script_to_payload( + script_hash, + script_path, + db, + job, + &tool.module, + tag_override, + tool.module.apply_preprocessor, + ) + .await?; + payload + } + FlowModuleValue::RawScript { + path, + content, + language, + lock, + tag, + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + .. + } => { + let path = path.unwrap_or_else(|| { + format!( + "{}/tools/{}", + job.runnable_path(), + tool.module.id + ) + }); + + let payload = raw_script_to_payload( + path, + content, + language, + lock, + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + &tool.module, + tag, + tool.module.delete_after_use.unwrap_or(false), + ); + payload + } + _ => { + return Err(Error::internal_err(format!( + "Unsupported tool: {}", + tool_call.function.name + ))); + } + }; + + let mut tx = db.begin().await?; + + let job_perms = windmill_common::auth::get_job_perms( + &mut *tx, + &job.id, + &job.workspace_id, + ) + .await? + .map(|x| x.into()); + + let (email, permissioned_as) = + if let Some(on_behalf_of) = job_payload.on_behalf_of.as_ref() { + (&on_behalf_of.email, on_behalf_of.permissioned_as.clone()) + } else { + (&job.permissioned_as_email, job.permissioned_as.to_owned()) + }; + + let job_priority = tool.module.priority.or(job.priority); + + let tx = PushIsolationLevel::Transaction(tx); + let (uuid, tx) = push( + db, + tx, + &job.workspace_id, + job_payload.payload, + PushArgs { args: &tool_call_args, extra: None }, + &job.created_by, + email, + permissioned_as, + Some(&format!("job-span-{}", job.id)), + None, + job.schedule_path(), + Some(job.id), + None, + None, + Some(job_id), + false, + false, + None, + job.visible_to_owner, + Some(job.tag.clone()), + job_payload.timeout, + None, + job_priority, + job_perms.as_ref(), + true, + ) + .await?; + + tx.commit().await?; + + let tool_job = get_mini_pulled_job(db, &uuid).await?; + + let Some(tool_job) = tool_job else { + return Err(Error::internal_err( + "Tool job not found".to_string(), + )); + }; + + let tool_job = Arc::new(tool_job); + + let job_dir = create_job_dir(&worker_dir, job.id).await; + + let (inner_job_completed_tx, inner_job_completed_rx) = + JobCompletedSender::new(&conn, 1); + + let inner_job_completed_rx = inner_job_completed_rx.expect( + "inner_job_completed_tx should be set as agent jobs are not supported on agent workers", + ); + + #[cfg(feature = "benchmark")] + let mut bench = windmill_common::bench::BenchmarkIter::new(); + + match handle_queued_job( + tool_job.clone(), + None, + None, + None, + None, + conn, + client, + hostname, + worker_name, + worker_dir, + &job_dir, + None, + base_internal_url, + inner_job_completed_tx, + occupancy_metrics, + killpill_rx, + None, + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await + { + Err(err) => { + let err_string = + format!("{}: {}", err.name(), err.to_string()); + let err_json = error_to_value(&err); + let _ = handle_non_flow_job_error( + db, + &tool_job, + 0, + None, + err_string.clone(), + err_json, + worker_name, + ) + .await; + messages.push(OpenAIMessage { + role: "tool".to_string(), + content: Some(OpenAIContent::Text(format!( + "Error running tool: {}", + err_string + ))), + tool_call_id: Some(tool_call.id.clone()), + agent_action: Some(AgentAction::ToolCall { + job_id, + function_name: tool_call.function.name.clone(), + module_id: tool.module.id.clone(), + }), + ..Default::default() + }); + update_flow_status_module_with_actions_success( + db, parent_job, false, + ) + .await?; + } + Ok(success) => { + let send_result = + inner_job_completed_rx.bounded_rx.try_recv().ok(); + + let result = if let Some(SendResult { + result: + SendResultPayload::JobCompleted(JobCompleted { + result, + .. + }), + .. + }) = send_result.as_ref() + { + job_completed_tx + .send( + send_result.as_ref().unwrap().result.clone(), + true, + ) + .await + .map_err(to_anyhow)?; + result + } else { + if let Some(send_result) = send_result { + job_completed_tx + .send(send_result.result, true) + .await + .map_err(to_anyhow)?; + } + return Err(Error::internal_err( + "Tool job completed but no result".to_string(), + )); + }; + + messages.push(OpenAIMessage { + role: "tool".to_string(), + content: Some(OpenAIContent::Text( + result.get().to_string(), + )), + tool_call_id: Some(tool_call.id.clone()), + agent_action: Some(AgentAction::ToolCall { + job_id, + function_name: tool_call.function.name.clone(), + module_id: tool.module.id.clone(), + }), + ..Default::default() + }); + update_flow_status_module_with_actions_success( + db, parent_job, success, + ) + .await?; + } + } + } else { + return Err(Error::internal_err(format!( + "Tool not found: {}", + tool_call.function.name + ))); + } + } + } + ParsedResponse::Image { base64_data } => { + // For image output with tools, we got an image response + let s3_object = upload_image_to_s3(&base64_data, job, client).await?; + return Ok(to_raw_value(&s3_object)); + } + } + } + Err(e) => { + let _status = resp.status(); + let text = resp + .text() + .await + .unwrap_or_else(|_| "".to_string()); + return Err(Error::internal_err(format!("API error: {} - {}", e, text))); + } + } + } + + // Return the final result + let final_messages: Vec = messages + .iter() + .map(|m| Message { message: m, agent_action: m.agent_action.as_ref() }) + .collect(); + + // Parse content as JSON for structured output, fallback to string if it fails + let output_value = match content { + Some(content_str) => match has_output_properties { + true => match content_str { + OpenAIContent::Text(text) => { + serde_json::from_str::>(&text).map_err(|_e| { + Error::internal_err(format!("Failed to parse structured output: {}", text)) + }) + } + OpenAIContent::Parts(_parts) => Err(Error::internal_err( + "Failed to parse structured output".to_string(), + )), + }, + false => Ok(match content_str { + OpenAIContent::Text(text) => to_raw_value(&text), + OpenAIContent::Parts(parts) => to_raw_value(&parts), + }), + }?, + None => to_raw_value(&""), + }; + + Ok(to_raw_value(&AIAgentResult { + output: output_value, + messages: final_messages, + })) +} diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index ace218c207..87f8282e9e 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -16,6 +16,7 @@ mod java_executor; #[cfg(feature = "ruby")] mod ruby_executor; +mod ai; mod ai_executor; mod bun_executor; pub mod common; diff --git a/frontend/src/lib/components/InputTransformForm.svelte b/frontend/src/lib/components/InputTransformForm.svelte index 617f822f46..fdf354da96 100644 --- a/frontend/src/lib/components/InputTransformForm.svelte +++ b/frontend/src/lib/components/InputTransformForm.svelte @@ -11,6 +11,7 @@ import type { Schema } from '$lib/common' import type { InputCat } from '$lib/utils' import { createEventDispatcher, getContext, untrack } from 'svelte' + import { computeShow } from '$lib/utils' import ArgInput from './ArgInput.svelte' import FieldHeader from './FieldHeader.svelte' @@ -58,6 +59,7 @@ hideHelpButton?: boolean class?: string editor?: SimpleEditor | undefined + otherArgs?: Record } let { @@ -79,7 +81,8 @@ enableAi = false, hideHelpButton = false, class: className = '', - editor = $bindable(undefined) + editor = $bindable(undefined), + otherArgs = {} }: Props = $props() let monaco: SimpleEditor | undefined = $state(undefined) @@ -87,6 +90,8 @@ let argInput: ArgInput | undefined = $state(undefined) let focusedPrev = false + let hidden = $state(false) + const variableMatch = (value: string): RegExpMatchArray | null => value.match(/^variable\('([^']+)'\)$/) const resourceMatch = (value: string): RegExpMatchArray | null => @@ -259,6 +264,47 @@ } } + function handleFieldVisibility( + schema: Schema | any, + arg: InputTransform | any, + otherArgs: Record + ) { + const schemaProperty = schema?.properties?.[argName] + if (schemaProperty?.showExpr) { + // Build args object with current field value and other context + const currentValue = propertyType === 'static' ? arg?.value : arg?.expr + + // Convert otherArgs from InputTransform objects to their actual values + const contextArgs = { + [argName]: currentValue + } + + // Extract values from InputTransform objects in otherArgs + Object.keys(otherArgs ?? {}).forEach((key) => { + const otherArg = otherArgs[key] + const otherArgValue = otherArg.type === 'static' ? otherArg.value : otherArg.expr + contextArgs[key] = otherArgValue + }) + + const shouldShow = computeShow(argName, schemaProperty.showExpr, contextArgs) + if (shouldShow) { + hidden = false + } else if (!hidden) { + hidden = true + // Clear the arg value when hidden (following SchemaForm pattern) + if (arg) { + arg.value = undefined + arg.expr = undefined + } + // Make sure validation passes when hidden + inputCheck = true + } + } else { + // No showExpr, always show + hidden = false + } + } + function onFocus() { focused = true if (isStaticTemplate(inputCat)) { @@ -359,12 +405,20 @@ $effect(() => { schema?.properties?.[argName]?.default && untrack(() => setDefaultCode()) }) + $effect.pre(() => { + // Monitor changes that affect field visibility + JSON.stringify(schema) + JSON.stringify(arg) + JSON.stringify(otherArgs) + + untrack(() => handleFieldVisibility(schema, arg, otherArgs)) + }) let connecting = $derived( $propPickerConfig?.propName == argName && $propPickerConfig?.insertionMode == 'connect' ) -{#if arg != undefined} +{#if arg != undefined && !hidden}
key !== argName) + )} />
{/if} diff --git a/frontend/src/lib/components/flows/flowInfers.ts b/frontend/src/lib/components/flows/flowInfers.ts index 8f419fce5c..7c24278cc2 100644 --- a/frontend/src/lib/components/flows/flowInfers.ts +++ b/frontend/src/lib/components/flows/flowInfers.ts @@ -62,50 +62,58 @@ export async function loadSchemaFromModule(module: FlowModule): Promise<{ type: 'object', format: 'ai-provider' }, + output_type: { + type: 'string', + description: + 'The type of output the AI agent will generate (text or image). Image output requires a configured workspace S3 storage, will ignore tools, and only works with OpenAI, Google AI and OpenRouter gemini-image-preview model.', + enum: ['text', 'image'], + default: 'text' + }, user_message: { - type: 'string' + type: 'string', + description: 'The message to give as input to the AI agent.' }, system_prompt: { - type: 'string' + type: 'string', + description: 'The system prompt to give as input to the AI agent.' }, - image: { - type: 'object', - description: 'Image to send to the AI agent (optional)', - format: 'resource-s3_object' + user_images: { + type: 'array', + description: + 'Array of images to give as input to the AI agent. Requires a configured workspace S3 storage.', + items: { + type: 'object' as const, + resourceType: 's3object' + } }, max_completion_tokens: { - type: 'number' + type: 'number', + description: 'The maximum number of output tokens.' }, temperature: { type: 'number', description: - 'Controls randomness in text generation. Range: 0.0 (deterministic) to 2.0 (random).' - }, - output_type: { - type: 'string', - description: - 'The type of output the AI agent will generate (text or image). Image output will ignore tools, and only works with OpenAI, Google AI and OpenRouter gemini-image-preview model.', - enum: ['text', 'image'], - default: 'text' + 'Controls randomness in text generation. Range: 0.0 (deterministic) to 2.0 (random).', + showExpr: "fields.output_type === 'text'" }, output_schema: { type: 'object', description: 'JSON schema that the AI agent will follow for its response format (only used if output_type is text)', - format: 'json-schema' + format: 'json-schema', + showExpr: "fields.output_type === 'text'" } }, - required: ['provider', 'model', 'user_message'], + required: ['provider', 'user_message', 'output_type'], type: 'object', order: [ 'provider', - 'model', + 'output_type', 'user_message', 'system_prompt', - 'image', + 'user_images', 'max_completion_tokens', 'temperature', - 'output_type', 'output_schema' ] } From 4b7149527b52ac42094af580e3788326c30a0c70 Mon Sep 17 00:00:00 2001 From: dieriba Date: Thu, 18 Sep 2025 08:31:02 +0200 Subject: [PATCH 05/17] feat: fix gcp cleanup and add ack-deadline for gcp push delivery #6631 --- backend/ee-repo-ref.txt | 2 +- backend/windmill-api/src/triggers/listener.rs | 21 ++++++--- .../src/triggers/mqtt/listener.rs | 2 + .../src/triggers/postgres/listener.rs | 3 ++ .../src/triggers/websocket/listener.rs | 2 + .../triggers/gcp/GcpTriggerEditorInner.svelte | 46 +++++++++---------- 6 files changed, 43 insertions(+), 33 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 8e99fde120..20dc3db333 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -89ecae7db8c46cfc571eb58bcd7f4ba6f78e8aba \ No newline at end of file +daa1c26c268c778e55756f02a459b6c7628c9267 \ No newline at end of file diff --git a/backend/windmill-api/src/triggers/listener.rs b/backend/windmill-api/src/triggers/listener.rs index d0ca14e9eb..aefd255068 100644 --- a/backend/windmill-api/src/triggers/listener.rs +++ b/backend/windmill-api/src/triggers/listener.rs @@ -32,6 +32,7 @@ use windmill_common::{ pub trait Listener: TriggerCrud + TriggerJobArgs { type Consumer: Send; type Extra: Send + Sync; + type ExtraState: Send + Sync; //to use in next PR to add job trigger kind to eow #[allow(unused)] @@ -53,6 +54,7 @@ pub trait Listener: TriggerCrud + TriggerJobArgs { listening_trigger: &ListeningTrigger, err_message: Arc>>, killpill_rx: tokio::sync::broadcast::Receiver<()>, + extra: Option<&Self::ExtraState>, ); async fn fetch_enabled_unlistened_triggers( &self, @@ -159,10 +161,15 @@ pub trait Listener: TriggerCrud + TriggerJobArgs { Ok(captures) } + async fn get_extra_state(&self) -> Option { + None + } + async fn cleanup( &self, _db: &DB, _listening_trigger: &ListeningTrigger, + _extra: Option<&Self::ExtraState>, ) -> Result<()> { Ok(()) } @@ -568,14 +575,14 @@ async fn listening( let killpill_rx_get_consumer = killpill_rx.resubscribe(); let loop_ping_status = Arc::new(RwLock::new(None)); - + let extra_state = listener.get_extra_state().await; tokio::select! { biased; _ = killpill_rx.recv() => { - let _ = listener.cleanup(&db, &listening_trigger).await; + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; } _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), Some("Connecting...".to_string())) => { - let _ = listener.cleanup(&db, &listening_trigger).await; + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; } consumer = { listener.get_consumer(&db, &listening_trigger, loop_ping_status.clone(), killpill_rx_get_consumer) @@ -583,18 +590,18 @@ async fn listening( tokio::select! { biased; _ = killpill_rx.recv() => { - let _ = listener.cleanup(&db, &listening_trigger).await; + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; return; } _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), None) => { - let _ = listener.cleanup(&db, &listening_trigger).await; + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; return; } _ = async { match consumer { Ok(Some(consumer)) => { listener.update_ping_and_loop_ping_status(&db, &listening_trigger, loop_ping_status.clone(), None).await; - let _ = listener.consume(&db, consumer, &listening_trigger, loop_ping_status.clone(), killpill_rx_consumer).await; + let _ = listener.consume(&db, consumer, &listening_trigger, loop_ping_status.clone(), killpill_rx_consumer, extra_state.as_ref()).await; tracing::debug!("Stopping consumer for trigger"); } Err(error) => { @@ -604,7 +611,7 @@ async fn listening( _ => {} } } => { - let _ = listener.cleanup(&db, &listening_trigger).await; + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; return; } } diff --git a/backend/windmill-api/src/triggers/mqtt/listener.rs b/backend/windmill-api/src/triggers/mqtt/listener.rs index 34af0b5eb1..af2d94b113 100644 --- a/backend/windmill-api/src/triggers/mqtt/listener.rs +++ b/backend/windmill-api/src/triggers/mqtt/listener.rs @@ -36,6 +36,7 @@ use crate::{ impl Listener for MqttTrigger { type Consumer = MqttClientResult; type Extra = (); + type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Mqtt; async fn get_consumer( @@ -100,6 +101,7 @@ impl Listener for MqttTrigger { listening_trigger: &ListeningTrigger, _err_message: Arc>>, _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, ) { tracing::info!( "Starting to listen for MQTT trigger {}", diff --git a/backend/windmill-api/src/triggers/postgres/listener.rs b/backend/windmill-api/src/triggers/postgres/listener.rs index 7bdfab0ed2..bfb527e3e7 100644 --- a/backend/windmill-api/src/triggers/postgres/listener.rs +++ b/backend/windmill-api/src/triggers/postgres/listener.rs @@ -133,6 +133,7 @@ impl PostgresSimpleClient { impl Listener for PostgresTrigger { type Consumer = (CopyBothDuplex, LogicalReplicationSettings); type Extra = (); + type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Postgres; async fn get_consumer( @@ -209,6 +210,7 @@ impl Listener for PostgresTrigger { listening_trigger: &ListeningTrigger, err_message: Arc>>, _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, ) { let (logical_replication_stream, logical_replication_settings) = consumer; pin_mut!(logical_replication_stream); @@ -403,6 +405,7 @@ impl Listener for PostgresTrigger { &self, db: &DB, listening_trigger: &ListeningTrigger, + _extra_state: Option<&Self::ExtraState>, ) -> Result<()> { let authed = listening_trigger .authed(db, &Self::TRIGGER_KIND.to_string()) diff --git a/backend/windmill-api/src/triggers/websocket/listener.rs b/backend/windmill-api/src/triggers/websocket/listener.rs index 80eee431c4..7fb22f2b08 100644 --- a/backend/windmill-api/src/triggers/websocket/listener.rs +++ b/backend/windmill-api/src/triggers/websocket/listener.rs @@ -164,6 +164,7 @@ impl Listener for WebsocketTrigger { Response>>, ); type Extra = ReturnMessageChannels; + type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Websocket; async fn get_consumer( &self, @@ -214,6 +215,7 @@ impl Listener for WebsocketTrigger { listening_trigger: &ListeningTrigger, err_message: Arc>>, mut killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, ) { let WebsocketConfig { ref url, .. } = listening_trigger.trigger_config; diff --git a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte index edb2e2d644..abb3ab8782 100644 --- a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte @@ -445,32 +445,28 @@
{/if} - -
- -
-
- Leave empty to use subscription default (600 seconds). This affects how long - messages remain in flight before being redelivered. -
-
- {:else} -
- No settings available for push delivery type -
{/if} + +
+ +
+
+ Leave empty to use subscription default (600 seconds). This affects how long + messages remain in flight before being redelivered. +
+
{:else} Date: Thu, 18 Sep 2025 11:20:26 +0200 Subject: [PATCH 06/17] feat(flow): Add helper to add expression to arrays (#6629) * Implement array expression helper for number arrays in JS mode - Add showArrayExprPicker state to manage helper UI - Add shouldShowArrayHelper() to check conditions (JS mode + number array) - Add 'Add item' button that shows picker UI when clicked - Implement picker row with disabled input, FlowPlugConnect, and Cancel button - Connect callback sets array expression [property] and updates Monaco editor - Include helpful text and proper cleanup on cancel/connect * Enhance array expression helper to append items to existing arrays - Check if current expr is already an array expression [...] - If it is, append new item to existing content: [existing, newItem] - If not or empty, create new array with single item: [newItem] - Update helper text to reflect append behavior - Maintains backward compatibility with non-array expressions * Add S3 resource array helper for JavaScript mode - Add shouldShowS3ArrayHelper() function to detect S3 resource arrays - Show direct FlowPlugConnect for S3 arrays instead of Add item button - Apply same append logic: add to existing array or create new one - Include helpful text explaining S3 resource connection - Support both s3_object and s3object resourceType variants * Add S3 resource catalog helper for static mode arrays - Add shouldShowS3ArrayStaticHelper() to detect S3 arrays in static mode - Show 'Add an object from the catalog' button below static S3 array inputs - Button switches to JavaScript mode and immediately activates connect mode - Sets initial empty array [] then replaces with [selectedPath] when connected - Includes helpful text explaining the mode switch and connection * Fix reactivity issue when switching from static to JS mode - Make button click handler async and await tick() before activating connect mode - Add Monaco editor update after setting expression in connect callback - Use tick().then() to ensure Monaco is available before calling setCode() - This ensures the SimpleEditor displays the new array expression immediately * Add plug icon to 'Add object from an expression' button - Import Plug icon from lucide-svelte - Add startIcon with Plug to the S3 array static helper button - Makes the button visually consistent with other connection-related UI elements * Unify S3 resource button style across static and JS modes - Replace 'Add S3 resource:' text + FlowPlugConnect with consistent Button style - Use same variant, color, size, and plug icon as static mode button - Maintain same functionality but with unified visual appearance - Both S3 helpers now use identical button styling * Consolidate and clean up array expression helpers - Extract appendPathToArrayExpr() to eliminate duplicate array building logic - Add switchToJsAndConnect() helper for consistent mode switching flow - Add emitChange() and updateEditor() utilities for consistent updates - Add safety reset of showArrayExprPicker when switching away from JS mode - Reduce code duplication across number and S3 array helpers - Improve maintainability and consistency * Remove number array helper functionality - Remove shouldShowArrayHelper() function for number arrays - Remove showArrayExprPicker state variable and related UI - Remove number array 'Add item' button and picker interface - Keep only S3 resource array helpers (static and JS modes) - Clean up unused safety reset logic for array picker * Create reusable S3ArrayHelperButton component - Extract S3 array helper button into dedicated component - Add consistent styling with Plug icon and configurable label - Replace both static and JavaScript mode button implementations - Reduce code duplication and improve maintainability - Component dispatches click event for parent handling * cleaning * Hide S3ArrayHelperButton when in connect mode - Add connecting prop to S3ArrayHelperButton component - Hide button when connecting is true to avoid UI clutter - Pass connecting state from InputTransformForm to both button instances - Improves UX by removing unnecessary button when plug is already active * cleaning * cleaning --- .../lib/components/InputTransformForm.svelte | 71 +++++++++++++++++++ .../lib/components/S3ArrayHelperButton.svelte | 20 ++++++ 2 files changed, 91 insertions(+) create mode 100644 frontend/src/lib/components/S3ArrayHelperButton.svelte diff --git a/frontend/src/lib/components/InputTransformForm.svelte b/frontend/src/lib/components/InputTransformForm.svelte index fdf354da96..c14a89da7b 100644 --- a/frontend/src/lib/components/InputTransformForm.svelte +++ b/frontend/src/lib/components/InputTransformForm.svelte @@ -38,6 +38,7 @@ import { twMerge } from 'tailwind-merge' import FlowPlugConnect from './FlowPlugConnect.svelte' import { deepEqual } from 'fast-equals' + import S3ArrayHelperButton from './S3ArrayHelperButton.svelte' interface Props { schema: Schema | { properties?: Record; required?: string[] } @@ -241,6 +242,51 @@ return inputCat === 'string' || inputCat === 'sql' || inputCat == 'yaml' } + function appendPathToArrayExpr(currentExpr: string | undefined, path: string) { + const trimmedExpr = currentExpr?.trim() || '' + + let newExpr = trimmedExpr + if (trimmedExpr.startsWith('[') && trimmedExpr.endsWith(']')) { + // Parse existing array and append new item + const innerContent = trimmedExpr.slice(1, -1).trim() + if (innerContent) { + newExpr = `[${innerContent}, ${path}]` + } else { + newExpr = `[${path}]` + } + } else { + // Create new array with single item + newExpr = `[${path}]` + } + arg.expr = newExpr + arg.type = 'javascript' + + // Update Monaco editor after setting the expression + tick().then(() => { + monaco?.setCode(newExpr) + }) + + // Dispatch change + dispatch('change', { argName, arg }) + } + + async function switchToJsAndConnect(onPath: (path: string) => void) { + // Switch to JavaScript mode + propertyType = 'javascript' + arg.type = 'javascript' + arg.expr = arg.expr || '[]' + arg.value = undefined + + // Wait for the component to re-render and Monaco to be available + await tick() + + // Activate connect mode + focusProp?.(argName, 'connect', (path) => { + onPath(path) + return true + }) + } + function connectProperty(rawValue: string) { // Extract path from variable('x') or resource('x') format const varMatch = variableMatch(rawValue) @@ -416,6 +462,10 @@ let connecting = $derived( $propPickerConfig?.propName == argName && $propPickerConfig?.insertionMode == 'connect' ) + let shouldShowS3ArrayHelper = $derived( + inputCat === 'list' && + ['s3object', 's3_object'].includes(schema?.properties?.[argName]?.items?.resourceType) + ) {#if arg != undefined && !hidden} @@ -492,6 +542,7 @@ on:selected={(e) => { if (e.detail == propertyType) return const staticTemplate = isStaticTemplate(inputCat) + if (e.detail === 'javascript') { if (arg.expr == undefined) { arg.expr = getDefaultExpr( @@ -683,6 +734,14 @@ bind:title={schema.properties[argName].title} bind:placeholder={schema.properties[argName].placeholder} /> + + {#if shouldShowS3ArrayHelper} + + switchToJsAndConnect((path) => appendPathToArrayExpr(arg.expr, path))} + /> + {/if} {:else if arg.expr != undefined}
{/if} + + {#if shouldShowS3ArrayHelper} + + focusProp?.(argName, 'connect', (path) => { + appendPathToArrayExpr(arg.expr, path) + return true + })} + /> + {/if} +
{:else} Not recognized input type {argName} ({arg.expr}, {propertyType}) diff --git a/frontend/src/lib/components/S3ArrayHelperButton.svelte b/frontend/src/lib/components/S3ArrayHelperButton.svelte new file mode 100644 index 0000000000..2ea6450602 --- /dev/null +++ b/frontend/src/lib/components/S3ArrayHelperButton.svelte @@ -0,0 +1,20 @@ + + +{#if !connecting} +
+ +
+{/if} From a0d7f5a4a0c862f7574efd190167218b6abd1ae2 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Thu, 18 Sep 2025 15:10:14 +0200 Subject: [PATCH 07/17] internal(git commands): use own bot to trigger ci + exit on failure (#6632) * use own bot to trigger ci + exit on failure * fix --- .github/workflows/git-commands.yaml | 46 ++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/.github/workflows/git-commands.yaml b/.github/workflows/git-commands.yaml index cafc4fed05..8a5d94df64 100644 --- a/.github/workflows/git-commands.yaml +++ b/.github/workflows/git-commands.yaml @@ -29,10 +29,16 @@ jobs: --health-retries 5 steps: + - uses: actions/create-github-app-token@v2 + id: app + with: + app-id: ${{ vars.INTERNAL_APP_ID }} + private-key: ${{ secrets.INTERNAL_APP_KEY }} + - name: Comment on PR - Starting uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app.outputs.token }} script: | github.rest.issues.createComment({ issue_number: context.issue.number, @@ -44,6 +50,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 with: + token: ${{ steps.app.outputs.token }} ref: ${{ github.event.issue.pull_request.head.ref }} fetch-depth: 0 @@ -70,21 +77,25 @@ jobs: - name: Run update-sqlx script env: DATABASE_URL: postgres://postgres:postgres@localhost:5432/windmill - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ steps.app.outputs.token }} run: | + set -e # Exit on any command failure PR_NUMBER=${{ github.event.issue.number }} BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName) echo "Checking out PR branch: $BRANCH_NAME" git checkout $BRANCH_NAME - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" + git config --local user.email "windmill-internal-app[bot]@users.noreply.github.com" + git config --local user.name "windmill-internal-app[bot]" git config pull.rebase true git pull origin $BRANCH_NAME - mkdir frontend/build + mkdir -p frontend/build cd backend cargo install sqlx-cli --version 0.8.5 sqlx migrate run - ./update_sqlx.sh --dir ./windmill-ee-private + if ! ./update_sqlx.sh --dir ./windmill-ee-private; then + gh pr comment $PR_NUMBER --body "❌ SQLx update failed. Please check the workflow logs for details." + exit 1 + fi # Pass the branch name to the next step echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV @@ -97,7 +108,7 @@ jobs: - name: Comment on PR - Completed uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app.outputs.token }} script: | github.rest.issues.createComment({ issue_number: context.issue.number, @@ -114,10 +125,16 @@ jobs: pull-requests: write issues: write steps: + - uses: actions/create-github-app-token@v2 + id: app + with: + app-id: ${{ vars.INTERNAL_APP_ID }} + private-key: ${{ secrets.INTERNAL_APP_KEY }} + - name: Comment on PR - Starting uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app.outputs.token }} script: | github.rest.issues.createComment({ issue_number: context.issue.number, @@ -129,6 +146,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 with: + token: ${{ steps.app.outputs.token }} ref: ${{ github.event.issue.pull_request.head.ref }} fetch-depth: 0 @@ -149,19 +167,19 @@ jobs: - name: Update ee-repo-ref.txt env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ steps.app.outputs.token }} run: | + PR_NUMBER=${{ github.event.issue.number }} + BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName) echo "Checking out PR branch: $BRANCH_NAME" git checkout $BRANCH_NAME - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" + git config --local user.email "windmill-internal-app[bot]@users.noreply.github.com" + git config --local user.name "windmill-internal-app[bot]" git config pull.rebase true git pull origin $BRANCH_NAME echo "${{ steps.get-commit-hash.outputs.commit_hash }}" > backend/ee-repo-ref.txt echo "Updated backend/ee-repo-ref.txt with commit hash: ${{ steps.get-commit-hash.outputs.commit_hash }}" # commit and push the changes - PR_NUMBER=${{ github.event.issue.number }} - BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName) git add backend/ee-repo-ref.txt git commit -m "Update ee-repo-ref.txt" || echo "No changes to commit" git push origin $BRANCH_NAME @@ -169,7 +187,7 @@ jobs: - name: Comment on PR - Completed uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app.outputs.token }} script: | github.rest.issues.createComment({ issue_number: context.issue.number, From 709a937ac2dab3d8a18b01f200976fa2d4625e89 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Thu, 18 Sep 2025 15:22:24 +0200 Subject: [PATCH 08/17] fix(aiagent): fix endpoint for azure (#6633) * fix ai agent for azure * cleaning * add parentheses --- backend/windmill-api/src/ai.rs | 31 +++------------- backend/windmill-common/src/ai_providers.rs | 36 +++++++++++++++++++ .../src/ai/providers/google_ai.rs | 7 ++-- .../src/ai/providers/openai.rs | 31 +++++++++++----- .../src/ai/providers/openrouter.rs | 5 +-- .../windmill-worker/src/ai/query_builder.rs | 3 +- backend/windmill-worker/src/ai_executor.rs | 7 +++- 7 files changed, 77 insertions(+), 43 deletions(-) diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index abde227214..3c3ba77bb5 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -10,8 +10,8 @@ use reqwest::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::collections::HashMap; -use windmill_common::ai_providers::{AIProvider, ProviderConfig, ProviderModel}; use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::ai_providers::{AIProvider, ProviderConfig, ProviderModel, AZURE_API_VERSION}; use windmill_common::error::{to_anyhow, Error, Result}; lazy_static::lazy_static! { @@ -25,9 +25,6 @@ lazy_static::lazy_static! { pub static ref AI_REQUEST_CACHE: Cache<(String, AIProvider), ExpiringAIRequestConfig> = Cache::new(500); } -const AZURE_API_VERSION: &str = "2025-04-01-preview"; -const OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; - #[derive(Deserialize, Debug)] struct AIOAuthResource { client_id: String, @@ -154,21 +151,13 @@ impl AIRequestConfig { let base_url = self.base_url.trim_end_matches('/'); - let is_azure = matches!(provider, AIProvider::OpenAI) && base_url != OPENAI_BASE_URL - || matches!(provider, AIProvider::AzureOpenAI); + let is_azure = provider.is_azure_openai(base_url); let is_anthropic = matches!(provider, AIProvider::Anthropic); let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some(); let url = if is_azure && method != Method::GET { - if base_url.ends_with("/deployments") { - let model = Self::get_azure_model(&body)?; - format!("{}/{}/{}", base_url, model, path) - } else if base_url.ends_with("/openai") { - let model = Self::get_azure_model(&body)?; - format!("{}/deployments/{}/{}", base_url, model, path) - } else { - format!("{}/{}", base_url, path) - } + let model = AIProvider::extract_model_from_body(&body)?; + AIProvider::build_azure_openai_url(base_url, &model, path) } else if is_anthropic_sdk { let truncated_base_url = base_url.trim_end_matches("/v1"); format!("{}/{}", truncated_base_url, path) @@ -233,18 +222,6 @@ impl AIRequestConfig { .map_err(|e| Error::internal_err(format!("Failed to reserialize request body: {}", e)))? .into()) } - - fn get_azure_model(body: &Bytes) -> Result { - #[derive(Deserialize, Debug)] - struct AzureModel { - model: String, - } - - let azure_model: AzureModel = serde_json::from_slice(body) - .map_err(|e| Error::internal_err(format!("Failed to parse request body: {}", e)))?; - - Ok(azure_model.model) - } } #[derive(Clone, Debug)] diff --git a/backend/windmill-common/src/ai_providers.rs b/backend/windmill-common/src/ai_providers.rs index 577f613f21..46d5d687c3 100644 --- a/backend/windmill-common/src/ai_providers.rs +++ b/backend/windmill-common/src/ai_providers.rs @@ -10,6 +10,9 @@ lazy_static::lazy_static! { static ref OPENAI_AZURE_BASE_PATH: Option = std::env::var("OPENAI_AZURE_BASE_PATH").ok(); } +pub const AZURE_API_VERSION: &str = "2025-04-01-preview"; +pub const OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; + #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] #[serde(rename_all = "lowercase")] pub enum AIProvider { @@ -78,6 +81,39 @@ impl AIProvider { pub fn is_anthropic(&self) -> bool { matches!(self, AIProvider::Anthropic) } + + /// Check if this provider/URL combination represents Azure OpenAI + pub fn is_azure_openai(&self, base_url: &str) -> bool { + (matches!(self, AIProvider::OpenAI) && base_url != OPENAI_BASE_URL) + || matches!(self, AIProvider::AzureOpenAI) + } + + /// Build Azure OpenAI URL with deployment model path + pub fn build_azure_openai_url(base_url: &str, model: &str, path: &str) -> String { + let base_url = base_url.trim_end_matches('/'); + + if base_url.ends_with("/deployments") { + format!("{}/{}/{}", base_url, model, path) + } else if base_url.ends_with("/openai") { + format!("{}/deployments/{}/{}", base_url, model, path) + } else { + format!("{}/{}", base_url, path) + } + } + + /// Extract model from request body (needed for Azure deployments) + pub fn extract_model_from_body(body: &[u8]) -> Result { + #[derive(serde::Deserialize)] + struct ModelRequest { + model: String, + } + + let model_request: ModelRequest = serde_json::from_slice(body).map_err(|e| { + Error::internal_err(format!("Failed to parse request body for model: {}", e)) + })?; + + Ok(model_request.model) + } } impl TryFrom<&str> for AIProvider { diff --git a/backend/windmill-worker/src/ai/providers/google_ai.rs b/backend/windmill-worker/src/ai/providers/google_ai.rs index 9b08d40e46..b5a27a5f99 100644 --- a/backend/windmill-worker/src/ai/providers/google_ai.rs +++ b/backend/windmill-worker/src/ai/providers/google_ai.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json; -use windmill_common::{client::AuthedClient, error::Error}; +use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; use crate::ai::{ image_handler::download_and_encode_s3_image, @@ -166,7 +166,7 @@ impl QueryBuilder for GoogleAIQueryBuilder { match args.output_type { OutputType::Text => { // For text output, use OpenAI-compatible format - let openai_builder = super::openai::OpenAIQueryBuilder::new(); + let openai_builder = super::openai::OpenAIQueryBuilder::new(AIProvider::GoogleAI); openai_builder .build_request(args, client, workspace_id) .await @@ -180,7 +180,7 @@ impl QueryBuilder for GoogleAIQueryBuilder { // For chat completions (text), use OpenAI parser if url.contains("/chat/completions") { - let openai_builder = super::openai::OpenAIQueryBuilder::new(); + let openai_builder = super::openai::OpenAIQueryBuilder::new(AIProvider::GoogleAI); return openai_builder.parse_response(response).await; } @@ -259,6 +259,7 @@ impl QueryBuilder for GoogleAIQueryBuilder { fn get_auth_headers( &self, api_key: &str, + _base_url: &str, output_type: &OutputType, ) -> Vec<(&'static str, String)> { match output_type { diff --git a/backend/windmill-worker/src/ai/providers/openai.rs b/backend/windmill-worker/src/ai/providers/openai.rs index 26bc804aed..421c5761f6 100644 --- a/backend/windmill-worker/src/ai/providers/openai.rs +++ b/backend/windmill-worker/src/ai/providers/openai.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json; -use windmill_common::{client::AuthedClient, error::Error}; +use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; use crate::ai::{ image_handler::download_and_encode_s3_image, @@ -92,11 +92,13 @@ pub struct OpenAIRequest<'a> { pub response_format: Option, } -pub struct OpenAIQueryBuilder; +pub struct OpenAIQueryBuilder { + provider_kind: AIProvider, +} impl OpenAIQueryBuilder { - pub fn new() -> Self { - Self + pub fn new(provider_kind: AIProvider) -> Self { + Self { provider_kind } } pub async fn prepare_messages_for_api( @@ -332,18 +334,29 @@ impl QueryBuilder for OpenAIQueryBuilder { } } - fn get_endpoint(&self, base_url: &str, _model: &str, output_type: &OutputType) -> String { - match output_type { - OutputType::Text => format!("{}/chat/completions", base_url), - OutputType::Image => format!("{}/responses", base_url), + fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String { + let path = match output_type { + OutputType::Text => "chat/completions", + OutputType::Image => "responses", + }; + + if self.provider_kind.is_azure_openai(base_url) { + AIProvider::build_azure_openai_url(base_url, model, path) + } else { + format!("{}/{}", base_url, path) } } fn get_auth_headers( &self, api_key: &str, + base_url: &str, _output_type: &OutputType, ) -> Vec<(&'static str, String)> { - vec![("Authorization", format!("Bearer {}", api_key))] + if self.provider_kind.is_azure_openai(base_url) { + vec![("api-key", api_key.to_string())] + } else { + vec![("Authorization", format!("Bearer {}", api_key))] + } } } diff --git a/backend/windmill-worker/src/ai/providers/openrouter.rs b/backend/windmill-worker/src/ai/providers/openrouter.rs index 9fc6c6f051..3d5dd0314a 100644 --- a/backend/windmill-worker/src/ai/providers/openrouter.rs +++ b/backend/windmill-worker/src/ai/providers/openrouter.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json; -use windmill_common::{client::AuthedClient, error::Error}; +use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; use crate::ai::{ providers::openai::{OpenAIQueryBuilder, OpenAIResponse}, @@ -59,7 +59,7 @@ pub struct OpenRouterQueryBuilder { impl OpenRouterQueryBuilder { pub fn new() -> Self { - Self { openai_builder: OpenAIQueryBuilder::new() } + Self { openai_builder: OpenAIQueryBuilder::new(AIProvider::OpenRouter) } } } @@ -195,6 +195,7 @@ impl QueryBuilder for OpenRouterQueryBuilder { fn get_auth_headers( &self, api_key: &str, + _base_url: &str, _output_type: &OutputType, ) -> Vec<(&'static str, String)> { vec![("Authorization", format!("Bearer {}", api_key))] diff --git a/backend/windmill-worker/src/ai/query_builder.rs b/backend/windmill-worker/src/ai/query_builder.rs index 07b9c96e4e..6b043726bc 100644 --- a/backend/windmill-worker/src/ai/query_builder.rs +++ b/backend/windmill-worker/src/ai/query_builder.rs @@ -54,6 +54,7 @@ pub trait QueryBuilder: Send + Sync { fn get_auth_headers( &self, api_key: &str, + base_url: &str, output_type: &OutputType, ) -> Vec<(&'static str, String)>; } @@ -65,6 +66,6 @@ pub fn create_query_builder(provider: &ProviderWithResource) -> Box Box::new(GoogleAIQueryBuilder::new()), AIProvider::OpenRouter => Box::new(OpenRouterQueryBuilder::new()), - _ => Box::new(OpenAIQueryBuilder::new()), // Use OpenAI as default for all other providers + _ => Box::new(OpenAIQueryBuilder::new(provider.kind.clone())), // Pass provider kind for Azure handling } } diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index ddc2cc480f..9d5ac8aa66 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -6,6 +6,7 @@ use ulid; use uuid::Uuid; use windmill_common::{ ai_providers::AIProvider, + ai_providers::AZURE_API_VERSION, cache, client::AuthedClient, db::DB, @@ -526,7 +527,7 @@ pub async fn run_agent( let endpoint = query_builder.get_endpoint(&base_url, args.provider.get_model(), output_type); - let auth_headers = query_builder.get_auth_headers(api_key, output_type); + let auth_headers = query_builder.get_auth_headers(api_key, &base_url, output_type); let mut request = HTTP_CLIENT .post(&endpoint) @@ -538,6 +539,10 @@ pub async fn run_agent( request = request.header(header_name, header_value); } + if args.provider.kind.is_azure_openai(&base_url) { + request = request.query(&[("api-version", AZURE_API_VERSION)]) + } + let resp = request .body(request_body) .send() From 39b2f547799a6d75012ca2595d532e87a3035e85 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 18 Sep 2025 16:02:16 +0200 Subject: [PATCH 09/17] feat(frontend): allow publishing script to hub from list view (#6634) --- .../components/common/table/ScriptRow.svelte | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/components/common/table/ScriptRow.svelte b/frontend/src/lib/components/common/table/ScriptRow.svelte index 6d4e4704d9..15ecddb34a 100644 --- a/frontend/src/lib/components/common/table/ScriptRow.svelte +++ b/frontend/src/lib/components/common/table/ScriptRow.svelte @@ -8,7 +8,7 @@ import type ShareModal from '$lib/components/ShareModal.svelte' import { ScriptService, type Script, DraftService } from '$lib/gen' - import { userStore, workspaceStore } from '$lib/stores' + import { hubBaseUrlStore, userStore, workspaceStore } from '$lib/stores' import { createEventDispatcher } from 'svelte' import Badge from '../badge/Badge.svelte' @@ -34,13 +34,15 @@ Pen, Share, Trash, - History + History, + Globe2 } from 'lucide-svelte' import ScriptVersionHistory from '$lib/components/ScriptVersionHistory.svelte' import { Drawer, DrawerContent } from '..' import NoMainFuncBadge from '$lib/components/NoMainFuncBadge.svelte' import Tooltip from '$lib/components/Tooltip.svelte' import { getDeployUiSettings } from '$lib/components/home/deploy_ui' + import { scriptToHubUrl } from '$lib/hub' interface Props { script: Script & { canWrite: boolean; use_codebase: boolean } @@ -300,6 +302,25 @@ copyToClipboard(script.path) } }, + { + displayName: 'Publish to Hub', + icon: Globe2, + action: () => { + window.open( + scriptToHubUrl( + script.content, + script.summary, + script.description ?? '', + script.kind, + script.language, + script.schema, + script.lock ?? '', + $hubBaseUrlStore + ).toString(), + '_blank' + ) + } + }, { displayName: script.archived ? 'Unarchive' : 'Archive', icon: Archive, From 4586355791cfd5c2a3d3f85a2836914d4e200c6a Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Thu, 18 Sep 2025 17:24:49 +0200 Subject: [PATCH 10/17] Better flow builder (#6596) * fly animation * output picker fly transition * svelte 5 migrate * createCache * cache integrations * fly transition for triggers add btn * better hub scripts fetching (code style + caching + fewer states) * loadItemsCached in WorkspaceScriptPickerQuick * usePromise fixes * createCache initial keys, no flicker at all * Prettier template editor * fix null access * fix bad z-index issue * Revert "Prettier template editor" This reverts commit ff2a9526568aab27f24a6e87f689057e0bc3dd68. * type error * ee repo ref * Update ee-repo-ref.txt * Fix formatting of ee-repo-ref.txt --------- Co-authored-by: Ruben Fiszel --- backend/ee-repo-ref.txt | 2 +- frontend/src/lib/components/DropdownV2.svelte | 10 +- .../lib/components/EditableSchemaForm.svelte | 10 +- .../src/lib/components/TemplateEditor.svelte | 6 +- .../components/common/popup/PopupV2.svelte | 2 + .../copilot/FlowCopilotInputsModal.svelte | 49 +++++----- .../flows/map/InsertModuleInner.svelte | 16 +--- .../flows/pickers/PickHubScriptQuick.svelte | 94 +++++++++---------- .../pickers/WorkspaceScriptPickerQuick.svelte | 41 +++++--- .../flows/propPicker/OutputPicker.svelte | 2 + .../components/meltComponents/Popover.svelte | 5 +- .../triggers/AddTriggersButton.svelte | 1 + frontend/src/lib/utils.ts | 29 ++++++ 13 files changed, 150 insertions(+), 117 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 20dc3db333..7d8849c19a 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -daa1c26c268c778e55756f02a459b6c7628c9267 \ No newline at end of file +daa1c26c268c778e55756f02a459b6c7628c9267 diff --git a/frontend/src/lib/components/DropdownV2.svelte b/frontend/src/lib/components/DropdownV2.svelte index 9415635bab..34404779ad 100644 --- a/frontend/src/lib/components/DropdownV2.svelte +++ b/frontend/src/lib/components/DropdownV2.svelte @@ -18,6 +18,7 @@ import { twMerge } from 'tailwind-merge' import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' import { untrack } from 'svelte' + import { fly } from 'svelte/transition' interface Props { aiId?: string | undefined @@ -33,6 +34,7 @@ customWidth?: number | undefined customMenu?: boolean class?: string | undefined + enableFlyTransition?: boolean buttonReplacement?: import('svelte').Snippet menu?: import('svelte').Snippet } @@ -51,6 +53,7 @@ customWidth = undefined, customMenu = false, class: classNames = undefined, + enableFlyTransition = false, buttonReplacement, menu }: Props = $props() @@ -148,7 +151,12 @@ {#if open && !hidePopup} -
+
{#if customMenu} {@render menu?.()} {:else} diff --git a/frontend/src/lib/components/EditableSchemaForm.svelte b/frontend/src/lib/components/EditableSchemaForm.svelte index 297ee73ad0..9c91888960 100644 --- a/frontend/src/lib/components/EditableSchemaForm.svelte +++ b/frontend/src/lib/components/EditableSchemaForm.svelte @@ -186,7 +186,7 @@ let index = 0 let hasChanged = false for (let k of properties) { - if (schema.properties[k].type === 'object' && schema.properties[k].properties) { + if (schema.properties[k]?.type === 'object' && schema.properties[k].properties) { hasChanged = hasChanged || alignOrderWithProperties(schema.properties[k]) } if (!norder.includes(k)) { @@ -220,7 +220,7 @@ function computeSelected(property: any) { if (!opened) return '' - if (property.type !== 'object') return property.type + if (property?.type !== 'object') return property?.type if (property.format === 'resource-s3_object') return 'S3' if (property.format?.startsWith('dynselect-')) return 'dynselect' if (property.format?.startsWith('dynmultiselect-')) return 'dynmultiselect' @@ -344,7 +344,7 @@ Object.entries(schema?.properties ?? {}) .filter(([_, property]) => { const props = property as any - return props.type === 'object' && DynamicInput.isDynInputFormat(props.format) + return props?.type === 'object' && DynamicInput.isDynInputFormat(props.format) }) .map(([fieldName, _]) => fieldName.replace(/\s+/g, '_')) ) @@ -652,7 +652,7 @@ {#if typeof args == 'object' && schema?.properties[argName]} { try { if (BROWSER) { diff --git a/frontend/src/lib/components/common/popup/PopupV2.svelte b/frontend/src/lib/components/common/popup/PopupV2.svelte index 5523b3fc73..bbd68fe47b 100644 --- a/frontend/src/lib/components/common/popup/PopupV2.svelte +++ b/frontend/src/lib/components/common/popup/PopupV2.svelte @@ -2,6 +2,7 @@ import Portal from '$lib/components/Portal.svelte' import { clickOutside } from '$lib/utils' import { createFloatingActions, type ComputeConfig } from 'svelte-floating-ui' + import { fly } from 'svelte/transition' interface Props { floatingConfig?: ComputeConfig @@ -58,6 +59,7 @@ class="border rounded-lg shadow-lg bg-surface z5000" style="position:absolute" use:floatingContent + transition:fly={{ duration: 100, y: -16 }} >
- { - open = false - dispatch('confirmed') - }} - on:canceled - title="Windmill AI wants to add the following inputs to the flow:" -> -
    - {#each inputs as input} -
  • {input}
  • - {/each} -
- - -
+
    + {#each inputs as input} +
  • {input}
  • + {/each} +
+ + + + diff --git a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte index c2e887063a..7c531a8d87 100644 --- a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte +++ b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte @@ -6,8 +6,6 @@ import ToggleHubWorkspaceQuick from '$lib/components/ToggleHubWorkspaceQuick.svelte' import TopLevelNode from '../pickers/TopLevelNode.svelte' - // import type { Writable } from 'svelte/store' - const dispatch = createEventDispatcher() interface Props { stop?: boolean @@ -32,26 +30,14 @@ $state(kind) let preFilter: 'all' | 'workspace' | 'hub' = $state('all') let loading = $state(false) - let small = $state(false) + let small = $derived(kind === 'preprocessor' || kind === 'failure') let width = $state(0) let height = $state(0) let displayPath = $derived(width > 650 || height > 400) - - $effect(() => { - small = kind === 'preprocessor' || kind === 'failure' - }) - - - -
+ let listHubIntegrationsCached = createCache( + (params: { kind: HubScriptKind & string }) => IntegrationService.listHubIntegrations(params), + { initial: { kind: 'script' } } + ) + let listHubScriptsCached = createCache( + async ({ + filter, + kind, + appFilter + }: { + filter: string + kind: HubScriptKind & string + appFilter: string | undefined + }) => + filter.length > 0 + ? await ScriptService.queryHubScripts({ text: filter, limit: 40, kind }) + : ((await ScriptService.getTopHubScripts({ limit: 40, kind, app: appFilter })).asks ?? []), + { initial: { filter: '', kind: 'script', appFilter: undefined } } + ) + + + From de922624497e50c4a8b2af6491e9b14f8dca88c0 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 18 Sep 2025 19:36:07 +0200 Subject: [PATCH 16/17] chore(main): release 1.544.1 (#6636) * chore(main): release 1.544.1 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 7 +++ backend/Cargo.lock | 56 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 52 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf65b0b98a..6320014cf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.544.1](https://github.com/windmill-labs/windmill/compare/v1.544.0...v1.544.1) (2025-09-18) + + +### Bug Fixes + +* fix onLoad auth issue ([bb4699b](https://github.com/windmill-labs/windmill/commit/bb4699bdc6d4e458175594db92b34fdf12fadf1d)) + ## [1.544.0](https://github.com/windmill-labs/windmill/compare/v1.543.0...v1.544.0) (2025-09-18) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 6050651dfc..3e064ffbd5 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15128,7 +15128,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "aws-sdk-config", @@ -15188,7 +15188,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "argon2", @@ -15308,7 +15308,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.544.0" +version = "1.544.1" dependencies = [ "base64 0.22.1", "chrono", @@ -15323,7 +15323,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.544.0" +version = "1.544.1" dependencies = [ "chrono", "serde", @@ -15336,7 +15336,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "axum", @@ -15355,7 +15355,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "async-recursion", @@ -15439,7 +15439,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.544.0" +version = "1.544.1" dependencies = [ "regex", "serde", @@ -15454,7 +15454,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "bytes", @@ -15478,7 +15478,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.544.0" +version = "1.544.1" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15490,7 +15490,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.544.0" +version = "1.544.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -15499,7 +15499,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "lazy_static", @@ -15511,7 +15511,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "serde_json", @@ -15523,7 +15523,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "gosyn", @@ -15535,7 +15535,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "lazy_static", @@ -15547,7 +15547,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "serde_json", @@ -15559,7 +15559,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "nu-parser", @@ -15570,7 +15570,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15581,7 +15581,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15593,7 +15593,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "async-recursion", @@ -15616,7 +15616,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "lazy_static", @@ -15630,7 +15630,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15647,7 +15647,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "lazy_static", @@ -15661,7 +15661,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "lazy_static", @@ -15679,7 +15679,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "getrandom 0.2.16", @@ -15704,7 +15704,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "serde_json", @@ -15714,7 +15714,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "async-recursion", @@ -15747,7 +15747,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.544.0" +version = "1.544.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15757,7 +15757,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.544.0" +version = "1.544.1" dependencies = [ "anyhow", "async-once-cell", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index d9f468da8e..290c3389b7 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.544.0" +version = "1.544.1" authors.workspace = true edition.workspace = true @@ -34,7 +34,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.544.0" +version = "1.544.1" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 3dc1c2af91..11dcb6c220 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.544.0 + version: 1.544.1 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 1798bf6914..6fc84ad172 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.544.0"; +export const VERSION = "v1.544.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index 9a83d1f69b..fc3e7f57e4 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { // } // }); -export const VERSION = "1.544.0"; +export const VERSION = "1.544.1"; export const WM_FORK_PREFIX = "wm-fork"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 1a19178aea..564fc8780f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.544.0", + "version": "1.544.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.544.0", + "version": "1.544.1", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 7a9cbe7606..d0ac4dde1a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.544.0", + "version": "1.544.1", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 54b657727a..b40e89781e 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.544.0" -wmill_pg = ">=1.544.0" +wmill = ">=1.544.1" +wmill_pg = ">=1.544.1" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 26368cda31..b202f0bba6 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.544.0 + version: 1.544.1 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 3adfa24ef3..acaa735084 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.544.0' + ModuleVersion = '1.544.1' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 02ecc17a75..137afcb36b 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.544.0" +version = "1.544.1" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 7ef4817ccf..e705ab949d 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.544.0" +version = "1.544.1" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 2ab6161147..9c3a5c3f6c 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.544.0", + "version": "1.544.1", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index d11f8f880f..a1bc3203c7 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.544.0", + "version": "1.544.1", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 488e553d9a..405633a255 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.544.0 +1.544.1 From 74a8d8a6f7cc7e16f23d6f7999e51e9f2f6581d4 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 18 Sep 2025 18:06:35 +0000 Subject: [PATCH 17/17] fix: improve flowtimeline --- .../components/FlowStatusViewerInner.svelte | 39 ++++++++++--------- .../src/lib/components/FlowTimeline.svelte | 11 +++++- frontend/src/lib/timelineCompute.svelte.ts | 6 +++ 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index ceda18153a..8480647d03 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -1538,25 +1538,26 @@
Empty flow
{/if}
-
- { - loadPreviousIters(key, amount) - }} - /> -
- {#if selected == 'assets' && render} + {#if selected == 'logs' && render} +
+ { + loadPreviousIters(key, amount) + }} + /> +
+ {:else if selected == 'assets' && render}
{ + flowDone + untrack(() => { + timelineCompute?.setFlowDone(flowDone) + }) + })
{min ? displayDate(new Date(min), true) : ''}
{#if max && min}{msToSec(max - min, 1)}s
{/if}
{max ? displayDate(new Date(max), true) : ''}{#if !max && min}{#if now} diff --git a/frontend/src/lib/timelineCompute.svelte.ts b/frontend/src/lib/timelineCompute.svelte.ts index 1708968bf2..ba01b8e7f3 100644 --- a/frontend/src/lib/timelineCompute.svelte.ts +++ b/frontend/src/lib/timelineCompute.svelte.ts @@ -54,6 +54,7 @@ export class TimelineCompute { reset() { this.min = undefined this.max = undefined + this.#flowDone = false this.items = this.computeItems(this.#durationStatuses) } @@ -67,6 +68,10 @@ export class TimelineCompute { this.#flowDone = flowDone } + setFlowDone(flowDone: boolean) { + this.#flowDone = flowDone + } + destroy() { if (this.#interval) { clearInterval(this.#interval) @@ -129,6 +134,7 @@ export class TimelineCompute { }) this.items = nitems this.min = nmin + this.max = isStillRunning || (cnt < this.#flowModules.length && !this.#flowDone) ? undefined : nmax if (this.max && this.min) {