diff --git a/backend/Cargo.lock b/backend/Cargo.lock index b842b0fd38..d9f350b70a 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16152,6 +16152,7 @@ dependencies = [ "windmill-parser-py", "windmill-parser-sql", "windmill-parser-ts", + "windmill-types", ] [[package]] @@ -16914,6 +16915,23 @@ dependencies = [ "windmill-trigger", ] +[[package]] +name = "windmill-types" +version = "1.628.3" +dependencies = [ + "anyhow", + "chrono", + "hex", + "itertools 0.14.0", + "rand 0.9.0", + "serde", + "serde_json", + "sqlx", + "strum 0.27.2", + "tracing", + "uuid", +] + [[package]] name = "windmill-worker" version = "1.628.3" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 72a4a05d67..0d44987be3 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -31,6 +31,7 @@ members = [ "./windmill-queue", "./windmill-worker", "./windmill-dep-map", + "./windmill-types", "./windmill-common", "./windmill-jseval", "./windmill-runtime-nativets", @@ -234,6 +235,7 @@ windmill-api = { path = "./windmill-api", default-features = false } windmill-queue = { path = "./windmill-queue" } windmill-worker = { path = "./windmill-worker" } windmill-dep-map = { path = "./windmill-dep-map" } +windmill-types = { path = "./windmill-types" } windmill-common = { path = "./windmill-common", default-features = false } windmill-audit = { path = "./windmill-audit" } windmill-git-sync = { path = "./windmill-git-sync" } diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 3462788bcf..46c6be19a7 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -134,12 +134,10 @@ impl ScriptWDraft { db: &DB, ) -> error::Result> { let (debouncing_settings, concurrency_settings) = - RunnableSettings::from_runnable_settings_handle( + windmill_common::runnable_settings::prefetch_cached_from_handle( self.runnable_settings.runnable_settings_handle, db, ) - .await? - .prefetch_cached(db) .await?; Ok(ScriptWDraft { @@ -851,11 +849,10 @@ async fn create_script_internal<'c>( } }; - let runnable_settings_handle = RunnableSettings { + let runnable_settings_handle = windmill_common::runnable_settings::insert_rs(RunnableSettings { debouncing_settings: ns.debouncing_settings.insert_cached(&db).await?, concurrency_settings: ns.concurrency_settings.insert_cached(&db).await?, - } - .insert_cached(&db) + }, &db) .await?; let ( @@ -1294,9 +1291,11 @@ async fn get_script_by_path( }; tx.commit().await?; - let script = not_found_if_none(script_o, "Script", path)? - .prefetch_cached(&db) - .await?; + let script = windmill_common::scripts::prefetch_cached_script_with_starred( + not_found_if_none(script_o, "Script", path)?, + &db, + ) + .await?; Ok(Json(script)) } @@ -1830,7 +1829,7 @@ async fn get_script_by_hash( tx.commit().await?; - Ok(Json(r.prefetch_cached(&db).await?)) + Ok(Json(windmill_common::scripts::prefetch_cached_script_with_starred(r, &db).await?)) } async fn raw_script_by_hash( @@ -2009,7 +2008,7 @@ async fn archive_script_by_hash( WebhookMessage::DeleteScript { workspace: w_id, hash: hash.to_string() }, ); - Ok(Json(script.prefetch_cached(&db).await?)) + Ok(Json(windmill_common::scripts::prefetch_cached_script(script, &db).await?)) } async fn delete_script_by_hash( @@ -2053,7 +2052,7 @@ async fn delete_script_by_hash( WebhookMessage::DeleteScript { workspace: w_id, hash: hash.to_string() }, ); - Ok(Json(script.prefetch_cached(&db).await?)) + Ok(Json(windmill_common::scripts::prefetch_cached_script(script, &db).await?)) } #[derive(Deserialize)] diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 7fe674b2ce..6974c5f244 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -38,7 +38,7 @@ use windmill_common::jobs::{ format_completed_job_result, format_result, DynamicInput, ENTRYPOINT_OVERRIDE, }; use windmill_common::runnable_settings::{ - ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, RunnableSettings, + ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, }; #[cfg(feature = "inline_preview")] use windmill_common::runtime_assets::{register_runtime_asset, InsertRuntimeAssetParams}; @@ -3471,9 +3471,7 @@ pub async fn run_workflow_as_code( let JobExtended { inner: job, raw_code, raw_lock, .. } = job; let (_debouncing_settings, concurrency_settings) = - RunnableSettings::from_runnable_settings_handle(job.runnable_settings_handle, &db) - .await? - .prefetch_cached(&db) + windmill_common::runnable_settings::prefetch_cached_from_handle(job.runnable_settings_handle, &db) .await?; let (job_payload, tag, _delete_after_use, timeout, on_behalf_of) = match job.job_kind { @@ -4645,13 +4643,13 @@ fn register_potential_assets_on_inline_execution( let columns = asset.columns.as_ref().map(|cols| { cols.iter() .map(|(col_name, col_access_type)| { - (col_name.clone(), (*col_access_type).into()) + (col_name.clone(), windmill_common::assets::asset_access_type_from_parser(*col_access_type)) }) .collect() }); register_runtime_asset(InsertRuntimeAssetParams { - access_type: asset.access_type.map(|a| a.into()), - asset_kind: asset.kind.into(), + access_type: asset.access_type.map(windmill_common::assets::asset_access_type_from_parser), + asset_kind: windmill_common::assets::asset_kind_from_parser(asset.kind), asset_path: asset.path, columns, job_id, diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 2206efd3ed..2234b4a0f1 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -426,7 +426,7 @@ pub(crate) async fn tarball_workspace( .await?; for script in scripts { - let script = script.prefetch_cached(&db).await?; + let script = windmill_common::scripts::prefetch_cached_script(script, &db).await?; let ext = match script.language { ScriptLang::Python3 => "py", ScriptLang::Deno => { diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 4faa1fefd3..d399c08840 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -98,6 +98,7 @@ backon.workspace = true openidconnect = { workspace = true, optional = true } strum.workspace = true strum_macros.workspace = true +windmill-types.workspace = true url.workspace = true urlencoding.workspace = true async-recursion.workspace = true diff --git a/backend/windmill-common/src/apps.rs b/backend/windmill-common/src/apps.rs index e27504a9f6..50657296c0 100644 --- a/backend/windmill-common/src/apps.rs +++ b/backend/windmill-common/src/apps.rs @@ -6,43 +6,19 @@ * LICENSE-AGPL for a copy of the license. */ -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; -use serde::{Deserialize, Serialize}; use serde_json::{from_value, Value}; use tokio::sync::RwLock; use crate::{error, scripts::ScriptLang}; +pub use windmill_types::apps::*; + lazy_static::lazy_static! { pub static ref APP_WORKSPACED_ROUTE: Arc> = Arc::new(RwLock::new(false)); } -/// Id in the `app_script` table. -#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash, Eq, PartialEq)] -#[serde(transparent)] -pub struct AppScriptId(pub i64); - -#[derive(Deserialize)] -pub struct ListAppQuery { - pub starred_only: Option, - pub path_exact: Option, - pub path_start: Option, - pub include_draft_only: Option, - pub with_deployment_msg: Option, -} - -#[derive(Deserialize)] -pub struct RawAppValue { - pub files: HashMap, -} - -pub struct AppInlineScript { - pub language: Option, - pub content: String, - pub lock: Option, -} - /// Traverse FlowValue while invoking provided by caller callback on leafs // #[async_recursion::async_recursion(?Send)] pub fn traverse_app_inline_scripts< diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index eb4c482c67..669614d64a 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -1,53 +1,8 @@ -use serde::{Deserialize, Serialize}; use sqlx::PgExecutor; -use std::collections::BTreeMap; use crate::{error, scripts::ScriptHash}; -#[derive( - Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type, PartialOrd, Ord, -)] -#[sqlx(type_name = "ASSET_KIND", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum AssetKind { - S3Object, - Resource, - // Avoid unnexpected crashes when deserializing old assets - Variable, // Deprecated - Ducklake, - DataTable, -} - -#[derive( - Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type, PartialOrd, Ord, -)] -#[sqlx(type_name = "ASSET_USAGE_KIND", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum AssetUsageKind { - Script, - Flow, - Job, -} - -#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] -#[sqlx(type_name = "ASSET_ACCESS_TYPE", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum AssetUsageAccessType { - R, - W, - RW, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Hash)] -pub struct AssetWithAltAccessType { - pub path: String, - pub kind: AssetKind, - pub access_type: Option, - pub alt_access_type: Option, - /// Map of column name to access type for column-level access tracking - #[serde(skip_serializing_if = "Option::is_none")] - pub columns: Option>, -} +pub use windmill_types::assets::*; pub async fn insert_static_asset_usage<'e>( executor: impl PgExecutor<'e>, @@ -111,59 +66,21 @@ pub async fn clear_static_asset_usage_by_script_hash<'e>( Ok(()) } -pub fn merge_asset_usage_access_types( - a: Option, - b: Option, -) -> Option { - use AssetUsageAccessType::*; - match (a, b) { - (None, _) | (_, None) => None, - (Some(R), Some(W)) | (Some(W), Some(R)) => Some(RW), - (Some(RW), _) | (_, Some(RW)) => Some(RW), - (Some(R), Some(R)) => Some(R), - (Some(W), Some(W)) => Some(W), +pub fn asset_kind_from_parser(parser_kind: windmill_parser::asset_parser::AssetKind) -> AssetKind { + match parser_kind { + windmill_parser::asset_parser::AssetKind::S3Object => AssetKind::S3Object, + windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource, + windmill_parser::asset_parser::AssetKind::Ducklake => AssetKind::Ducklake, + windmill_parser::asset_parser::AssetKind::DataTable => AssetKind::DataTable, } } -pub fn merge_asset_columns( - a: &Option>, - b: &Option>, -) -> Option> { - match (a, b) { - (None, None) => None, - (Some(cols), None) | (None, Some(cols)) => Some(cols.clone()), - (Some(cols_a), Some(cols_b)) => { - let mut merged = cols_a.clone(); - for (col, access_b) in cols_b { - let access_a = merged.get(col); - let merged_access = - merge_asset_usage_access_types(access_a.cloned(), Some(*access_b)); - if let Some(access) = merged_access { - merged.insert(col.clone(), access); - } - } - Some(merged) - } - } -} - -impl From for AssetKind { - fn from(parser_kind: windmill_parser::asset_parser::AssetKind) -> Self { - match parser_kind { - windmill_parser::asset_parser::AssetKind::S3Object => AssetKind::S3Object, - windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource, - windmill_parser::asset_parser::AssetKind::Ducklake => AssetKind::Ducklake, - windmill_parser::asset_parser::AssetKind::DataTable => AssetKind::DataTable, - } - } -} - -impl From for AssetUsageAccessType { - fn from(parser_kind: windmill_parser::asset_parser::AssetUsageAccessType) -> Self { - match parser_kind { - windmill_parser::asset_parser::AssetUsageAccessType::R => AssetUsageAccessType::R, - windmill_parser::asset_parser::AssetUsageAccessType::W => AssetUsageAccessType::W, - windmill_parser::asset_parser::AssetUsageAccessType::RW => AssetUsageAccessType::RW, - } +pub fn asset_access_type_from_parser( + parser_kind: windmill_parser::asset_parser::AssetUsageAccessType, +) -> AssetUsageAccessType { + match parser_kind { + windmill_parser::asset_parser::AssetUsageAccessType::R => AssetUsageAccessType::R, + windmill_parser::asset_parser::AssetUsageAccessType::W => AssetUsageAccessType::W, + windmill_parser::asset_parser::AssetUsageAccessType::RW => AssetUsageAccessType::RW, } } diff --git a/backend/windmill-common/src/flow_status.rs b/backend/windmill-common/src/flow_status.rs index d1d33f67ef..c9aa706ac9 100644 --- a/backend/windmill-common/src/flow_status.rs +++ b/backend/windmill-common/src/flow_status.rs @@ -1,541 +1 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use std::collections::HashMap; -use std::time::Duration; - -use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -use crate::flows::FlowValue; - -const MINUTES: Duration = Duration::from_secs(60); -const HOURS: Duration = MINUTES.saturating_mul(60); - -pub const MAX_RETRY_ATTEMPTS: u32 = u32::MAX; -pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6); - -pub fn is_retry_default(v: &RetryStatus) -> bool { - v.fail_count == 0 && v.failed_jobs.is_empty() -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct FlowStatus { - pub step: i32, - pub modules: Vec, - pub failure_module: Box, - pub preprocessor_module: Option, - - #[serde(skip_serializing_if = "HashMap::is_empty")] - #[serde(default)] - pub user_states: HashMap, - #[serde(default)] - pub cleanup_module: FlowCleanupModule, - #[serde(default)] - #[serde(skip_serializing_if = "is_retry_default")] - pub retry: RetryStatus, - #[serde(skip_serializing_if = "Option::is_none")] - pub approval_conditions: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub restarted_from: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub stream_job: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub chat_input_enabled: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub memory_id: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -#[serde(default)] -pub struct RetryStatus { - pub fail_count: u32, - pub failed_jobs: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -#[serde(default)] -pub struct ApprovalConditions { - pub user_auth_required: bool, - pub user_groups_required: Vec, - pub self_approval_disabled: bool, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -#[serde(default)] -pub struct RestartedFrom { - pub flow_job_id: Uuid, - pub step_id: String, - pub branch_or_iteration_n: Option, - pub flow_version: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Iterator { - pub index: usize, - #[serde(skip_serializing_if = "Option::is_none")] - pub itered: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub itered_len: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct BranchAllStatus { - pub branch: usize, - pub len: usize, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde( - tag = "type", - rename_all(serialize = "lowercase", deserialize = "lowercase") -)] -pub enum BranchChosen { - Default, - Branch { branch: usize }, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Approval { - pub resume_id: u16, - pub approver: String, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct FlowStatusModuleWParent { - #[serde(skip_serializing_if = "Option::is_none")] - pub parent_module: Option, - #[serde(flatten)] - pub module_status: FlowStatusModule, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -pub struct FlowCleanupModule { - #[serde(default)] - #[serde(skip_serializing_if = "Vec::is_empty")] - pub flow_jobs_to_clean: Vec, -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct FlowJobsDuration { - pub started_at: Vec>>, - pub duration_ms: Vec>, -} - -impl FlowJobsDuration { - pub fn set(&mut self, position: Option, value: &Option) { - if let Some(position) = position { - if position >= self.started_at.len() - || position >= self.duration_ms.len() - || value.is_none() - { - return; - } - let value = value.clone().unwrap(); - self.started_at[position] = Some(value.started_at); - self.duration_ms[position] = Some(value.duration_ms); - } - } - - pub fn push(&mut self, value: &Option) { - self.started_at.push(value.as_ref().map(|x| x.started_at)); - self.duration_ms.push(value.as_ref().map(|x| x.duration_ms)); - } - - pub fn new(n: usize) -> Self { - Self { started_at: vec![None; n], duration_ms: vec![None; n] } - } -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct FlowJobDuration { - pub started_at: chrono::DateTime, - pub duration_ms: i64, -} - -impl FlowJobsDuration { - pub fn truncate(&mut self, n: usize) { - self.started_at.truncate(n); - self.duration_ms.truncate(n); - } -} - -#[derive(Deserialize)] -struct UntaggedFlowStatusModule { - #[serde(rename = "type")] - type_: String, - id: Option, - count: Option, - progress: Option, - job: Option, - iterator: Option, - flow_jobs: Option>, - flow_jobs_success: Option>>, - flow_jobs_duration: Option, - branch_chosen: Option, - branchall: Option, - parallel: Option, - while_loop: Option, - approvers: Option>, - failed_retries: Option>, - skipped: Option, - agent_actions: Option>, - agent_actions_success: Option>, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum AgentAction { - ToolCall { - job_id: uuid::Uuid, - function_name: String, - module_id: String, - }, - McpToolCall { - call_id: uuid::Uuid, - function_name: String, - resource_path: String, - #[serde(skip_serializing_if = "Option::is_none")] - arguments: Option, - }, - Message {}, - WebSearch {}, -} - -#[derive(Serialize, Debug, Clone)] -#[serde(tag = "type")] -pub enum FlowStatusModule { - WaitingForPriorSteps { - id: String, - }, - WaitingForEvents { - id: String, - count: u16, - job: Uuid, - }, - WaitingForExecutor { - id: String, - job: Uuid, - }, - InProgress { - id: String, - job: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - progress: Option, - #[serde(skip_serializing_if = "Option::is_none")] - iterator: Option, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs_success: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs_duration: Option, - #[serde(skip_serializing_if = "Option::is_none")] - branch_chosen: Option, - #[serde(skip_serializing_if = "Option::is_none")] - branchall: Option, - #[serde(skip_serializing_if = "std::ops::Not::not")] - parallel: bool, - #[serde(skip_serializing_if = "std::ops::Not::not")] - while_loop: bool, - #[serde(skip_serializing_if = "Option::is_none")] - agent_actions: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - agent_actions_success: Option>, - }, - Success { - id: String, - job: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs_success: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs_duration: Option, - #[serde(skip_serializing_if = "Option::is_none")] - branch_chosen: Option, - #[serde(default)] - #[serde(skip_serializing_if = "Vec::is_empty")] - approvers: Vec, - #[serde(skip_serializing_if = "Vec::is_empty")] - failed_retries: Vec, - skipped: bool, - #[serde(skip_serializing_if = "Option::is_none")] - agent_actions: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - agent_actions_success: Option>, - }, - Failure { - id: String, - job: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs_success: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs_duration: Option, - #[serde(skip_serializing_if = "Option::is_none")] - branch_chosen: Option, - #[serde(skip_serializing_if = "Vec::is_empty")] - failed_retries: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - agent_actions: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - agent_actions_success: Option>, - }, -} - -impl<'de> Deserialize<'de> for FlowStatusModule { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let untagged: UntaggedFlowStatusModule = - UntaggedFlowStatusModule::deserialize(deserializer)?; - - match untagged.type_.as_str() { - "WaitingForPriorSteps" => Ok(FlowStatusModule::WaitingForPriorSteps { - id: untagged - .id - .ok_or_else(|| serde::de::Error::missing_field("id"))?, - }), - "WaitingForEvents" => Ok(FlowStatusModule::WaitingForEvents { - id: untagged - .id - .ok_or_else(|| serde::de::Error::missing_field("id"))?, - count: untagged - .count - .ok_or_else(|| serde::de::Error::missing_field("count"))?, - job: untagged - .job - .ok_or_else(|| serde::de::Error::missing_field("job"))?, - }), - "WaitingForExecutor" => Ok(FlowStatusModule::WaitingForExecutor { - id: untagged - .id - .ok_or_else(|| serde::de::Error::missing_field("id"))?, - job: untagged - .job - .ok_or_else(|| serde::de::Error::missing_field("job"))?, - }), - "InProgress" => Ok(FlowStatusModule::InProgress { - id: untagged - .id - .ok_or_else(|| serde::de::Error::missing_field("id"))?, - job: untagged - .job - .ok_or_else(|| serde::de::Error::missing_field("job"))?, - iterator: untagged.iterator, - flow_jobs: untagged.flow_jobs, - flow_jobs_success: untagged.flow_jobs_success, - flow_jobs_duration: untagged.flow_jobs_duration, - branch_chosen: untagged.branch_chosen, - branchall: untagged.branchall, - parallel: untagged.parallel.unwrap_or(false), - while_loop: untagged.while_loop.unwrap_or(false), - progress: untagged.progress, - agent_actions: untagged.agent_actions, - agent_actions_success: untagged.agent_actions_success, - }), - "Success" => Ok(FlowStatusModule::Success { - id: untagged - .id - .ok_or_else(|| serde::de::Error::missing_field("id"))?, - job: untagged - .job - .ok_or_else(|| serde::de::Error::missing_field("job"))?, - flow_jobs: untagged.flow_jobs, - flow_jobs_success: untagged.flow_jobs_success, - flow_jobs_duration: untagged.flow_jobs_duration, - branch_chosen: untagged.branch_chosen, - approvers: untagged.approvers.unwrap_or_default(), - failed_retries: untagged.failed_retries.unwrap_or_default(), - skipped: untagged.skipped.unwrap_or(false), - agent_actions: untagged.agent_actions, - agent_actions_success: untagged.agent_actions_success, - }), - "Failure" => Ok(FlowStatusModule::Failure { - id: untagged - .id - .ok_or_else(|| serde::de::Error::missing_field("id"))?, - job: untagged - .job - .ok_or_else(|| serde::de::Error::missing_field("job"))?, - flow_jobs: untagged.flow_jobs, - flow_jobs_success: untagged.flow_jobs_success, - flow_jobs_duration: untagged.flow_jobs_duration, - branch_chosen: untagged.branch_chosen, - failed_retries: untagged.failed_retries.unwrap_or_default(), - agent_actions: untagged.agent_actions, - agent_actions_success: untagged.agent_actions_success, - }), - other => Err(serde::de::Error::unknown_variant( - other, - &[ - "WaitingForPriorSteps", - "WaitingForEvents", - "WaitingForExecutor", - "InProgress", - "Success", - "Failure", - ], - )), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum JobResult { - SingleJob(Uuid), - ListJob(Vec), -} - -impl FlowStatusModule { - pub fn job(&self) -> Option { - match self { - FlowStatusModule::WaitingForPriorSteps { .. } => None, - FlowStatusModule::WaitingForEvents { job, .. } => Some(*job), - FlowStatusModule::WaitingForExecutor { job, .. } => Some(*job), - FlowStatusModule::InProgress { job, .. } => Some(*job), - FlowStatusModule::Success { job, .. } => Some(*job), - FlowStatusModule::Failure { job, .. } => Some(*job), - } - } - - pub fn flow_jobs(&self) -> Option> { - match self { - FlowStatusModule::InProgress { flow_jobs, .. } => flow_jobs.clone(), - FlowStatusModule::Success { flow_jobs, .. } => flow_jobs.clone(), - FlowStatusModule::Failure { flow_jobs, .. } => flow_jobs.clone(), - _ => None, - } - } - - pub fn branch_chosen(&self) -> Option { - match self { - FlowStatusModule::InProgress { branch_chosen, .. } => branch_chosen.clone(), - FlowStatusModule::Success { branch_chosen, .. } => branch_chosen.clone(), - FlowStatusModule::Failure { branch_chosen, .. } => branch_chosen.clone(), - _ => None, - } - } - - pub fn flow_jobs_success(&self) -> Option>> { - match self { - FlowStatusModule::InProgress { flow_jobs_success, .. } => flow_jobs_success.clone(), - FlowStatusModule::Success { flow_jobs_success, .. } => flow_jobs_success.clone(), - FlowStatusModule::Failure { flow_jobs_success, .. } => flow_jobs_success.clone(), - _ => None, - } - } - - pub fn flow_jobs_duration(&self) -> Option { - match self { - FlowStatusModule::InProgress { flow_jobs_duration, .. } => flow_jobs_duration.clone(), - FlowStatusModule::Success { flow_jobs_duration, .. } => flow_jobs_duration.clone(), - FlowStatusModule::Failure { flow_jobs_duration, .. } => flow_jobs_duration.clone(), - _ => None, - } - } - - pub fn job_result(&self) -> Option { - self.flow_jobs() - .map(JobResult::ListJob) - .or_else(|| self.job().map(JobResult::SingleJob)) - } - - pub fn id(&self) -> String { - match self { - FlowStatusModule::WaitingForPriorSteps { id, .. } => id.clone(), - FlowStatusModule::WaitingForEvents { id, .. } => id.clone(), - FlowStatusModule::WaitingForExecutor { id, .. } => id.clone(), - FlowStatusModule::InProgress { id, .. } => id.clone(), - FlowStatusModule::Success { id, .. } => id.clone(), - FlowStatusModule::Failure { id, .. } => id.clone(), - } - } - - pub fn is_failure(&self) -> bool { - match self { - FlowStatusModule::Failure { .. } => true, - _ => false, - } - } - - pub fn agent_actions(&self) -> Option> { - match self { - FlowStatusModule::InProgress { agent_actions, .. } => agent_actions.clone(), - FlowStatusModule::Success { agent_actions, .. } => agent_actions.clone(), - FlowStatusModule::Failure { agent_actions, .. } => agent_actions.clone(), - _ => None, - } - } - - pub fn agent_actions_success(&self) -> Option> { - match self { - FlowStatusModule::InProgress { agent_actions_success, .. } => { - agent_actions_success.clone() - } - FlowStatusModule::Success { agent_actions_success, .. } => { - agent_actions_success.clone() - } - FlowStatusModule::Failure { agent_actions_success, .. } => { - agent_actions_success.clone() - } - _ => None, - } - } -} - -impl FlowStatus { - pub fn new(f: &FlowValue) -> Self { - Self { - step: if f.preprocessor_module.is_some() { - -1 - } else { - 0 - }, - approval_conditions: None, - modules: f - .modules - .iter() - .map(|m| FlowStatusModule::WaitingForPriorSteps { id: m.id.clone() }) - .collect(), - failure_module: Box::new(FlowStatusModuleWParent { - parent_module: None, - module_status: FlowStatusModule::WaitingForPriorSteps { - id: f - .failure_module - .as_ref() - .map(|x| x.id.clone()) - .unwrap_or_else(|| "failure".to_string()), - }, - }), - preprocessor_module: if f.preprocessor_module.is_some() { - Some(FlowStatusModule::WaitingForPriorSteps { - id: f.preprocessor_module.as_ref().unwrap().id.clone(), - }) - } else { - None - }, - cleanup_module: FlowCleanupModule { flow_jobs_to_clean: vec![] }, - retry: RetryStatus { fail_count: 0, failed_jobs: vec![] }, - restarted_from: None, - user_states: HashMap::new(), - stream_job: None, - chat_input_enabled: f.chat_input_enabled, - memory_id: None, - } - } - - /// current module status ... excluding failure_module - pub fn current_step(&self) -> Option<&FlowStatusModule> { - let i = usize::try_from(self.step).ok()?; - self.modules.get(i) - } -} +pub use windmill_types::flow_status::*; diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 420530ecf3..9c10bf62ce 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -6,1177 +6,18 @@ * LICENSE-AGPL for a copy of the license. */ -use std::{ - collections::{BTreeMap, HashMap}, - time::Duration, - u8, -}; +pub use windmill_types::flows::*; -use anyhow::Context; -use rand::Rng; -use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize, Serializer}; -use serde_json::value::RawValue; use sqlx::types::Json; use sqlx::types::JsonRawValue; use crate::{ - assets::AssetWithAltAccessType, cache, db::DB, - error::{Error, Result as WindmillResult}, - more_serde::{default_empty_string, default_id, default_null, default_true, is_default}, - runnable_settings::{ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings}, - scripts::{Schema, ScriptHash, ScriptLang}, + error::Error, worker::{to_raw_value, Connection}, }; -#[derive(Debug, Serialize, Deserialize, sqlx::FromRow)] -pub struct Flow { - pub workspace_id: String, - pub path: String, - pub summary: String, - pub description: String, - pub value: Json>, - pub edited_by: String, - pub edited_at: chrono::DateTime, - pub archived: bool, - pub schema: Option, - pub extra_perms: serde_json::Value, - #[serde(skip_serializing_if = "Option::is_none")] - pub draft_only: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub dedicated_worker: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub tag: Option, - #[serde(skip_serializing_if = "is_none_or_false")] - pub ws_error_handler_muted: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub timeout: Option, - #[serde(skip_serializing_if = "is_none_or_false")] - pub visible_to_runner_only: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_behalf_of_email: Option, -} - -#[derive(Serialize, sqlx::FromRow)] -pub struct FlowWithStarred { - #[sqlx(flatten)] - #[serde(flatten)] - pub flow: Flow, - #[serde(skip_serializing_if = "Option::is_none")] - pub starred: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub lock_error_logs: Option, - pub version_id: i64, -} - -fn is_none_or_false(b: &Option) -> bool { - b.is_none() || !b.unwrap() -} - -#[derive(Serialize, sqlx::FromRow)] -pub struct ListableFlow { - pub workspace_id: String, - pub path: String, - pub summary: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - pub edited_by: Option, - pub edited_at: Option>, - pub archived: bool, - pub extra_perms: serde_json::Value, - pub starred: bool, - pub has_draft: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub draft_only: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ws_error_handler_muted: Option, - #[sqlx(default)] - #[serde(skip_serializing_if = "Option::is_none")] - pub deployment_msg: Option, -} - -fn validate_retry(retry: &Retry, module_id: &str) -> WindmillResult<()> { - if retry.exponential.attempts > 0 && retry.exponential.seconds == 0 { - return Err(Error::BadRequest(format!( - "Module '{}': Exponential backoff base (seconds) must be greater than 0. A base of 0 would cause immediate retries.", - module_id - ))); - } - Ok(()) -} - -fn validate_flow_value<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let raw_value = Box::::deserialize(deserializer)?; - - let flow_value: FlowValue = serde_json::from_str(raw_value.get()) - .map_err(|e| serde::de::Error::custom(format!("Invalid flow value: {}", e)))?; - - FlowModule::traverse_modules(&flow_value.modules, &mut |module| { - if let Some(ref retry) = module.retry { - validate_retry(retry, &module.id)?; - } - return Ok(()); - }) - .map_err(|e| serde::de::Error::custom(e.to_string()))?; - - if let Some(ref _failure_module) = flow_value.failure_module { - //add validation logic here for failure module - } - - if let Some(ref _preprocessor_module) = flow_value.preprocessor_module { - //add validation logic here for preprocessor module - } - - Ok(raw_value) -} - -#[derive(Debug, Serialize, Deserialize, sqlx::FromRow)] -pub struct NewFlow { - pub path: String, - pub summary: String, - pub description: Option, - #[serde(deserialize_with = "validate_flow_value")] - pub value: Box, - pub schema: Option, - pub draft_only: Option, - pub tag: Option, - pub dedicated_worker: Option, - pub timeout: Option, - pub deployment_message: Option, - pub visible_to_runner_only: Option, - pub on_behalf_of_email: Option, - pub ws_error_handler_muted: Option, -} - -impl NewFlow { - pub fn parse_flow_value(&self) -> crate::error::Result { - serde_json::from_str(self.value.get()).map_err(|e| { - crate::error::Error::InternalErr(format!("Failed to parse flow value: {}", e)) - }) - } -} - -#[derive(Deserialize, Serialize, Debug, Clone, Default)] -pub struct FlowValue { - pub modules: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(default)] - pub failure_module: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(default)] - pub preprocessor_module: Option>, - #[serde(default)] - #[serde(skip_serializing_if = "is_default")] - pub same_worker: bool, - #[serde(flatten)] - pub concurrency_settings: ConcurrencySettings, - #[serde(flatten)] - pub debouncing_settings: DebouncingSettings, - #[serde(skip_serializing_if = "Option::is_none")] - pub skip_expr: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ttl: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ignore_s3_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub early_return: Option, - #[serde(skip_serializing_if = "Option::is_none")] - // Priority at the flow level - pub priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub chat_input_enabled: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub flow_env: Option>>, -} - -impl FlowValue { - pub fn get_flow_module_at_step(&self, step: Step) -> anyhow::Result<&FlowModule> { - let flow_module = match step { - Step::PreprocessorStep => self - .preprocessor_module - .as_deref() - .with_context(|| format!("no preprocessor module")), - Step::Step { idx, .. } => self - .modules - .get(idx) - .with_context(|| format!("no module found at index: {idx}")), - Step::FailureStep => self - .failure_module - .as_deref() - .with_context(|| format!("no failure module")), - }; - - flow_module - } - - /// Traverse FlowValue while invoking provided by caller callback on leafs - // #[async_recursion::async_recursion(?Send)] - // TODO: We may be want this async. - pub fn traverse_leafs crate::error::Result<()>>( - modules: Vec<&FlowModule>, - cb: &mut C, - ) -> crate::error::Result<()> { - use FlowModuleValue::*; - for module in modules { - match serde_json::from_str::(module.value.get())? { - s @ (Script { .. } - | RawScript { .. } - | Flow { .. } - | FlowScript { .. } - | Identity) => cb(&s, &module.id)?, - ForloopFlow { modules, .. } | WhileloopFlow { modules, .. } => { - Self::traverse_leafs(modules.iter().collect(), cb)? - } - AIAgent { tools, .. } => { - for tool in tools { - match &tool.value { - ToolValue::FlowModule(module_value) => cb(module_value, &tool.id)?, - ToolValue::Mcp(_) => { - // MCP tools don't have a FlowModuleValue to traverse - } - ToolValue::Websearch(_) => { - // Websearch tools don't have a FlowModuleValue to traverse - } - } - } - } - BranchOne { default, branches, .. } => { - Self::traverse_leafs(default.iter().collect(), cb)?; - for branch in branches { - Self::traverse_leafs(branch.modules.iter().collect(), cb)?; - } - } - BranchAll { branches, .. } => { - for branch in branches { - Self::traverse_leafs(branch.modules.iter().collect(), cb)?; - } - } - } - } - Ok(()) - } -} - -#[derive(Debug, Copy, Clone)] -pub enum Step { - Step { idx: usize, len: usize }, - PreprocessorStep, - FailureStep, -} - -impl Step { - pub fn from_i32_and_len(step: i32, len: usize) -> Self { - if step < 0 { - Step::PreprocessorStep - } else if (step as usize) < len { - Step::Step { idx: step as usize, len } - } else { - Step::FailureStep - } - } - - pub fn get_step_index(&self) -> Option { - match self { - Step::Step { idx, .. } => Some(*idx), - _ => None, - } - } - - pub fn is_index_step(&self) -> bool { - matches!(self, Step::Step { .. }) - } - - pub fn is_preprocessor_step(&self) -> bool { - matches!(self, Step::PreprocessorStep) - } - - pub fn is_failure_step(&self) -> bool { - matches!(self, Step::FailureStep) - } - - pub fn is_last_step(&self) -> bool { - matches!(self, Step::Step { idx, len } if *idx == len - 1) - } -} - -#[derive(Default, Deserialize, Serialize, Debug, Clone)] -pub struct StopAfterIf { - pub expr: String, - pub skip_if_stopped: bool, - pub error_message: Option, -} - -#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] -pub struct RetryIf { - pub expr: String, -} - -#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] -#[serde(default)] -pub struct Retry { - pub constant: ConstantDelay, - pub exponential: ExponentialDelay, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry_if: Option, -} - -impl Retry { - /// Takes the number of previous retries and returns the interval until the next retry if any. - /// - /// May return [`Duration::ZERO`] to retry immediately. - pub fn interval(&self, previous_attempts: u32, silent: bool) -> Option { - let Self { constant, exponential, .. } = self; - - if previous_attempts < constant.attempts { - Some(Duration::from_secs(constant.seconds as u64)) - } else if previous_attempts - constant.attempts < exponential.attempts { - let exp = previous_attempts.saturating_add(1) as u32; - let mut secs = exponential.multiplier * exponential.seconds.saturating_pow(exp); - if let Some(random_factor) = exponential.random_factor { - if random_factor > 0 { - let random_component = - rand::rng().random_range(0..(std::cmp::min(random_factor, 100) as u16)); - secs = match rand::rng().random_bool(1.0 / 2.0) { - true => secs.saturating_add(secs * random_component / 100), - false => secs.saturating_sub(secs * random_component / 100), - }; - } - } - if !silent { - tracing::warn!("Rescheduling job in {} seconds due to failure", secs); - } - Some(Duration::from_secs(secs as u64)) - } else { - None - } - } - - pub fn has_attempts(&self) -> bool { - self.constant.attempts != 0 || self.exponential.attempts != 0 - } - - pub fn max_attempts(&self) -> u32 { - self.constant - .attempts - .saturating_add(self.exponential.attempts) - } - - pub fn max_interval(&self) -> Option { - self.max_attempts() - .checked_sub(1) - .and_then(|p| self.interval(p, true)) - } -} - -#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] -#[serde(default)] -pub struct ConstantDelay { - pub attempts: u32, - pub seconds: u16, -} - -/// multiplier * seconds ^ failures (+/- jitter of the previous value, if any) -#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] -#[serde(default)] -pub struct ExponentialDelay { - pub attempts: u32, - pub multiplier: u16, - pub seconds: u16, - pub random_factor: Option, // percentage, defaults to 0 for no jitter -} - -impl Default for ExponentialDelay { - fn default() -> Self { - Self { attempts: 0, multiplier: 1, seconds: 0, random_factor: None } - } -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct Suspend { - #[serde(skip_serializing_if = "Option::is_none")] - pub required_events: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub timeout: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub resume_form: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub user_auth_required: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub user_groups_required: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub self_approval_disabled: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub hide_cancel: Option, - #[serde(skip_serializing_if = "false_or_empty")] - pub continue_on_disapprove_timeout: Option, -} - -fn false_or_empty(v: &Option) -> bool { - v.is_none() || v.as_ref().is_some_and(|x| !x) -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct Mock { - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub return_value: Option, -} - -#[derive(Deserialize, Serialize, Debug, Clone, Default)] -pub struct FlowModule { - #[serde(default = "default_id")] - pub id: String, - pub value: Box, - #[serde(skip_serializing_if = "Option::is_none")] - pub stop_after_if: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub stop_after_all_iters_if: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub summary: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub suspend: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub mock: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub sleep: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ttl: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ignore_s3_path: Option, - #[serde( - default, - deserialize_with = "raw_value_to_input_transform::<_, i32>", - skip_serializing_if = "Option::is_none" - )] - pub timeout: Option, - #[serde(skip_serializing_if = "Option::is_none")] - // Priority at the flow step level - pub priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub delete_after_use: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub continue_on_error: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub skip_if: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub apply_preprocessor: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub pass_flow_input_directly: Option, -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct SkipIf { - pub expr: String, -} - -#[derive(Deserialize)] -pub struct FlowModuleValueWithParallel { - #[serde(rename = "type")] - pub type_: String, - pub parallel: Option, - #[serde( - default, - deserialize_with = "raw_value_to_input_transform::<_, u16>", - skip_serializing_if = "Option::is_none" - )] - pub parallelism: Option, -} - -#[derive(Deserialize)] -pub struct FlowModuleValueWithSkipFailures { - pub skip_failures: Option, - pub parallel: Option, - #[serde( - default, - deserialize_with = "raw_value_to_input_transform::<_, u16>", - skip_serializing_if = "Option::is_none" - )] - pub parallelism: Option, -} - -#[derive(Deserialize)] -pub struct BranchWithSkipFailures { - pub skip_failure: Option, -} - -#[derive(Deserialize)] -pub struct FlowModuleWithBranches { - pub branches: Vec, -} - -impl FlowModule { - pub fn id_append(&mut self, s: &str) { - self.id = format!("{}-{}", self.id, s); - } - pub fn get_value(&self) -> anyhow::Result { - serde_json::from_str::(self.value.get()).map_err(crate::error::to_anyhow) - } - - pub fn get_value_with_skip_failures(&self) -> anyhow::Result { - serde_json::from_str::(self.value.get()) - .map_err(crate::error::to_anyhow) - } - - pub fn get_branches_skip_failures(&self) -> anyhow::Result { - serde_json::from_str::(self.value.get()) - .map_err(crate::error::to_anyhow) - } - - pub fn is_flow(&self) -> bool { - self.get_type().is_ok_and(|x| x == "flow") - } - - pub fn get_value_with_parallel(&self) -> anyhow::Result { - serde_json::from_str::(self.value.get()) - .map_err(crate::error::to_anyhow) - } - - pub fn is_ai_agent(&self) -> bool { - self.get_type().is_ok_and(|x| x == "aiagent") - } - - pub fn is_simple(&self) -> bool { - //todo: flow modules could also be simple execpt for the fact that the case of having single parallel flow approval step is not handled well (Create SuspendedTimeout) - self.get_type() - .is_ok_and(|x| x == "script" || x == "rawscript" || x == "flowscript") - } - - pub fn get_type(&self) -> anyhow::Result<&str> { - #[derive(Deserialize)] - pub struct FlowModuleValueType<'a> { - pub r#type: &'a str, - } - - serde_json::from_str::(self.value.get()) - .map_err(crate::error::to_anyhow) - .map(|x| x.r#type) - } - - pub fn traverse_modules crate::error::Result<()>>( - modules: &Vec, - cb: &mut C, - ) -> crate::error::Result<()> { - for module in modules { - cb(module)?; - match module - .get_value() - .map_err(|e| Error::BadRequest(format!("Module '{}': {}", module.id, e)))? - { - FlowModuleValue::ForloopFlow { modules, .. } - | FlowModuleValue::WhileloopFlow { modules, .. } => { - Self::traverse_modules(&modules, cb)?; - } - FlowModuleValue::BranchOne { branches, default, .. } => { - for branch in branches { - Self::traverse_modules(&branch.modules, cb)?; - } - Self::traverse_modules(&default, cb)?; - } - FlowModuleValue::BranchAll { branches, .. } => { - for branch in branches { - Self::traverse_modules(&branch.modules, cb)?; - } - } - FlowModuleValue::AIAgent { tools, .. } => { - for tool in tools { - match &tool.value { - ToolValue::FlowModule(module_value) => match module_value { - FlowModuleValue::ForloopFlow { modules, .. } - | FlowModuleValue::WhileloopFlow { modules, .. } => { - Self::traverse_modules(&modules, cb)?; - } - FlowModuleValue::BranchOne { branches, default, .. } => { - for branch in branches { - Self::traverse_modules(&branch.modules, cb)?; - } - Self::traverse_modules(&default, cb)?; - } - FlowModuleValue::BranchAll { branches, .. } => { - for branch in branches { - Self::traverse_modules(&branch.modules, cb)?; - } - } - _ => {} - }, - ToolValue::Mcp(_) => { - // MCP tools don't have a FlowModule to traverse - } - ToolValue::Websearch(_) => { - // Websearch tools don't have a FlowModule to traverse - } - } - } - } - _ => {} - } - } - Ok(()) - } -} - -#[derive(Deserialize)] -pub struct UntaggedInputTransform { - #[serde(rename = "type")] - pub type_: String, - pub value: Option>, - pub expr: Option, -} - -impl<'de> Deserialize<'de> for InputTransform { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let untagged: UntaggedInputTransform = UntaggedInputTransform::deserialize(deserializer)?; - - let input_transform = TryInto::::try_into(untagged) - .map_err(|e| serde::de::Error::custom(e))?; - - Ok(input_transform) - } -} - -#[derive(Serialize, Debug, Clone)] -#[serde( - tag = "type", - rename_all(serialize = "lowercase", deserialize = "lowercase") -)] -pub enum InputTransform { - Static { - #[serde(default = "default_null")] - value: Box, - }, - Javascript { - #[serde(default = "default_empty_string")] - expr: String, - }, - Ai, -} - -impl InputTransform { - pub fn new_static_value(value: Box) -> InputTransform { - InputTransform::Static { value } - } - - pub fn new_javascript_expr(expr: &str) -> InputTransform { - InputTransform::Javascript { expr: expr.to_owned() } - } -} - -impl TryFrom for InputTransform { - type Error = anyhow::Error; - fn try_from(value: UntaggedInputTransform) -> Result { - let input_transform = match value.type_.as_str() { - "static" => InputTransform::new_static_value(value.value.unwrap_or_else(default_null)), - "javascript" => InputTransform::new_javascript_expr(&value.expr.unwrap_or_default()), - "ai" => InputTransform::Ai, - other => { - return Err(anyhow::anyhow!( - "got value: {other} for field `type`, expected value: `static` or `javascript`" - )) - } - }; - - Ok(input_transform) - } -} - -#[derive(Deserialize)] -#[serde(untagged)] -enum RawValueOrFormatted { - RawValue(T), - Formatted { r#type: String, value: Option, expr: Option }, -} - -fn raw_value_to_input_transform<'de, D, T>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, - T: DeserializeOwned + Serialize, -{ - let val = Option::>::deserialize(deserializer)?; - let input_tranform = match val { - Some(RawValueOrFormatted::RawValue(v)) => { - Some(InputTransform::new_static_value(to_raw_value(&v))) - } - Some(RawValueOrFormatted::Formatted { r#type, expr, value }) => { - let untaged_input_transform = UntaggedInputTransform { - type_: r#type, - expr, - value: value.map(|val| to_raw_value(&val)), - }; - let input_transform = TryInto::::try_into(untaged_input_transform) - .map_err(|e| serde::de::Error::custom(e))?; - Some(input_transform) - } - _ => None, - }; - Ok(input_tranform) -} - -/// Id in the `flow_node` table. -#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash, Eq, PartialEq)] -#[serde(transparent)] -pub struct FlowNodeId(pub i64); - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Branch { - #[serde(skip_serializing_if = "Option::is_none")] - pub summary: Option, - #[serde(default = "default_empty_string")] - pub expr: String, - pub modules: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub modules_node: Option, - #[serde(default = "default_true")] - pub skip_failure: bool, - #[serde(default = "default_true")] - pub parallel: bool, -} - -// Tool types for AI Agent -#[derive(Serialize, Debug, Clone, Deserialize)] -pub struct AgentTool { - pub id: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub summary: Option, - pub value: ToolValue, -} - -// Convert FlowModule -> AgentTool -impl From for AgentTool { - fn from(flow_module: FlowModule) -> Self { - let module_value = serde_json::from_str::(flow_module.value.get()) - .unwrap_or(FlowModuleValue::Identity); - - AgentTool { - id: flow_module.id, - summary: flow_module.summary, - value: ToolValue::FlowModule(module_value), - } - } -} - -// Convert AgentTool -> FlowModule (only for FlowModule type tools) -impl From<&AgentTool> for Option { - fn from(tool: &AgentTool) -> Self { - match &tool.value { - ToolValue::FlowModule(module_value) => Some(FlowModule { - id: tool.id.clone(), - value: to_raw_value(module_value), - summary: tool.summary.clone(), - ..Default::default() - }), - ToolValue::Mcp(_) => None, // MCP tools can't be converted to FlowModule - ToolValue::Websearch(_) => None, // Websearch tools can't be converted to FlowModule - } - } -} - -#[derive(Serialize, Debug, Clone)] -#[serde(tag = "tool_type", rename_all = "lowercase")] -pub enum ToolValue { - FlowModule(FlowModuleValue), - Mcp(McpToolValue), - Websearch(WebsearchToolValue), -} - -// Custom deserializer for backward compatibility with old flows -impl<'de> Deserialize<'de> for ToolValue { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - use serde::de::Error; - - let content = serde_json::Value::deserialize(deserializer)?; - - // First, try to deserialize as the new tagged format (with tool_type field) - #[derive(Deserialize)] - #[serde(tag = "tool_type", rename_all = "lowercase")] - enum TaggedToolValue { - FlowModule(FlowModuleValue), - Mcp(McpToolValue), - Websearch(WebsearchToolValue), - } - - if let Ok(tagged) = TaggedToolValue::deserialize(&content) { - return Ok(match tagged { - TaggedToolValue::FlowModule(v) => ToolValue::FlowModule(v), - TaggedToolValue::Mcp(v) => ToolValue::Mcp(v), - TaggedToolValue::Websearch(v) => ToolValue::Websearch(v), - }); - } - - // Fall back to legacy format (direct FlowModuleValue without tool_type) - FlowModuleValue::deserialize(&content) - .map(ToolValue::FlowModule) - .map_err(|_| { - D::Error::custom( - "expected ToolValue with tool_type field or legacy FlowModuleValue", - ) - }) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct McpToolValue { - pub resource_path: String, - #[serde(default)] - pub include_tools: Vec, - #[serde(default)] - pub exclude_tools: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -pub struct WebsearchToolValue { - // WebSearch tools don't need additional configuration - // The tool is enabled just by adding it to the agent -} - -fn is_none_or_empty_vec(expr: &Option>) -> bool { - expr.is_none() || expr.as_ref().unwrap().is_empty() -} - -#[derive(Serialize, Debug, Clone)] -#[serde( - tag = "type", - rename_all(serialize = "lowercase", deserialize = "lowercase") -)] -pub enum FlowModuleValue { - /// Reference to another script on the workspace - Script { - #[serde(default)] - #[serde(alias = "input_transform")] - input_transforms: HashMap, - path: String, - #[serde(skip_serializing_if = "Option::is_none")] - hash: Option, - #[serde(skip_serializing_if = "Option::is_none")] - tag_override: Option, - #[serde(skip_serializing_if = "Option::is_none")] - is_trigger: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pass_flow_input_directly: Option, - }, - - /// Reference to another flow on the workspace - Flow { - #[serde(default)] - #[serde(alias = "input_transform")] - input_transforms: HashMap, - path: String, - #[serde(skip_serializing_if = "Option::is_none")] - pass_flow_input_directly: Option, - }, - - /// For loop node - ForloopFlow { - iterator: InputTransform, - modules: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - modules_node: Option, - #[serde(default = "default_true")] - skip_failures: bool, - parallel: bool, - #[serde(skip_serializing_if = "Option::is_none")] - parallelism: Option, - #[serde(skip_serializing_if = "Option::is_none")] - squash: Option, - }, - - /// While loop node - WhileloopFlow { - modules: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - modules_node: Option, - #[serde(default = "default_false")] - skip_failures: bool, - #[serde(skip_serializing_if = "Option::is_none")] - squash: Option, - }, - - /// Branch-one node - BranchOne { - branches: Vec, - default: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - default_node: Option, - }, - - /// Branch-all node - BranchAll { - branches: Vec, - #[serde(default = "default_true")] - parallel: bool, - }, - - /// Inline script node - /// Only exists if parsed from value from `flow_version` | `flow` table. - RawScript { - #[serde(default)] - #[serde(alias = "input_transform", serialize_with = "ordered_map")] - input_transforms: HashMap, - content: String, - #[serde(skip_serializing_if = "Option::is_none")] - lock: Option, - #[serde(skip_serializing_if = "Option::is_none")] - path: Option, - #[serde(skip_serializing_if = "is_none_or_empty")] - tag: Option, - language: ScriptLang, - #[serde(flatten)] - concurrency_settings: ConcurrencySettingsWithCustom, - #[serde(skip_serializing_if = "Option::is_none")] - is_trigger: Option, - #[serde(skip_serializing_if = "is_none_or_empty_vec")] - assets: Option>, - }, - - /// Just a placeholder - Identity, - - /// Also Inline script node, but instead of being baked into flow, it references `flow_node` - /// Internal only, never exposed to the frontend. - /// Only exists if parsed from value from `flow_version_lite` table. - FlowScript { - #[serde(default)] - #[serde(alias = "input_transform", serialize_with = "ordered_map")] - input_transforms: HashMap, - id: FlowNodeId, - #[serde(skip_serializing_if = "is_none_or_empty")] - tag: Option, - language: ScriptLang, - #[serde(flatten)] - concurrency_settings: ConcurrencySettingsWithCustom, - #[serde(skip_serializing_if = "Option::is_none")] - is_trigger: Option, - #[serde(skip_serializing_if = "is_none_or_empty_vec")] - assets: Option>, - }, - - // AI agent node - AIAgent { - input_transforms: HashMap, - tools: Vec, - }, -} - -fn is_none_or_empty(expr: &Option) -> bool { - expr.is_none() || expr.as_ref().unwrap().is_empty() -} - -#[derive(Deserialize)] -struct UntaggedFlowModuleValue { - #[serde(rename = "type")] - type_: String, - #[serde(alias = "input_transform")] - input_transforms: Option>, - path: Option, - hash: Option, - tag_override: Option, - iterator: Option, - modules: Option>, - skip_failures: Option, - parallel: Option, - #[serde(default, deserialize_with = "raw_value_to_input_transform::<_, u16>")] - parallelism: Option, - branches: Option>, - default: Option>, - content: Option, - lock: Option, - tag: Option, - language: Option, - is_trigger: Option, - id: Option, - default_node: Option, - modules_node: Option, - assets: Option>, - tools: Option>, - pass_flow_input_directly: Option, - squash: Option, - #[serde(flatten)] - concurrency_settings: ConcurrencySettingsWithCustom, -} - -impl<'de> Deserialize<'de> for FlowModuleValue { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let untagged: UntaggedFlowModuleValue = UntaggedFlowModuleValue::deserialize(deserializer)?; - - match untagged.type_.as_str() { - "script" => Ok(FlowModuleValue::Script { - input_transforms: untagged.input_transforms.unwrap_or_default(), - path: untagged - .path - .ok_or_else(|| serde::de::Error::missing_field("path"))?, - hash: untagged.hash, - tag_override: untagged.tag_override, - is_trigger: untagged.is_trigger, - pass_flow_input_directly: untagged.pass_flow_input_directly, - }), - "flow" => Ok(FlowModuleValue::Flow { - input_transforms: untagged.input_transforms.unwrap_or_default(), - path: untagged - .path - .ok_or_else(|| serde::de::Error::missing_field("path"))?, - pass_flow_input_directly: untagged.pass_flow_input_directly, - }), - "forloopflow" => Ok(FlowModuleValue::ForloopFlow { - iterator: untagged - .iterator - .ok_or_else(|| serde::de::Error::missing_field("iterator"))?, - modules: untagged - .modules - .ok_or_else(|| serde::de::Error::missing_field("modules"))?, - modules_node: untagged.modules_node, - skip_failures: untagged.skip_failures.unwrap_or(true), - parallel: untagged.parallel.unwrap_or(false), - parallelism: untagged.parallelism, - squash: untagged.squash, - }), - "whileloopflow" => Ok(FlowModuleValue::WhileloopFlow { - modules: untagged - .modules - .ok_or_else(|| serde::de::Error::missing_field("modules"))?, - modules_node: untagged.modules_node, - skip_failures: untagged.skip_failures.unwrap_or(false), - squash: untagged.squash, - }), - "branchone" => Ok(FlowModuleValue::BranchOne { - branches: untagged - .branches - .ok_or_else(|| serde::de::Error::missing_field("branches"))?, - default: untagged - .default - .ok_or_else(|| serde::de::Error::missing_field("default"))?, - default_node: untagged.default_node, - }), - "branchall" => Ok(FlowModuleValue::BranchAll { - branches: untagged - .branches - .ok_or_else(|| serde::de::Error::missing_field("branches"))?, - parallel: untagged.parallel.unwrap_or(true), - }), - "rawscript" => Ok(FlowModuleValue::RawScript { - input_transforms: untagged.input_transforms.unwrap_or_default(), - content: untagged - .content - .ok_or_else(|| serde::de::Error::missing_field("content"))?, - lock: untagged.lock, - path: untagged.path, - tag: untagged.tag, - language: untagged - .language - .ok_or_else(|| serde::de::Error::missing_field("language"))?, - concurrency_settings: untagged.concurrency_settings, - is_trigger: untagged.is_trigger, - assets: untagged.assets, - }), - "flowscript" => Ok(FlowModuleValue::FlowScript { - input_transforms: untagged.input_transforms.unwrap_or_default(), - id: untagged - .id - .ok_or_else(|| serde::de::Error::missing_field("id"))?, - tag: untagged.tag, - language: untagged - .language - .ok_or_else(|| serde::de::Error::missing_field("language"))?, - concurrency_settings: untagged.concurrency_settings, - is_trigger: untagged.is_trigger, - assets: untagged.assets, - }), - "identity" => Ok(FlowModuleValue::Identity), - "aiagent" => Ok(FlowModuleValue::AIAgent { - input_transforms: untagged.input_transforms.unwrap_or_default(), - tools: untagged - .tools - .ok_or_else(|| serde::de::Error::missing_field("tools"))?, - }), - other => Err(serde::de::Error::unknown_variant( - other, - &[ - "script", - "flow", - "forloopflow", - "whileloopflow", - "branchone", - "branchall", - "rawscript", - "identity", - "aiagent", - ], - )), - } - } -} - -impl Into> for FlowModuleValue { - fn into(self) -> Box { - to_raw_value(&self) - } -} - -fn ordered_map(value: &HashMap, serializer: S) -> Result -where - S: Serializer, -{ - let ordered: BTreeMap<_, _> = value.iter().collect(); - ordered.serialize(serializer) -} - -#[derive(Deserialize)] -pub struct ListFlowQuery { - pub without_description: Option, - pub path_start: Option, - pub path_exact: Option, - pub edited_by: Option, - pub show_archived: Option, - pub order_by: Option, - pub order_desc: Option, - pub starred_only: Option, - pub include_draft_only: Option, - pub with_deployment_msg: Option, - pub dedicated_worker: Option, -} - -pub fn add_virtual_items_if_necessary(modules: &mut Vec) { - if modules.len() > 0 - && (modules[modules.len() - 1].sleep.is_some() - || modules[modules.len() - 1].suspend.is_some()) - { - modules.push(FlowModule { - id: format!("{}-v", modules[modules.len() - 1].id), - value: crate::worker::to_raw_value(&FlowModuleValue::Identity), - stop_after_if: None, - stop_after_all_iters_if: None, - summary: Some("Virtual module needed for suspend/sleep when last module".to_string()), - mock: None, - retry: None, - sleep: None, - suspend: None, - cache_ttl: None, - cache_ignore_s3_path: None, - timeout: None, - priority: None, - delete_after_use: None, - continue_on_error: None, - skip_if: None, - apply_preprocessor: None, - pass_flow_input_directly: None, - }); - } -} - /// Resolve the value of a flow if any. pub async fn resolve_maybe_value( e: &sqlx::PgPool, diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index d9e500ef86..9362f7ff02 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -4,557 +4,24 @@ use bytes::Bytes; use futures_core::Stream; use indexmap::IndexMap; use once_cell::sync::OnceCell; -use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use sqlx::types::Json; use tokio::io::AsyncReadExt; -use uuid::Uuid; -pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE"; -pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000; - -pub const EMAIL_ERROR_HANDLER_USER_EMAIL: &str = "email_error_handler@windmill.dev"; +pub use windmill_types::jobs::*; use crate::{ - apps::AppScriptId, auth::is_super_admin_email, client::AuthedClient, db::{AuthedRef, UserDbWithAuthed, DB}, error::{self, to_anyhow, Error}, - flow_status::{FlowStatus, RestartedFrom}, - flows::{FlowNodeId, FlowValue, Retry}, get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, - runnable_settings::{ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings}, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, users::username_to_permissioned_as, utils::{StripPath, HTTP_CLIENT}, worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, TMP_DIR}, - FlowVersionInfo, ScriptHashInfo, + FlowVersionInfo, ScriptHashInfo, Tag, }; -#[derive(Debug, Deserialize, Clone)] -pub struct DynamicInput { - #[serde(rename = "x-windmill-dyn-select-code")] - pub x_windmill_dyn_select_code: String, - #[serde(rename = "x-windmill-dyn-select-lang")] - pub x_windmill_dyn_select_lang: ScriptLang, -} - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone)] -#[sqlx(type_name = "JOB_TRIGGER_KIND", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum JobTriggerKind { - Webhook, - Http, - Websocket, - Kafka, - Email, - Nats, - Mqtt, - Sqs, - Postgres, - Schedule, - Gcp, - Nextcloud, -} - -impl std::fmt::Display for JobTriggerKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let kind = match self { - JobTriggerKind::Webhook => "webhook", - JobTriggerKind::Http => "http", - JobTriggerKind::Websocket => "websocket", - JobTriggerKind::Kafka => "kafka", - JobTriggerKind::Email => "email", - JobTriggerKind::Nats => "nats", - JobTriggerKind::Mqtt => "mqtt", - JobTriggerKind::Sqs => "sqs", - JobTriggerKind::Postgres => "postgres", - JobTriggerKind::Schedule => "schedule", - JobTriggerKind::Gcp => "gcp", - JobTriggerKind::Nextcloud => "nextcloud", - }; - write!(f, "{}", kind) - } -} - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Default)] -#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum JobKind { - Script, - #[allow(non_camel_case_types)] - Script_Hub, - Preview, - Dependencies, - Flow, - FlowPreview, - SingleStepFlow, - Identity, - FlowDependencies, - AppDependencies, - #[default] - Noop, - DeploymentCallback, - FlowScript, - FlowNode, - AppScript, - AIAgent, - #[serde(rename = "unassigned_script")] - #[sqlx(rename = "unassigned_script")] - UnassignedScript, - #[serde(rename = "unassigned_flow")] - #[sqlx(rename = "unassigned_flow")] - UnassignedFlow, - #[serde(rename = "unassigned_singlestepflow")] - #[sqlx(rename = "unassigned_singlestepflow")] - UnassignedSinglestepFlow, -} - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone)] -#[sqlx(type_name = "JOB_STATUS", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum JobStatus { - Success, - Failure, - Canceled, - Skipped, -} - -impl JobKind { - pub fn is_flow(&self) -> bool { - matches!( - self, - JobKind::Flow | JobKind::FlowPreview | JobKind::SingleStepFlow | JobKind::FlowNode - ) - } - - pub fn is_dependency(&self) -> bool { - matches!( - self, - JobKind::FlowDependencies | JobKind::AppDependencies | JobKind::Dependencies - ) - } -} - -#[derive(sqlx::FromRow, Debug, Serialize, Clone)] -pub struct QueuedJob { - pub workspace_id: String, - pub id: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - pub parent_job: Option, - pub created_by: String, - pub created_at: chrono::DateTime, - #[serde(skip_serializing_if = "Option::is_none")] - pub started_at: Option>, - pub scheduled_for: chrono::DateTime, - pub running: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub script_hash: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub script_path: Option, - pub script_entrypoint_override: Option, - pub args: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub logs: Option, - pub canceled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub canceled_by: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub canceled_reason: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_ping: Option>, - pub job_kind: JobKind, - #[serde(skip_serializing_if = "Option::is_none")] - pub schedule_path: Option, - pub permissioned_as: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub flow_status: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub workflow_as_code_status: Option>>, - pub is_flow_step: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub language: Option, - pub same_worker: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub pre_run_error: Option, - pub email: String, - pub visible_to_owner: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub suspend: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub mem_peak: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub root_job: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub leaf_jobs: Option, - pub tag: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrent_limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_time_window_s: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub timeout: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub flow_step_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ttl: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ignore_s3_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub preprocessed: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub runnable_settings_handle: Option, -} - -impl QueuedJob { - pub fn script_path(&self) -> &str { - self.script_path - .as_ref() - .map(String::as_str) - .unwrap_or("tmp/main") - } - pub fn is_flow(&self) -> bool { - self.job_kind.is_flow() - } - - pub fn full_path_with_workspace(&self) -> String { - format!( - "{}/{}/{}", - self.workspace_id, - if self.is_flow() { "flow" } else { "script" }, - self.script_path() - ) - } - - pub fn parse_flow_status(&self) -> Option { - self.flow_status - .as_ref() - .and_then(|v| serde_json::from_str::((**v).get()).ok()) - } -} - -impl Default for QueuedJob { - fn default() -> Self { - Self { - workspace_id: "".to_string(), - id: Uuid::default(), - parent_job: None, - created_by: "".to_string(), - created_at: chrono::Utc::now(), - started_at: None, - scheduled_for: chrono::Utc::now(), - running: false, - script_hash: None, - script_path: None, - args: None, - logs: None, - canceled: false, - canceled_by: None, - canceled_reason: None, - last_ping: None, - job_kind: JobKind::Identity, - schedule_path: None, - permissioned_as: "".to_string(), - workflow_as_code_status: None, - flow_status: None, - is_flow_step: false, - language: None, - script_entrypoint_override: None, - same_worker: false, - pre_run_error: None, - email: "".to_string(), - visible_to_owner: false, - suspend: None, - mem_peak: None, - root_job: None, - leaf_jobs: None, - tag: "deno".to_string(), - concurrent_limit: None, - concurrency_time_window_s: None, - timeout: None, - flow_step_id: None, - cache_ttl: None, - cache_ignore_s3_path: None, - priority: None, - preprocessed: None, - runnable_settings_handle: None, - } - } -} - -#[derive(Debug, sqlx::FromRow, Serialize, Clone)] -pub struct CompletedJob { - pub workspace_id: String, - pub id: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - pub parent_job: Option, - pub created_by: String, - pub created_at: chrono::DateTime, - pub started_at: Option>, - pub completed_at: Option>, - pub duration_ms: i64, - pub success: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub script_hash: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub script_path: Option, - pub args: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub result: Option>>, - pub result_columns: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub logs: Option, - pub deleted: bool, - pub canceled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub canceled_by: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub canceled_reason: Option, - pub job_kind: JobKind, - #[serde(skip_serializing_if = "Option::is_none")] - pub schedule_path: Option, - pub permissioned_as: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub flow_status: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub workflow_as_code_status: Option>>, - pub is_flow_step: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub language: Option, - pub is_skipped: bool, - pub email: String, - pub visible_to_owner: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub mem_peak: Option, - pub tag: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub labels: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub preprocessed: Option, -} - -impl CompletedJob { - pub fn json_result(&self) -> Option { - self.result - .as_ref() - .map(|r| serde_json::from_str(r.get()).ok()) - .flatten() - } - - pub fn parse_flow_status(&self) -> Option { - self.flow_status - .as_ref() - .and_then(|v| serde_json::from_str::((**v).get()).ok()) - } -} - -#[derive(Debug, Clone)] -pub enum JobPayload { - /// Execute Hub Script - ScriptHub { - path: String, - apply_preprocessor: bool, - }, - - /// Execute script - ScriptHash { - hash: ScriptHash, - path: String, - cache_ttl: Option, - cache_ignore_s3_path: Option, - dedicated_worker: Option, - language: ScriptLang, - priority: Option, - apply_preprocessor: bool, - concurrency_settings: ConcurrencySettings, - debouncing_settings: DebouncingSettings, - }, - - /// Execute flow step (can be subflow only). - FlowNode { - id: FlowNodeId, // flow_node(id). - path: String, // flow node inner path (e.g. `outer/branchall-42`). - }, - - /// Execute flow step - FlowScript { - id: FlowNodeId, // flow_node(id). - path: String, - language: ScriptLang, - cache_ttl: Option, - cache_ignore_s3_path: Option, - dedicated_worker: Option, - concurrency_settings: ConcurrencySettings, - }, - - /// Inline App Script - AppScript { - id: AppScriptId, // app_script(id). - path: Option, - language: ScriptLang, - cache_ttl: Option, - }, - - /// Script/App/FlowAsCode Preview - Code(RawCode), - - /// Script Dependency Job - Dependencies { - path: String, - hash: ScriptHash, - language: ScriptLang, - dedicated_worker: Option, - debouncing_settings: DebouncingSettings, - }, - - /// Flow Dependency Job - FlowDependencies { - path: String, - dedicated_worker: Option, - version: i64, - debouncing_settings: DebouncingSettings, - }, - - /// App Dependency Job - AppDependencies { - path: String, - version: i64, - debouncing_settings: DebouncingSettings, - }, - - /// Flow Dependency Job, exposed with API. Requirements can be partially or fully predefined - RawFlowDependencies { - path: String, - flow_value: FlowValue, - }, - - /// Dependency Job, exposed with API. Requirements can be predefined - RawScriptDependencies { - script_path: String, - /// Will reflect raw requirements content (e.g. requirements.in) - content: String, - language: ScriptLang, - }, - - /// Flow Job - Flow { - path: String, - dedicated_worker: Option, - apply_preprocessor: bool, - version: i64, - }, - - RestartedFlow { - completed_job_id: Uuid, - step_id: String, - branch_or_iteration_n: Option, - flow_version: Option, - }, - - /// Flow Preview - RawFlow { - value: FlowValue, - path: Option, - restarted_from: Option, - }, - - /// Flow consisting of single script - SingleStepFlow { - path: String, - hash: Option, - flow_version: Option, - args: HashMap>, - retry: Option, - error_handler_path: Option, - error_handler_args: Option>>, - skip_handler: Option, - cache_ttl: Option, - cache_ignore_s3_path: Option, - priority: Option, - tag_override: Option, - trigger_path: Option, - apply_preprocessor: bool, - concurrency_settings: ConcurrencySettings, - debouncing_settings: DebouncingSettings, - }, - DeploymentCallback { - path: String, - debouncing_settings: DebouncingSettings, - }, - Identity, - Noop, - AIAgent { - path: String, - }, -} - -#[derive(Clone, Serialize, Deserialize, Debug)] -pub struct SkipHandler { - pub path: String, - pub args: HashMap>, - pub stop_condition: String, - pub stop_message: String, -} - -#[derive(Clone, Deserialize, Debug, Default)] -pub struct RawCode { - pub content: String, - pub path: Option, - pub hash: Option, - pub language: ScriptLang, - pub lock: Option, - pub cache_ttl: Option, - pub cache_ignore_s3_path: Option, - pub dedicated_worker: Option, - #[serde(flatten)] - pub concurrency_settings: ConcurrencySettingsWithCustom, - #[serde(flatten)] - // NOTE: Since we can only deserialize the struct, - // even though the older versions pass `custom_debounce_key` to RawCode, - // we can still have `debounce_key` in DebouncingSettings - // we just add alias `custom_debounce_key` - // however, serializing this settings will produce `debounce_key` - pub debouncing_settings: DebouncingSettings, -} - -impl JobPayload { - pub fn job_kind(&self) -> JobKind { - match self { - JobPayload::Noop => JobKind::Noop, - JobPayload::Identity => JobKind::Identity, - JobPayload::Code { .. } => JobKind::Preview, - JobPayload::AIAgent { .. } => JobKind::AIAgent, - JobPayload::FlowNode { .. } => JobKind::FlowNode, - JobPayload::ScriptHash { .. } => JobKind::Script, - JobPayload::AppScript { .. } => JobKind::AppScript, - JobPayload::RawFlow { .. } => JobKind::FlowPreview, - JobPayload::ScriptHub { .. } => JobKind::Script_Hub, - JobPayload::FlowScript { .. } => JobKind::FlowScript, - JobPayload::Dependencies { .. } => JobKind::Dependencies, - JobPayload::SingleStepFlow { .. } => JobKind::SingleStepFlow, - JobPayload::AppDependencies { .. } => JobKind::AppDependencies, - JobPayload::FlowDependencies { .. } => JobKind::FlowDependencies, - JobPayload::RawScriptDependencies { .. } => JobKind::Dependencies, - JobPayload::RawFlowDependencies { .. } => JobKind::FlowDependencies, - JobPayload::DeploymentCallback { .. } => JobKind::DeploymentCallback, - JobPayload::Flow { .. } | JobPayload::RestartedFlow { .. } => JobKind::Flow, - } - } -} - -type Tag = String; - -#[derive(Clone, Debug)] -pub struct OnBehalfOf { - pub email: String, - pub permissioned_as: String, -} - pub fn get_has_preprocessor_from_content_and_lang( content: &str, language: &ScriptLang, @@ -671,11 +138,6 @@ pub async fn script_path_to_payload<'e>( )) } -#[inline(always)] -pub fn generate_dynamic_input_key(workspace_id: &str, path: &str) -> String { - format!("{workspace_id}:{path}") -} - pub async fn get_payload_tag_from_prefixed_path( path: &str, db: &DB, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 20ba0d4509..99fa1c46ca 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -819,7 +819,6 @@ use crate::{ auth::{PermsCache, FLOW_PERMS_CACHE, HASH_PERMS_CACHE}, db::{AuthedRef, UserDbWithAuthed}, error::to_anyhow, - runnable_settings::RunnableSettings, scripts::{ScriptHash, ScriptRunnableSettingsHandle, ScriptRunnableSettingsInline}, }; @@ -853,14 +852,13 @@ impl ScriptHashInfo { self, db: &DB, ) -> error::Result> { + let rs = runnable_settings::from_handle( + self.runnable_settings.runnable_settings_handle, + db, + ) + .await?; let (debouncing_settings, concurrency_settings) = - RunnableSettings::from_runnable_settings_handle( - self.runnable_settings.runnable_settings_handle, - db, - ) - .await? - .prefetch_cached(db) - .await?; + runnable_settings::prefetch_cached(&rs, db).await?; Ok(ScriptHashInfo { path: self.path, diff --git a/backend/windmill-common/src/more_serde.rs b/backend/windmill-common/src/more_serde.rs index 97df70219c..8288d79887 100644 --- a/backend/windmill-common/src/more_serde.rs +++ b/backend/windmill-common/src/more_serde.rs @@ -8,60 +8,4 @@ //! helpers for serde + serde derive attributes -use crate::utils::rd_string; -use serde::{Deserialize, Deserializer}; -use serde_json::value::RawValue; -use std::{fmt::Display, str::FromStr}; - -pub fn default_true() -> bool { - true -} - -pub fn default_false() -> bool { - false -} - -pub fn default_null() -> Box { - RawValue::from_string("null".to_string()).unwrap() -} - -pub fn default_empty_string() -> String { - String::new() -} - -pub fn default_id() -> String { - rd_string(6) -} - -pub fn is_default(t: &T) -> bool { - &T::default() == t -} - -pub fn maybe_number_opt<'de, T, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, - T: FromStr + serde::Deserialize<'de>, - ::Err: Display, -{ - #[derive(Deserialize)] - #[serde(untagged)] - enum NumericOrNull<'a, T> { - String(String), - Str(&'a str), - RawT(T), - Null, - } - - match NumericOrNull::::deserialize(deserializer)? { - NumericOrNull::String(s) => match s.as_str() { - "" => Ok(None), - _ => T::from_str(&s).map(Some).map_err(serde::de::Error::custom), - }, - NumericOrNull::Str(s) => match s { - "" => Ok(None), - _ => T::from_str(s).map(Some).map_err(serde::de::Error::custom), - }, - NumericOrNull::RawT(i) => Ok(Some(i)), - NumericOrNull::Null => Ok(None), - } -} +pub use windmill_types::more_serde::*; diff --git a/backend/windmill-common/src/runnable_settings/settings.rs b/backend/windmill-common/src/runnable_settings/settings.rs index 6dc4eb38ca..04b62fbaab 100644 --- a/backend/windmill-common/src/runnable_settings/settings.rs +++ b/backend/windmill-common/src/runnable_settings/settings.rs @@ -1,9 +1,5 @@ -use std::{ - future::Future, - hash::{Hash, Hasher}, -}; +use std::future::Future; -use serde::{Deserialize, Serialize}; use sqlx::{Pool, Postgres}; use crate::{ @@ -14,49 +10,43 @@ use crate::{ DB, }; -#[derive(Deserialize, Clone, Copy, Serialize, Default, Hash)] -pub struct RunnableSettings { - pub debouncing_settings: Option, - pub concurrency_settings: Option, +pub use windmill_types::runnable_settings::*; + +pub async fn prefetch_cached( + rs: &RunnableSettings, + db: &DB, +) -> error::Result<(DebouncingSettings, ConcurrencySettings)> { + Ok(( + if let Some(hash) = rs.debouncing_settings { + DebouncingSettings::get(hash, db).await? + } else { + Default::default() + }, + if let Some(hash) = rs.concurrency_settings { + ConcurrencySettings::get(hash, db).await? + } else { + Default::default() + }, + )) } -impl RunnableSettings { - pub async fn prefetch_cached<'a>( - &self, - db: &DB, - ) -> error::Result<(DebouncingSettings, ConcurrencySettings)> { - Ok(( - if let Some(hash) = self.debouncing_settings { - DebouncingSettings::get(hash, db).await? - } else { - Default::default() - }, - if let Some(hash) = self.concurrency_settings { - ConcurrencySettings::get(hash, db).await? - } else { - Default::default() - }, - )) - } +pub async fn prefetch_cached_from_handle( + hash: Option, + db: &DB, +) -> error::Result<(DebouncingSettings, ConcurrencySettings)> { + let rs = from_handle(hash, db).await?; + prefetch_cached(&rs, db).await +} - pub async fn prefetch_cached_from_handle<'a>( - hash: Option, - db: &'a DB, - ) -> error::Result<(DebouncingSettings, ConcurrencySettings)> { - Self::from_runnable_settings_handle(hash, db) - .await? - .prefetch_cached(db) - .await - } - /// Returns error if provided `hash` has no corresponding entry in db - /// If `hash` is None, returnes Default - pub fn from_runnable_settings_handle<'a>( - hash: Option, - db: &'a DB, - ) -> impl Future> + 'a { - async move { - if let Some(hash) = hash { - super::RUNNABLE_SETTINGS_REFERENCES +/// Returns error if provided `hash` has no corresponding entry in db +/// If `hash` is None, returns Default +pub fn from_handle<'a>( + hash: Option, + db: &'a DB, +) -> impl Future> + 'a { + async move { + if let Some(hash) = hash { + super::RUNNABLE_SETTINGS_REFERENCES .get_or_insert_async(hash, async { sqlx::query_as!( RunnableSettings, @@ -68,138 +58,45 @@ impl RunnableSettings { .map_err(error::Error::from) }) .await - } else { - Ok(Self::default()) - } + } else { + Ok(RunnableSettings::default()) } } +} - pub async fn insert_cached(self, db: &Pool) -> error::Result> { - if !min_version_supports_runnable_settings_v0().await - || (self.debouncing_settings.is_none() && self.concurrency_settings.is_none()) - { - return Ok(None); - } +pub async fn insert_rs(rs: RunnableSettings, db: &Pool) -> error::Result> { + use std::hash::{Hash, Hasher}; - let hash = { - let mut h = std::hash::DefaultHasher::new(); - self.hash(&mut h); - h.finish() as i64 - }; + if !min_version_supports_runnable_settings_v0().await + || (rs.debouncing_settings.is_none() && rs.concurrency_settings.is_none()) + { + return Ok(None); + } - super::RUNNABLE_SETTINGS_REFERENCES - .get_or_insert_async(hash, async { - sqlx::query!( - "INSERT INTO runnable_settings (hash, debouncing_settings, concurrency_settings) + let hash = { + let mut h = std::hash::DefaultHasher::new(); + rs.hash(&mut h); + h.finish() as i64 + }; + + super::RUNNABLE_SETTINGS_REFERENCES + .get_or_insert_async(hash, async { + sqlx::query!( + "INSERT INTO runnable_settings (hash, debouncing_settings, concurrency_settings) VALUES ($1, $2, $3) ON CONFLICT (hash) DO NOTHING", - hash, - self.debouncing_settings, - self.concurrency_settings - ) - .execute(db) - .await?; - // .map_err(error::Error::from) - Ok(self) - }) + hash, + rs.debouncing_settings, + rs.concurrency_settings + ) + .execute(db) .await?; + Ok(rs) + }) + .await?; - Ok(Some(hash)) - } -} - -// TODO: Add validation logic. -#[derive( - Debug, Clone, Serialize, Deserialize, Default, Hash, PartialEq, sqlx::FromRow, sqlx::Type, -)] -pub struct DebouncingSettings { - #[serde(skip_serializing_if = "Option::is_none", alias = "custom_debounce_key")] - /// debounce key is usually stored in the db - /// including when: - /// - /// 1. User have created custom debounce key from ui or cli - /// 2. User used default one - /// - /// Default: hash(path + step_id + inputs) - pub debounce_key: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - /// Debouncing delay will be determined by the first job with the key. - /// All subsequent jobs with Some will get debounced. - /// If the job has no delay, it will execute immediately, fully ignoring pending delays. - pub debounce_delay_s: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub max_total_debouncing_time: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub max_total_debounces_amount: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - /// top level arguments to preserve - /// For every debounce selected arguments will be saved - /// in the end (when job finally starts) arguments will be appended and passed to runnable - /// - /// NOTE: selected args should be the lists. - pub debounce_args_to_accumulate: Option>, -} - -#[derive( - Debug, Default, Clone, Serialize, Deserialize, Hash, PartialEq, sqlx::FromRow, sqlx::Decode, -)] -pub struct ConcurrencySettings { - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrent_limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_time_window_s: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow, Default)] -pub struct ConcurrencySettingsWithCustom { - #[serde(skip_serializing_if = "Option::is_none")] - pub custom_concurrency_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrent_limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_time_window_s: Option, -} - -impl DebouncingSettings { - pub fn maybe_fallback( - self, - debounce_key: Option, - debounce_delay_s: Option, - ) -> Self { - Self { - debounce_key: self.debounce_key.or(debounce_key), - debounce_delay_s: self.debounce_delay_s.or(debounce_delay_s), - ..self - } - } - - pub fn is_legacy_compatible(&self) -> bool { - self.max_total_debouncing_time.is_none() - && self.max_total_debounces_amount.is_none() - && self.debounce_args_to_accumulate.is_none() - } -} - -impl ConcurrencySettings { - pub fn maybe_fallback( - self, - concurrency_key: Option, - concurrent_limit: Option, - concurrency_time_window_s: Option, - ) -> Self { - Self { - concurrency_key: self.concurrency_key.or(concurrency_key), - concurrent_limit: self.concurrent_limit.or(concurrent_limit), - concurrency_time_window_s: self.concurrency_time_window_s.or(concurrency_time_window_s), - } - } + Ok(Some(hash)) } impl super::private_mod::RunnableSettingsTraitInternal for DebouncingSettings { @@ -235,31 +132,3 @@ impl super::private_mod::RunnableSettingsTraitInternal for ConcurrencySettings { } } impl super::RunnableSettingsTrait for ConcurrencySettings {} - -impl From for ConcurrencySettingsWithCustom { - fn from( - ConcurrencySettings { concurrency_key, concurrent_limit, concurrency_time_window_s }: ConcurrencySettings, - ) -> Self { - ConcurrencySettingsWithCustom { - custom_concurrency_key: concurrency_key, - concurrency_time_window_s, - concurrent_limit, - } - } -} - -impl From for ConcurrencySettings { - fn from( - ConcurrencySettingsWithCustom { - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - }: ConcurrencySettingsWithCustom, - ) -> Self { - ConcurrencySettings { - concurrency_key: custom_concurrency_key, - concurrency_time_window_s, - concurrent_limit, - } - } -} diff --git a/backend/windmill-common/src/runtime_assets.rs b/backend/windmill-common/src/runtime_assets.rs index 811e62e9d8..09abef389d 100644 --- a/backend/windmill-common/src/runtime_assets.rs +++ b/backend/windmill-common/src/runtime_assets.rs @@ -52,7 +52,7 @@ fn extract_assets_from_raw_value( if prefix { let s = serde_json::from_str::(value.get()).ok()?; let (kind, path) = parse_asset_syntax(&s, false)?; - assets.push(RuntimeAsset { path: path.to_string(), kind: kind.into() }); + assets.push(RuntimeAsset { path: path.to_string(), kind: crate::assets::asset_kind_from_parser(kind) }); } } None diff --git a/backend/windmill-common/src/schedule.rs b/backend/windmill-common/src/schedule.rs index cc7c9af758..c7223f54af 100644 --- a/backend/windmill-common/src/schedule.rs +++ b/backend/windmill-common/src/schedule.rs @@ -6,70 +6,4 @@ * LICENSE-AGPL for a copy of the license. */ -use chrono::DateTime; -use serde::{Deserialize, Serialize}; -use sqlx::FromRow; - -use crate::flows::Retry; - -#[derive(FromRow, Serialize, Deserialize, Debug, Clone)] -pub struct Schedule { - pub workspace_id: String, - pub path: String, - pub edited_by: String, - pub edited_at: DateTime, - pub schedule: String, - pub timezone: String, - pub enabled: bool, - pub script_path: String, - pub is_flow: bool, - pub args: Option>>, - pub extra_perms: serde_json::Value, - pub email: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_failure: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_failure_times: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_failure_exact: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_failure_extra_args: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_recovery: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_recovery_times: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_recovery_extra_args: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_success: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_success_extra_args: Option>>, - pub ws_error_handler_muted: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option, - pub no_flow_overlap: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub summary: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub tag: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub paused_until: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub cron_version: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub dynamic_skip: Option, -} - -impl Schedule { - pub fn parse_retry(self) -> Option { - self.retry.map(|r| serde_json::from_value(r).ok()).flatten() - } -} - -pub fn schedule_to_user(path: &str) -> String { - format!("schedule-{}", path.replace('/', "-")) -} +pub use windmill_types::schedule::*; diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 2d72cec70f..2f87cb14b0 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -6,17 +6,11 @@ * LICENSE-AGPL for a copy of the license. */ -use std::{ - fmt::{self, Display}, - hash::{Hash, Hasher}, - ops::Deref, - str::FromStr, -}; +pub use windmill_types::scripts::*; use crate::{ - assets::AssetWithAltAccessType, error::{to_anyhow, Error}, - runnable_settings::{ConcurrencySettings, DebouncingSettings, RunnableSettings}, + runnable_settings::{self}, utils::http_get_from_hub, workspace_dependencies::WorkspaceDependenciesAnnotatedRefs, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, PRIVATE_HUB_MIN_VERSION, @@ -26,708 +20,119 @@ use crate::worker::HUB_CACHE_DIR; use anyhow::Context; use backon::ConstantBuilder; use backon::{BackoffBuilder, Retryable}; -use itertools::Itertools; use regex::Regex; -use serde::de::Error as _; -use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize}; use crate::utils::StripPath; -#[derive( - Serialize, - Deserialize, - Debug, - PartialEq, - Copy, - Clone, - Hash, - Eq, - sqlx::Type, - Default, - Ord, - PartialOrd, -)] -#[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum ScriptLang { - Nativets, - #[default] - Deno, - Python3, - Go, - Bash, - Powershell, - Postgresql, - Bun, - Bunnative, - Mysql, - Bigquery, - Snowflake, - Graphql, - Mssql, - OracleDB, - DuckDb, - Php, - Rust, - Ansible, - CSharp, - Nu, - Java, - Ruby, - // for related places search: ADD_NEW_LANG +pub fn extract_workspace_dependencies_annotated_refs( + lang: &ScriptLang, + code: &str, + runnable_path: &str, +) -> Option> { + use ScriptLang::*; + lazy_static::lazy_static! { + static ref RE_PYTHON: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap(); + } + match lang { + // TODO: Maybe use regex + Bun | Bunnative | Nativets => WorkspaceDependenciesAnnotatedRefs::parse( + "//", + "package_json", + code, + None, + runnable_path, + ), + Python3 => WorkspaceDependenciesAnnotatedRefs::parse( + "#", + "requirements", + code, + Some(&RE_PYTHON), + runnable_path, + ), + Go => { + WorkspaceDependenciesAnnotatedRefs::parse("//", "go_mod", code, None, runnable_path) + } + Php => WorkspaceDependenciesAnnotatedRefs::parse( + "//", + "composer_json", + code, + None, + runnable_path, + ), + _ => return None, + } } -impl ScriptLang { - pub fn as_str(&self) -> &'static str { - match self { - ScriptLang::Bun => "bun", - ScriptLang::Bunnative => "bunnative", - ScriptLang::Nativets => "nativets", - ScriptLang::Deno => "deno", - ScriptLang::Python3 => "python3", - ScriptLang::Go => "go", - ScriptLang::Bash => "bash", - ScriptLang::Powershell => "powershell", - ScriptLang::Postgresql => "postgresql", - ScriptLang::Mysql => "mysql", - ScriptLang::Bigquery => "bigquery", - ScriptLang::Snowflake => "snowflake", - ScriptLang::Mssql => "mssql", - ScriptLang::Graphql => "graphql", - ScriptLang::OracleDB => "oracledb", - ScriptLang::DuckDb => "duckdb", - ScriptLang::Php => "php", - ScriptLang::Rust => "rust", - ScriptLang::Ansible => "ansible", - ScriptLang::CSharp => "csharp", - ScriptLang::Nu => "nu", - ScriptLang::Java => "java", - ScriptLang::Ruby => "ruby", - // for related places search: ADD_NEW_LANG - } - } +pub async fn prefetch_cached_script( + script: Script, + db: &DB, +) -> crate::error::Result> { + let rs = runnable_settings::from_handle( + script.runnable_settings.runnable_settings_handle, + db, + ) + .await?; + let (debouncing_settings, concurrency_settings) = + runnable_settings::prefetch_cached(&rs, db).await?; - pub fn as_dependencies_filename(&self) -> Option { - use ScriptLang::*; - Some( - match self { - Bun | Bunnative | Nativets => "package.json", - Python3 => "requirements.in", - // Go => "go.mod", - Php => "composer.json", - _ => return None, - } - .to_owned(), - ) - } - - pub fn as_comment_lit(&self) -> String { - use ScriptLang::*; - match self { - Nativets | Bun | Bunnative | Deno | Go | Php | CSharp | Java => "//", - Python3 | Bash | Powershell | Graphql | Ansible | Nu | Ruby => "#", - Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--", - Rust => "//!", - // for related places search: ADD_NEW_LANG - } - .to_owned() - } - - pub fn extract_workspace_dependencies_annotated_refs( - &self, - code: &str, - runnable_path: &str, - ) -> Option> { - use ScriptLang::*; - lazy_static::lazy_static! { - static ref RE_PYTHON: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap(); - } - match self { - // TODO: Maybe use regex - Bun | Bunnative | Nativets => WorkspaceDependenciesAnnotatedRefs::parse( - "//", - "package_json", - code, - None, - runnable_path, + Ok(Script { + workspace_id: script.workspace_id, + hash: script.hash, + path: script.path, + parent_hashes: script.parent_hashes, + summary: script.summary, + description: script.description, + content: script.content, + created_by: script.created_by, + created_at: script.created_at, + archived: script.archived, + schema: script.schema, + deleted: script.deleted, + is_template: script.is_template, + extra_perms: script.extra_perms, + lock: script.lock, + lock_error_logs: script.lock_error_logs, + language: script.language, + kind: script.kind, + tag: script.tag, + draft_only: script.draft_only, + envs: script.envs, + dedicated_worker: script.dedicated_worker, + ws_error_handler_muted: script.ws_error_handler_muted, + priority: script.priority, + cache_ttl: script.cache_ttl, + cache_ignore_s3_path: script.cache_ignore_s3_path, + timeout: script.timeout, + delete_after_use: script.delete_after_use, + restart_unless_cancelled: script.restart_unless_cancelled, + visible_to_runner_only: script.visible_to_runner_only, + no_main_func: script.no_main_func, + codebase: script.codebase, + has_preprocessor: script.has_preprocessor, + on_behalf_of_email: script.on_behalf_of_email, + assets: script.assets, + runnable_settings: ScriptRunnableSettingsInline { + concurrency_settings: concurrency_settings.maybe_fallback( + script.runnable_settings.concurrency_key, + script.runnable_settings.concurrent_limit, + script.runnable_settings.concurrency_time_window_s, ), - Python3 => WorkspaceDependenciesAnnotatedRefs::parse( - "#", - "requirements", - code, - Some(&RE_PYTHON), - runnable_path, + debouncing_settings: debouncing_settings.maybe_fallback( + script.runnable_settings.debounce_key, + script.runnable_settings.debounce_delay_s, ), - Go => { - WorkspaceDependenciesAnnotatedRefs::parse("//", "go_mod", code, None, runnable_path) - } - Php => WorkspaceDependenciesAnnotatedRefs::parse( - "//", - "composer_json", - code, - None, - runnable_path, - ), - _ => return None, - } - } + }, + }) } -impl FromStr for ScriptLang { - type Err = Error; - fn from_str(s: &str) -> Result { - let language = match s.to_lowercase().as_str() { - "bun" => ScriptLang::Bun, - "bunnative" => ScriptLang::Bunnative, - "nativets" => ScriptLang::Nativets, - "deno" => ScriptLang::Deno, - "python3" => ScriptLang::Python3, - "go" => ScriptLang::Go, - "bash" => ScriptLang::Bash, - "powershell" => ScriptLang::Powershell, - "postgresql" => ScriptLang::Postgresql, - "mysql" => ScriptLang::Mysql, - "bigquery" => ScriptLang::Bigquery, - "snowflake" => ScriptLang::Snowflake, - "mssql" => ScriptLang::Mssql, - "graphql" => ScriptLang::Graphql, - "oracledb" => ScriptLang::OracleDB, - "php" => ScriptLang::Php, - "rust" => ScriptLang::Rust, - "ansible" => ScriptLang::Ansible, - "csharp" => ScriptLang::CSharp, - "nu" => ScriptLang::Nu, - "java" => ScriptLang::Java, - "ruby" => ScriptLang::Ruby, - // for related places search: ADD_NEW_LANG - language => { - return Err(anyhow::anyhow!("{} is currently not supported", language).into()) - } - }; - - Ok(language) - } -} - -#[derive(Eq, PartialEq, Debug, Hash, Clone, Copy, sqlx::Type)] -#[sqlx(transparent)] -pub struct ScriptHash(pub i64); - -impl Deref for ScriptHash { - type Target = i64; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl Into for ScriptHash { - fn into(self) -> u64 { - self.0 as u64 - } -} - -impl From for ScriptHash { - fn from(value: i64) -> Self { - Self(value) - } -} - -#[derive(PartialEq, sqlx::Type, Debug)] -#[sqlx(transparent, no_pg_array)] -pub struct ScriptHashes(pub Vec); - -impl Display for ScriptHash { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", to_hex_string(&self.0)) - } -} -impl Serialize for ScriptHash { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: serde::Serializer, - { - serializer.serialize_str(to_hex_string(&self.0).as_str()) - } -} -impl<'de> Deserialize<'de> for ScriptHash { - fn deserialize(deserializer: D) -> std::result::Result - where - D: Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - let i = to_i64(&s).map_err(|e| { - tracing::error!("Could not deserialize ScriptHash. Note, input should be in Hex and digit amount should be divisible by 16 (can be padded). err: {}", &e); - D::Error::custom(format!("{}", e)) - })?; - Ok(ScriptHash(i)) - } -} - -impl Serialize for ScriptHashes { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: serde::Serializer, - { - let mut seq = serializer.serialize_seq(Some(self.0.len()))?; - for element in &self.0 { - seq.serialize_element(&ScriptHash(*element))?; - } - seq.end() - } -} - -#[derive(Serialize, Deserialize, Debug, Hash, sqlx::Type)] -#[sqlx(type_name = "SCRIPT_KIND", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum ScriptKind { - Trigger, - Failure, - Script, - Approval, - Preprocessor, -} - -impl Display for ScriptKind { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - fmt.write_str(match self { - ScriptKind::Trigger => "trigger", - ScriptKind::Failure => "failure", - ScriptKind::Script => "script", - ScriptKind::Approval => "approval", - ScriptKind::Preprocessor => "preprocessor", - })?; - Ok(()) - } -} - -const PREVIEW_IS_CODEBASE_HASH: i64 = -42; -const PREVIEW_IS_TAR_CODEBASE_HASH: i64 = -43; -const PREVIEW_IS_ESM_CODEBASE_HASH: i64 = -44; -const PREVIEW_IS_TAR_ESM_CODEBASE_HASH: i64 = -45; - -pub fn is_special_codebase_hash(hash: i64) -> bool { - hash == PREVIEW_IS_CODEBASE_HASH - || hash == PREVIEW_IS_TAR_CODEBASE_HASH - || hash == PREVIEW_IS_ESM_CODEBASE_HASH - || hash == PREVIEW_IS_TAR_ESM_CODEBASE_HASH -} - -pub fn codebase_to_hash(is_tar: bool, is_esm: bool) -> i64 { - if is_tar { - if is_esm { - PREVIEW_IS_TAR_ESM_CODEBASE_HASH - } else { - PREVIEW_IS_TAR_CODEBASE_HASH - } - } else { - if is_esm { - PREVIEW_IS_ESM_CODEBASE_HASH - } else { - PREVIEW_IS_CODEBASE_HASH - } - } -} - -pub fn hash_to_codebase_id(job_id: &str, hash: i64) -> Option { - match hash { - PREVIEW_IS_CODEBASE_HASH => Some(job_id.to_string()), - PREVIEW_IS_TAR_CODEBASE_HASH => Some(format!("{}.tar", job_id)), - PREVIEW_IS_ESM_CODEBASE_HASH => Some(format!("{}.esm", job_id)), - PREVIEW_IS_TAR_ESM_CODEBASE_HASH => Some(format!("{}.esm.tar", job_id)), - _ => None, - } -} - -pub struct CodebaseInfo { - pub is_tar: bool, - pub is_esm: bool, -} - -pub fn id_to_codebase_info(id: &str) -> CodebaseInfo { - let is_tar = id.ends_with(".tar"); - let is_esm = id.contains(".esm"); - CodebaseInfo { is_tar, is_esm } -} -#[derive(Serialize, sqlx::FromRow, Debug)] -pub struct Script { - pub workspace_id: String, - pub hash: ScriptHash, - pub path: String, - pub parent_hashes: Option, - pub summary: String, - pub description: String, - pub content: String, - pub created_by: String, - pub created_at: chrono::DateTime, - pub archived: bool, - pub schema: Option, - pub deleted: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub is_template: Option, - pub extra_perms: serde_json::Value, - #[serde(skip_serializing_if = "Option::is_none")] - pub lock: Option, - pub lock_error_logs: Option, - pub language: ScriptLang, - pub kind: ScriptKind, - pub tag: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub draft_only: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub envs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub dedicated_worker: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ws_error_handler_muted: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ttl: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ignore_s3_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub timeout: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub delete_after_use: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub restart_unless_cancelled: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub visible_to_runner_only: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub no_main_func: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub codebase: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub has_preprocessor: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_behalf_of_email: Option, - #[serde(skip_serializing_if = "Option::is_none")] - #[sqlx(json(nullable))] - pub assets: Option>, - #[serde(flatten)] - #[sqlx(flatten)] - pub runnable_settings: SR, -} - -// Not serializable -#[derive(sqlx::FromRow, Debug, Clone)] -pub struct ScriptRunnableSettingsHandle { - // legacy - for backwards compatibility - // don't add new values. - pub concurrency_key: Option, - pub concurrent_limit: Option, - pub concurrency_time_window_s: Option, - pub debounce_key: Option, - pub debounce_delay_s: Option, - - // add here as well. - pub runnable_settings_handle: Option, -} - -// Not sqlx queriable -#[derive(Serialize, Debug, Clone, Default)] -pub struct ScriptRunnableSettingsInline { - #[serde(flatten)] - pub concurrency_settings: ConcurrencySettings, - #[serde(flatten)] - pub debouncing_settings: DebouncingSettings, -} - -impl Script { - pub async fn prefetch_cached<'a>( - self, - db: &DB, - ) -> crate::error::Result> { - let (debouncing_settings, concurrency_settings) = - RunnableSettings::from_runnable_settings_handle( - self.runnable_settings.runnable_settings_handle, - db, - ) - .await? - .prefetch_cached(db) - .await?; - - Ok(Script { - workspace_id: self.workspace_id, - hash: self.hash, - path: self.path, - parent_hashes: self.parent_hashes, - summary: self.summary, - description: self.description, - content: self.content, - created_by: self.created_by, - created_at: self.created_at, - archived: self.archived, - schema: self.schema, - deleted: self.deleted, - is_template: self.is_template, - extra_perms: self.extra_perms, - lock: self.lock, - lock_error_logs: self.lock_error_logs, - language: self.language, - kind: self.kind, - tag: self.tag, - draft_only: self.draft_only, - envs: self.envs, - dedicated_worker: self.dedicated_worker, - ws_error_handler_muted: self.ws_error_handler_muted, - priority: self.priority, - cache_ttl: self.cache_ttl, - cache_ignore_s3_path: self.cache_ignore_s3_path, - timeout: self.timeout, - delete_after_use: self.delete_after_use, - restart_unless_cancelled: self.restart_unless_cancelled, - visible_to_runner_only: self.visible_to_runner_only, - no_main_func: self.no_main_func, - codebase: self.codebase, - has_preprocessor: self.has_preprocessor, - on_behalf_of_email: self.on_behalf_of_email, - assets: self.assets, - runnable_settings: ScriptRunnableSettingsInline { - concurrency_settings: concurrency_settings.maybe_fallback( - self.runnable_settings.concurrency_key, - self.runnable_settings.concurrent_limit, - self.runnable_settings.concurrency_time_window_s, - ), - debouncing_settings: debouncing_settings.maybe_fallback( - self.runnable_settings.debounce_key, - self.runnable_settings.debounce_delay_s, - ), - }, - }) - } -} - -#[derive(Serialize, sqlx::FromRow)] -pub struct ScriptWithStarred { - #[sqlx(flatten)] - #[serde(flatten)] - pub script: Script, - #[serde(skip_serializing_if = "Option::is_none")] - pub starred: Option, -} -impl ScriptWithStarred { - pub async fn prefetch_cached<'a>( - self, - db: &DB, - ) -> crate::error::Result> { - Ok(ScriptWithStarred { - script: self.script.prefetch_cached(db).await?, - starred: self.starred, - }) - } -} - -#[derive(Serialize, sqlx::FromRow)] -pub struct ListableScript { - pub hash: ScriptHash, - pub path: String, - pub summary: String, - pub created_at: chrono::DateTime, - pub archived: bool, - pub extra_perms: serde_json::Value, - pub language: ScriptLang, - pub starred: bool, - pub tag: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub has_draft: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub draft_only: Option, - pub has_deploy_errors: bool, - pub ws_error_handler_muted: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub no_main_func: Option, - #[serde(skip_serializing_if = "is_false")] - pub use_codebase: bool, - #[sqlx(default)] - #[serde(skip_serializing_if = "Option::is_none")] - pub deployment_msg: Option, - pub kind: ScriptKind, -} - -fn is_false(x: &bool) -> bool { - return !x; -} - -#[derive(Serialize)] -pub struct ScriptHistory { - pub script_hash: ScriptHash, - #[serde(skip_serializing_if = "Option::is_none")] - pub deployment_msg: Option, -} - -#[derive(Deserialize)] -pub struct ScriptHistoryUpdate { - pub deployment_msg: Option, -} - -#[derive(Serialize, Deserialize, Debug, sqlx::Type, Clone)] -#[sqlx(transparent)] -#[serde(transparent)] -pub struct Schema(pub sqlx::types::Json>); - -impl Hash for Schema { - fn hash(&self, state: &mut H) { - self.0.get().hash(state); - } -} - -#[derive(Serialize, Deserialize, Hash, Debug)] -pub struct NewScript { - pub path: String, - pub parent_hash: Option, - pub summary: String, - pub description: String, - pub content: String, - pub schema: Option, - pub is_template: Option, - #[serde(default = "Option::default")] - #[serde(deserialize_with = "lock_deserialize")] - pub lock: Option, - pub language: ScriptLang, - pub kind: Option, - pub tag: Option, - pub draft_only: Option, - pub envs: Option>, - #[serde(flatten)] - pub concurrency_settings: ConcurrencySettings, - #[serde(flatten)] - pub debouncing_settings: DebouncingSettings, - pub cache_ttl: Option, - pub cache_ignore_s3_path: Option, - pub dedicated_worker: Option, - pub ws_error_handler_muted: Option, - pub priority: Option, - pub timeout: Option, - pub delete_after_use: Option, - pub restart_unless_cancelled: Option, - pub deployment_message: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub visible_to_runner_only: Option, - pub no_main_func: Option, - pub codebase: Option, - pub has_preprocessor: Option, - pub on_behalf_of_email: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub assets: Option>, -} - -fn lock_deserialize<'de, D>(deserializer: D) -> Result, D::Error> -where - D: serde::de::Deserializer<'de>, -{ - struct StringOrArrayVisitor; - - impl<'de> serde::de::Visitor<'de> for StringOrArrayVisitor { - type Value = Option; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("either a string or an array of strings") - } - - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - Ok(Some(v.to_string())) - } - - fn visit_none(self) -> Result - where - E: serde::de::Error, - { - Ok(None) - } - - fn visit_unit(self) -> Result - where - E: serde::de::Error, - { - Ok(None) - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let mut split_lock: Vec = vec![]; - loop { - if let Ok(Some(elem)) = seq.next_element::() { - split_lock.push(elem); - } else { - break; - } - } - let lock = split_lock.join("\n"); - return Ok(Some(lock)); - } - } - deserializer.deserialize_any(StringOrArrayVisitor) -} - -#[derive(Debug, Deserialize)] -pub struct ListScriptQuery { - pub without_description: Option, - pub path_start: Option, - pub path_exact: Option, - pub created_by: Option, - pub first_parent_hash: Option, - pub last_parent_hash: Option, - pub parent_hash: Option, - pub show_archived: Option, - pub order_by: Option, - pub order_desc: Option, - pub is_template: Option, - pub kinds: Option, - pub starred_only: Option, - pub include_without_main: Option, - pub include_draft_only: Option, - pub with_deployment_msg: Option, - #[serde(default, deserialize_with = "from_seq")] - pub languages: Option>, - pub dedicated_worker: Option, -} - -fn from_seq<'de, D>(deserializer: D) -> Result>, D::Error> -where - D: Deserializer<'de>, -{ - let s = ::deserialize(deserializer)?; - - let languages: Vec = s - .split(",") - .map(ScriptLang::from_str) - .try_collect() - .map_err(|e| serde::de::Error::custom(e.to_string()))?; - - let languages = if languages.is_empty() { - None - } else { - Some(languages) - }; - - Ok(languages) -} - -pub fn to_i64(s: &str) -> crate::error::Result { - let v = hex::decode(s)?; - if v.len() < 8 { - return Err(crate::error::Error::BadRequest(format!( - "hex string did not decode to an u64: {s}", - ))); - } - let nb: u64 = u64::from_be_bytes( - v[0..8] - .try_into() - .map_err(|_| hex::FromHexError::InvalidStringLength)?, - ); - Ok(nb as i64) -} - -pub fn to_hex_string(i: &i64) -> String { - hex::encode(i.to_be_bytes()) +pub async fn prefetch_cached_script_with_starred( + sws: ScriptWithStarred, + db: &DB, +) -> crate::error::Result> { + Ok(ScriptWithStarred { + script: prefetch_cached_script(sws.script, db).await?, + starred: sws.starred, + }) } pub async fn get_hub_script_by_path( @@ -896,21 +301,6 @@ async fn get_full_hub_script_by_path_inner( Ok(script) } -#[derive(Deserialize, Serialize)] -pub struct HubScript { - pub content: String, - pub lockfile: Option, - pub language: ScriptLang, - pub schema: Box, - pub summary: Option, -} - -pub fn hash_script(ns: &NewScript) -> i64 { - let mut dh = std::hash::DefaultHasher::new(); - ns.hash(&mut dh); - dh.finish() as i64 -} - pub async fn fetch_script_for_update<'a>( path: &str, w_id: &str, @@ -989,14 +379,13 @@ pub async fn clone_script<'c>( ))); }; + let rs = runnable_settings::from_handle( + s.runnable_settings.runnable_settings_handle, + db, + ) + .await?; let (debouncing_settings, concurrency_settings) = - RunnableSettings::from_runnable_settings_handle( - s.runnable_settings.runnable_settings_handle, - db, - ) - .await? - .prefetch_cached(db) - .await?; + runnable_settings::prefetch_cached(&rs, db).await?; let ns = NewScript { path: s.path.clone(), diff --git a/backend/windmill-common/src/triggers.rs b/backend/windmill-common/src/triggers.rs index 6a5d2e4f45..59b827a8cb 100644 --- a/backend/windmill-common/src/triggers.rs +++ b/backend/windmill-common/src/triggers.rs @@ -1,101 +1,9 @@ use lazy_static::lazy_static; use quick_cache::sync::Cache; -use serde::{Deserialize, Serialize}; -use std::fmt; -use strum_macros::EnumIter; -use crate::jobs::JobTriggerKind; - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, EnumIter)] -#[sqlx(type_name = "TRIGGER_KIND", rename_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum TriggerKind { - Webhook, - Http, - Websocket, - Kafka, - DefaultEmail, - Email, - Nats, - Mqtt, - Sqs, - Postgres, - Gcp, - Nextcloud, -} - -impl TriggerKind { - pub fn to_key(&self) -> String { - match self { - TriggerKind::Webhook => "webhook".to_string(), - TriggerKind::Http => "http".to_string(), - TriggerKind::Websocket => "websocket".to_string(), - TriggerKind::Kafka => "kafka".to_string(), - TriggerKind::Email => "email".to_string(), - TriggerKind::DefaultEmail => "email".to_string(), // to the user we also show kind email for default email - TriggerKind::Nats => "nats".to_string(), - TriggerKind::Mqtt => "mqtt".to_string(), - TriggerKind::Sqs => "sqs".to_string(), - TriggerKind::Postgres => "postgres".to_string(), - TriggerKind::Gcp => "gcp".to_string(), - TriggerKind::Nextcloud => "nextcloud".to_string(), - } - } -} - -impl fmt::Display for TriggerKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s = match self { - TriggerKind::Webhook => "webhook", - TriggerKind::Http => "http", - TriggerKind::Websocket => "websocket", - TriggerKind::Kafka => "kafka", - TriggerKind::Email => "email", - TriggerKind::DefaultEmail => "default_email", - TriggerKind::Nats => "nats", - TriggerKind::Mqtt => "mqtt", - TriggerKind::Sqs => "sqs", - TriggerKind::Postgres => "postgres", - TriggerKind::Gcp => "gcp", - TriggerKind::Nextcloud => "nextcloud", - }; - write!(f, "{}", s) - } -} - -#[derive(Eq, PartialEq, Hash)] -pub enum HubOrWorkspaceId { - Hub, - WorkspaceId(String), -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] -pub struct RunnableFormat { - pub version: RunnableFormatVersion, - pub has_preprocessor: bool, -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] -pub enum RunnableFormatVersion { - V1, - V2, -} - -pub type RunnableFormatCacheKey = (HubOrWorkspaceId, i64, TriggerKind); +pub use windmill_types::triggers::*; lazy_static! { pub static ref RUNNABLE_FORMAT_VERSION_CACHE: Cache = Cache::new(1000); } - -#[derive(Debug, Clone)] -pub struct TriggerMetadata { - pub trigger_path: Option, - pub trigger_kind: JobTriggerKind, -} - -impl TriggerMetadata { - pub fn new(trigger_path: Option, trigger_kind: JobTriggerKind) -> TriggerMetadata { - TriggerMetadata { trigger_path, trigger_kind } - } -} diff --git a/backend/windmill-common/src/workspace_dependencies.rs b/backend/windmill-common/src/workspace_dependencies.rs index c8f29df555..68f20967f4 100644 --- a/backend/windmill-common/src/workspace_dependencies.rs +++ b/backend/windmill-common/src/workspace_dependencies.rs @@ -386,7 +386,7 @@ impl WorkspaceDependenciesPrefetched { Box::pin(async { let r = if let Some(wdar) = - language.extract_workspace_dependencies_annotated_refs(code, runnable_path) + crate::scripts::extract_workspace_dependencies_annotated_refs(&language, code, runnable_path) { tracing::debug!(workspace_id, ?language, "found explicit annotations"); diff --git a/backend/windmill-dep-map/src/lib.rs b/backend/windmill-dep-map/src/lib.rs index 43aceb1d38..39f2413bf7 100644 --- a/backend/windmill-dep-map/src/lib.rs +++ b/backend/windmill-dep-map/src/lib.rs @@ -88,7 +88,7 @@ pub fn extract_referenced_paths( ) -> Option> { let mut referenced_paths = vec![]; if let Some(wk_deps_refs) = language - .and_then(|l| l.extract_workspace_dependencies_annotated_refs(raw_code, script_path)) + .and_then(|l| windmill_common::scripts::extract_workspace_dependencies_annotated_refs(&l, raw_code, script_path)) .map(|r| r.external) { let l = language.expect("should be some"); diff --git a/backend/windmill-dep-map/src/scoped_dependency_map.rs b/backend/windmill-dep-map/src/scoped_dependency_map.rs index 49646524f0..4770fd516d 100644 --- a/backend/windmill-dep-map/src/scoped_dependency_map.rs +++ b/backend/windmill-dep-map/src/scoped_dependency_map.rs @@ -327,7 +327,7 @@ SELECT importer_node_id, imported_path )); } FlowModuleValue::FlowScript { .. } => { - return Err(Error::internal_err("FlowScript is not supposed to be in flow.")); + return Err(Error::internal_err("FlowScript is not supposed to be in flow.").into()); } _ => {} } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 281d93a435..763622a9c9 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -1175,7 +1175,7 @@ async fn commit_completed_job( } if completed_job.concurrent_limit.is_some() - || RunnableSettings::prefetch_cached_from_handle(completed_job.runnable_settings_handle, db) + || windmill_common::runnable_settings::prefetch_cached_from_handle(completed_job.runnable_settings_handle, db) .await? .1 .concurrent_limit @@ -2845,7 +2845,7 @@ impl PulledJobResult { }; let DebouncingSettings { debounce_delay_s, debounce_args_to_accumulate, .. } = - RunnableSettings::prefetch_cached_from_handle(j.runnable_settings_handle, db) + windmill_common::runnable_settings::prefetch_cached_from_handle(j.runnable_settings_handle, db) .await? .0; @@ -3136,9 +3136,7 @@ pub async fn pull( #[cfg(feature = "private")] let concurrency_settings = if let Some(ref j) = job { - RunnableSettings::from_runnable_settings_handle(j.runnable_settings_handle, db) - .await? - .prefetch_cached(db) + windmill_common::runnable_settings::prefetch_cached_from_handle(j.runnable_settings_handle, db) .await? .1 .maybe_fallback(None, j.concurrent_limit, j.concurrency_time_window_s) @@ -3206,9 +3204,7 @@ pub async fn pull( }; let concurrency_settings = - RunnableSettings::from_runnable_settings_handle(job.runnable_settings_handle, db) - .await? - .prefetch_cached(db) + windmill_common::runnable_settings::prefetch_cached_from_handle(job.runnable_settings_handle, db) .await? .1 .maybe_fallback(None, job.concurrent_limit, job.concurrency_time_window_s); @@ -5371,11 +5367,10 @@ pub async fn push<'c, 'd>( (job_kind, scheduled_for_o) }; - let runnable_settings_handle = RunnableSettings { + let runnable_settings_handle = windmill_common::runnable_settings::insert_rs(RunnableSettings { debouncing_settings: debouncing_settings.insert_cached(_db).await?, concurrency_settings: concurrency_settings.insert_cached(_db).await?, - } - .insert_cached(_db) + }, _db) .await?; let (guarded_concurrent_limit, guarded_concurrency_time_window_s) = diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 09c4cd11f1..4528521894 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -24,7 +24,6 @@ use windmill_common::jobs::JobPayload; use windmill_common::jobs::JobTriggerKind; use windmill_common::runnable_settings::ConcurrencySettings; use windmill_common::runnable_settings::DebouncingSettings; -use windmill_common::runnable_settings::RunnableSettings; use windmill_common::schedule::schedule_to_user; use windmill_common::scripts::ScriptHash; use windmill_common::triggers::TriggerMetadata; @@ -345,9 +344,7 @@ pub async fn push_scheduled_job<'c>( .await?; let (debouncing_settings, concurrency_settings) = - RunnableSettings::from_runnable_settings_handle(runnable_settings_handle, db) - .await? - .prefetch_cached(db) + windmill_common::runnable_settings::prefetch_cached_from_handle(runnable_settings_handle, db) .await?; if schedule.retry.is_some() { diff --git a/backend/windmill-types/Cargo.toml b/backend/windmill-types/Cargo.toml new file mode 100644 index 0000000000..4eb18fe88e --- /dev/null +++ b/backend/windmill-types/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "windmill-types" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_types" +path = "src/lib.rs" + +[dependencies] +serde.workspace = true +serde_json.workspace = true +chrono.workspace = true +uuid.workspace = true +sqlx = { workspace = true, features = ["postgres"] } +rand.workspace = true +hex.workspace = true +anyhow.workspace = true +tracing.workspace = true +itertools.workspace = true +strum.workspace = true diff --git a/backend/windmill-types/src/apps.rs b/backend/windmill-types/src/apps.rs new file mode 100644 index 0000000000..1008396bb4 --- /dev/null +++ b/backend/windmill-types/src/apps.rs @@ -0,0 +1,30 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use crate::scripts::ScriptLang; + +/// Id in the `app_script` table. +#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash, Eq, PartialEq)] +#[serde(transparent)] +pub struct AppScriptId(pub i64); + +#[derive(Deserialize)] +pub struct ListAppQuery { + pub starred_only: Option, + pub path_exact: Option, + pub path_start: Option, + pub include_draft_only: Option, + pub with_deployment_msg: Option, +} + +#[derive(Deserialize)] +pub struct RawAppValue { + pub files: HashMap, +} + +pub struct AppInlineScript { + pub language: Option, + pub content: String, + pub lock: Option, +} diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs new file mode 100644 index 0000000000..5f75388410 --- /dev/null +++ b/backend/windmill-types/src/assets.rs @@ -0,0 +1,83 @@ +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive( + Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type, PartialOrd, Ord, +)] +#[sqlx(type_name = "ASSET_KIND", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum AssetKind { + S3Object, + Resource, + // Avoid unnexpected crashes when deserializing old assets + Variable, // Deprecated + Ducklake, + DataTable, +} + +#[derive( + Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type, PartialOrd, Ord, +)] +#[sqlx(type_name = "ASSET_USAGE_KIND", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum AssetUsageKind { + Script, + Flow, + Job, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] +#[sqlx(type_name = "ASSET_ACCESS_TYPE", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum AssetUsageAccessType { + R, + W, + RW, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Hash)] +pub struct AssetWithAltAccessType { + pub path: String, + pub kind: AssetKind, + pub access_type: Option, + pub alt_access_type: Option, + /// Map of column name to access type for column-level access tracking + #[serde(skip_serializing_if = "Option::is_none")] + pub columns: Option>, +} + +pub fn merge_asset_usage_access_types( + a: Option, + b: Option, +) -> Option { + use AssetUsageAccessType::*; + match (a, b) { + (None, _) | (_, None) => None, + (Some(R), Some(W)) | (Some(W), Some(R)) => Some(RW), + (Some(RW), _) | (_, Some(RW)) => Some(RW), + (Some(R), Some(R)) => Some(R), + (Some(W), Some(W)) => Some(W), + } +} + +pub fn merge_asset_columns( + a: &Option>, + b: &Option>, +) -> Option> { + match (a, b) { + (None, None) => None, + (Some(cols), None) | (None, Some(cols)) => Some(cols.clone()), + (Some(cols_a), Some(cols_b)) => { + let mut merged = cols_a.clone(); + for (col, access_b) in cols_b { + let access_a = merged.get(col); + let merged_access = + merge_asset_usage_access_types(access_a.cloned(), Some(*access_b)); + if let Some(access) = merged_access { + merged.insert(col.clone(), access); + } + } + Some(merged) + } + } +} diff --git a/backend/windmill-types/src/flow_status.rs b/backend/windmill-types/src/flow_status.rs new file mode 100644 index 0000000000..6306141ab9 --- /dev/null +++ b/backend/windmill-types/src/flow_status.rs @@ -0,0 +1,531 @@ +use std::collections::HashMap; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::flows::FlowValue; + +const MINUTES: Duration = Duration::from_secs(60); +const HOURS: Duration = MINUTES.saturating_mul(60); + +pub const MAX_RETRY_ATTEMPTS: u32 = u32::MAX; +pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6); + +pub fn is_retry_default(v: &RetryStatus) -> bool { + v.fail_count == 0 && v.failed_jobs.is_empty() +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct FlowStatus { + pub step: i32, + pub modules: Vec, + pub failure_module: Box, + pub preprocessor_module: Option, + + #[serde(skip_serializing_if = "HashMap::is_empty")] + #[serde(default)] + pub user_states: HashMap, + #[serde(default)] + pub cleanup_module: FlowCleanupModule, + #[serde(default)] + #[serde(skip_serializing_if = "is_retry_default")] + pub retry: RetryStatus, + #[serde(skip_serializing_if = "Option::is_none")] + pub approval_conditions: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub restarted_from: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_job: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_input_enabled: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub memory_id: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default)] +pub struct RetryStatus { + pub fail_count: u32, + pub failed_jobs: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default)] +pub struct ApprovalConditions { + pub user_auth_required: bool, + pub user_groups_required: Vec, + pub self_approval_disabled: bool, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default)] +pub struct RestartedFrom { + pub flow_job_id: Uuid, + pub step_id: String, + pub branch_or_iteration_n: Option, + pub flow_version: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Iterator { + pub index: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub itered: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub itered_len: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct BranchAllStatus { + pub branch: usize, + pub len: usize, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde( + tag = "type", + rename_all(serialize = "lowercase", deserialize = "lowercase") +)] +pub enum BranchChosen { + Default, + Branch { branch: usize }, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Approval { + pub resume_id: u16, + pub approver: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct FlowStatusModuleWParent { + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_module: Option, + #[serde(flatten)] + pub module_status: FlowStatusModule, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +pub struct FlowCleanupModule { + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub flow_jobs_to_clean: Vec, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct FlowJobsDuration { + pub started_at: Vec>>, + pub duration_ms: Vec>, +} + +impl FlowJobsDuration { + pub fn set(&mut self, position: Option, value: &Option) { + if let Some(position) = position { + if position >= self.started_at.len() + || position >= self.duration_ms.len() + || value.is_none() + { + return; + } + let value = value.clone().unwrap(); + self.started_at[position] = Some(value.started_at); + self.duration_ms[position] = Some(value.duration_ms); + } + } + + pub fn push(&mut self, value: &Option) { + self.started_at.push(value.as_ref().map(|x| x.started_at)); + self.duration_ms.push(value.as_ref().map(|x| x.duration_ms)); + } + + pub fn new(n: usize) -> Self { + Self { started_at: vec![None; n], duration_ms: vec![None; n] } + } + + pub fn truncate(&mut self, n: usize) { + self.started_at.truncate(n); + self.duration_ms.truncate(n); + } +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct FlowJobDuration { + pub started_at: chrono::DateTime, + pub duration_ms: i64, +} + +#[derive(Deserialize)] +struct UntaggedFlowStatusModule { + #[serde(rename = "type")] + type_: String, + id: Option, + count: Option, + progress: Option, + job: Option, + iterator: Option, + flow_jobs: Option>, + flow_jobs_success: Option>>, + flow_jobs_duration: Option, + branch_chosen: Option, + branchall: Option, + parallel: Option, + while_loop: Option, + approvers: Option>, + failed_retries: Option>, + skipped: Option, + agent_actions: Option>, + agent_actions_success: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum AgentAction { + ToolCall { + job_id: uuid::Uuid, + function_name: String, + module_id: String, + }, + McpToolCall { + call_id: uuid::Uuid, + function_name: String, + resource_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + arguments: Option, + }, + Message {}, + WebSearch {}, +} + +#[derive(Serialize, Debug, Clone)] +#[serde(tag = "type")] +pub enum FlowStatusModule { + WaitingForPriorSteps { + id: String, + }, + WaitingForEvents { + id: String, + count: u16, + job: Uuid, + }, + WaitingForExecutor { + id: String, + job: Uuid, + }, + InProgress { + id: String, + job: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + progress: Option, + #[serde(skip_serializing_if = "Option::is_none")] + iterator: Option, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_success: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_duration: Option, + #[serde(skip_serializing_if = "Option::is_none")] + branch_chosen: Option, + #[serde(skip_serializing_if = "Option::is_none")] + branchall: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + parallel: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + while_loop: bool, + #[serde(skip_serializing_if = "Option::is_none")] + agent_actions: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + agent_actions_success: Option>, + }, + Success { + id: String, + job: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_success: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_duration: Option, + #[serde(skip_serializing_if = "Option::is_none")] + branch_chosen: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + approvers: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + failed_retries: Vec, + skipped: bool, + #[serde(skip_serializing_if = "Option::is_none")] + agent_actions: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + agent_actions_success: Option>, + }, + Failure { + id: String, + job: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_success: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_duration: Option, + #[serde(skip_serializing_if = "Option::is_none")] + branch_chosen: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + failed_retries: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + agent_actions: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + agent_actions_success: Option>, + }, +} + +impl<'de> Deserialize<'de> for FlowStatusModule { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let untagged: UntaggedFlowStatusModule = + UntaggedFlowStatusModule::deserialize(deserializer)?; + + match untagged.type_.as_str() { + "WaitingForPriorSteps" => Ok(FlowStatusModule::WaitingForPriorSteps { + id: untagged + .id + .ok_or_else(|| serde::de::Error::missing_field("id"))?, + }), + "WaitingForEvents" => Ok(FlowStatusModule::WaitingForEvents { + id: untagged + .id + .ok_or_else(|| serde::de::Error::missing_field("id"))?, + count: untagged + .count + .ok_or_else(|| serde::de::Error::missing_field("count"))?, + job: untagged + .job + .ok_or_else(|| serde::de::Error::missing_field("job"))?, + }), + "WaitingForExecutor" => Ok(FlowStatusModule::WaitingForExecutor { + id: untagged + .id + .ok_or_else(|| serde::de::Error::missing_field("id"))?, + job: untagged + .job + .ok_or_else(|| serde::de::Error::missing_field("job"))?, + }), + "InProgress" => Ok(FlowStatusModule::InProgress { + id: untagged + .id + .ok_or_else(|| serde::de::Error::missing_field("id"))?, + job: untagged + .job + .ok_or_else(|| serde::de::Error::missing_field("job"))?, + iterator: untagged.iterator, + flow_jobs: untagged.flow_jobs, + flow_jobs_success: untagged.flow_jobs_success, + flow_jobs_duration: untagged.flow_jobs_duration, + branch_chosen: untagged.branch_chosen, + branchall: untagged.branchall, + parallel: untagged.parallel.unwrap_or(false), + while_loop: untagged.while_loop.unwrap_or(false), + progress: untagged.progress, + agent_actions: untagged.agent_actions, + agent_actions_success: untagged.agent_actions_success, + }), + "Success" => Ok(FlowStatusModule::Success { + id: untagged + .id + .ok_or_else(|| serde::de::Error::missing_field("id"))?, + job: untagged + .job + .ok_or_else(|| serde::de::Error::missing_field("job"))?, + flow_jobs: untagged.flow_jobs, + flow_jobs_success: untagged.flow_jobs_success, + flow_jobs_duration: untagged.flow_jobs_duration, + branch_chosen: untagged.branch_chosen, + approvers: untagged.approvers.unwrap_or_default(), + failed_retries: untagged.failed_retries.unwrap_or_default(), + skipped: untagged.skipped.unwrap_or(false), + agent_actions: untagged.agent_actions, + agent_actions_success: untagged.agent_actions_success, + }), + "Failure" => Ok(FlowStatusModule::Failure { + id: untagged + .id + .ok_or_else(|| serde::de::Error::missing_field("id"))?, + job: untagged + .job + .ok_or_else(|| serde::de::Error::missing_field("job"))?, + flow_jobs: untagged.flow_jobs, + flow_jobs_success: untagged.flow_jobs_success, + flow_jobs_duration: untagged.flow_jobs_duration, + branch_chosen: untagged.branch_chosen, + failed_retries: untagged.failed_retries.unwrap_or_default(), + agent_actions: untagged.agent_actions, + agent_actions_success: untagged.agent_actions_success, + }), + other => Err(serde::de::Error::unknown_variant( + other, + &[ + "WaitingForPriorSteps", + "WaitingForEvents", + "WaitingForExecutor", + "InProgress", + "Success", + "Failure", + ], + )), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum JobResult { + SingleJob(Uuid), + ListJob(Vec), +} + +impl FlowStatusModule { + pub fn job(&self) -> Option { + match self { + FlowStatusModule::WaitingForPriorSteps { .. } => None, + FlowStatusModule::WaitingForEvents { job, .. } => Some(*job), + FlowStatusModule::WaitingForExecutor { job, .. } => Some(*job), + FlowStatusModule::InProgress { job, .. } => Some(*job), + FlowStatusModule::Success { job, .. } => Some(*job), + FlowStatusModule::Failure { job, .. } => Some(*job), + } + } + + pub fn flow_jobs(&self) -> Option> { + match self { + FlowStatusModule::InProgress { flow_jobs, .. } => flow_jobs.clone(), + FlowStatusModule::Success { flow_jobs, .. } => flow_jobs.clone(), + FlowStatusModule::Failure { flow_jobs, .. } => flow_jobs.clone(), + _ => None, + } + } + + pub fn branch_chosen(&self) -> Option { + match self { + FlowStatusModule::InProgress { branch_chosen, .. } => branch_chosen.clone(), + FlowStatusModule::Success { branch_chosen, .. } => branch_chosen.clone(), + FlowStatusModule::Failure { branch_chosen, .. } => branch_chosen.clone(), + _ => None, + } + } + + pub fn flow_jobs_success(&self) -> Option>> { + match self { + FlowStatusModule::InProgress { flow_jobs_success, .. } => flow_jobs_success.clone(), + FlowStatusModule::Success { flow_jobs_success, .. } => flow_jobs_success.clone(), + FlowStatusModule::Failure { flow_jobs_success, .. } => flow_jobs_success.clone(), + _ => None, + } + } + + pub fn flow_jobs_duration(&self) -> Option { + match self { + FlowStatusModule::InProgress { flow_jobs_duration, .. } => flow_jobs_duration.clone(), + FlowStatusModule::Success { flow_jobs_duration, .. } => flow_jobs_duration.clone(), + FlowStatusModule::Failure { flow_jobs_duration, .. } => flow_jobs_duration.clone(), + _ => None, + } + } + + pub fn job_result(&self) -> Option { + self.flow_jobs() + .map(JobResult::ListJob) + .or_else(|| self.job().map(JobResult::SingleJob)) + } + + pub fn id(&self) -> String { + match self { + FlowStatusModule::WaitingForPriorSteps { id, .. } => id.clone(), + FlowStatusModule::WaitingForEvents { id, .. } => id.clone(), + FlowStatusModule::WaitingForExecutor { id, .. } => id.clone(), + FlowStatusModule::InProgress { id, .. } => id.clone(), + FlowStatusModule::Success { id, .. } => id.clone(), + FlowStatusModule::Failure { id, .. } => id.clone(), + } + } + + pub fn is_failure(&self) -> bool { + match self { + FlowStatusModule::Failure { .. } => true, + _ => false, + } + } + + pub fn agent_actions(&self) -> Option> { + match self { + FlowStatusModule::InProgress { agent_actions, .. } => agent_actions.clone(), + FlowStatusModule::Success { agent_actions, .. } => agent_actions.clone(), + FlowStatusModule::Failure { agent_actions, .. } => agent_actions.clone(), + _ => None, + } + } + + pub fn agent_actions_success(&self) -> Option> { + match self { + FlowStatusModule::InProgress { agent_actions_success, .. } => { + agent_actions_success.clone() + } + FlowStatusModule::Success { agent_actions_success, .. } => { + agent_actions_success.clone() + } + FlowStatusModule::Failure { agent_actions_success, .. } => { + agent_actions_success.clone() + } + _ => None, + } + } +} + +impl FlowStatus { + pub fn new(f: &FlowValue) -> Self { + Self { + step: if f.preprocessor_module.is_some() { + -1 + } else { + 0 + }, + approval_conditions: None, + modules: f + .modules + .iter() + .map(|m| FlowStatusModule::WaitingForPriorSteps { id: m.id.clone() }) + .collect(), + failure_module: Box::new(FlowStatusModuleWParent { + parent_module: None, + module_status: FlowStatusModule::WaitingForPriorSteps { + id: f + .failure_module + .as_ref() + .map(|x| x.id.clone()) + .unwrap_or_else(|| "failure".to_string()), + }, + }), + preprocessor_module: if f.preprocessor_module.is_some() { + Some(FlowStatusModule::WaitingForPriorSteps { + id: f.preprocessor_module.as_ref().unwrap().id.clone(), + }) + } else { + None + }, + cleanup_module: FlowCleanupModule { flow_jobs_to_clean: vec![] }, + retry: RetryStatus { fail_count: 0, failed_jobs: vec![] }, + restarted_from: None, + user_states: HashMap::new(), + stream_job: None, + chat_input_enabled: f.chat_input_enabled, + memory_id: None, + } + } + + /// current module status ... excluding failure_module + pub fn current_step(&self) -> Option<&FlowStatusModule> { + let i = usize::try_from(self.step).ok()?; + self.modules.get(i) + } +} diff --git a/backend/windmill-types/src/flows.rs b/backend/windmill-types/src/flows.rs new file mode 100644 index 0000000000..a1a2038452 --- /dev/null +++ b/backend/windmill-types/src/flows.rs @@ -0,0 +1,1123 @@ +use std::{ + collections::{BTreeMap, HashMap}, + time::Duration, +}; + +use rand::Rng; +use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::value::RawValue; +use sqlx::types::Json; +use sqlx::types::JsonRawValue; + +use crate::{ + assets::AssetWithAltAccessType, + more_serde::{default_empty_string, default_id, default_null, default_true, is_default}, + runnable_settings::{ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings}, + scripts::{Schema, ScriptHash, ScriptLang}, + to_raw_value, +}; + +#[derive(Debug, Serialize, Deserialize, sqlx::FromRow)] +pub struct Flow { + pub workspace_id: String, + pub path: String, + pub summary: String, + pub description: String, + pub value: Json>, + pub edited_by: String, + pub edited_at: chrono::DateTime, + pub archived: bool, + pub schema: Option, + pub extra_perms: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub dedicated_worker: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tag: Option, + #[serde(skip_serializing_if = "is_none_or_false")] + pub ws_error_handler_muted: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option, + #[serde(skip_serializing_if = "is_none_or_false")] + pub visible_to_runner_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_behalf_of_email: Option, +} + +#[derive(Serialize, sqlx::FromRow)] +pub struct FlowWithStarred { + #[sqlx(flatten)] + #[serde(flatten)] + pub flow: Flow, + #[serde(skip_serializing_if = "Option::is_none")] + pub starred: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub lock_error_logs: Option, + pub version_id: i64, +} + +pub fn is_none_or_false(b: &Option) -> bool { + b.is_none() || !b.unwrap() +} + +#[derive(Serialize, sqlx::FromRow)] +pub struct ListableFlow { + pub workspace_id: String, + pub path: String, + pub summary: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + pub edited_by: Option, + pub edited_at: Option>, + pub archived: bool, + pub extra_perms: serde_json::Value, + pub starred: bool, + pub has_draft: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ws_error_handler_muted: Option, + #[sqlx(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub deployment_msg: Option, +} + +#[derive(Debug, Serialize, Deserialize, sqlx::FromRow)] +pub struct NewFlow { + pub path: String, + pub summary: String, + pub description: Option, + #[serde(deserialize_with = "validate_flow_value")] + pub value: Box, + pub schema: Option, + pub draft_only: Option, + pub tag: Option, + pub dedicated_worker: Option, + pub timeout: Option, + pub deployment_message: Option, + pub visible_to_runner_only: Option, + pub on_behalf_of_email: Option, + pub ws_error_handler_muted: Option, +} + +impl NewFlow { + pub fn parse_flow_value(&self) -> anyhow::Result { + serde_json::from_str(self.value.get()) + .map_err(|e| anyhow::anyhow!("Failed to parse flow value: {}", e)) + } +} + +fn validate_retry(retry: &Retry, module_id: &str) -> anyhow::Result<()> { + if retry.exponential.attempts > 0 && retry.exponential.seconds == 0 { + return Err(anyhow::anyhow!( + "Module '{}': Exponential backoff base (seconds) must be greater than 0. A base of 0 would cause immediate retries.", + module_id + )); + } + Ok(()) +} + +fn validate_flow_value<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let raw_value = Box::::deserialize(deserializer)?; + + let flow_value: FlowValue = serde_json::from_str(raw_value.get()) + .map_err(|e| serde::de::Error::custom(format!("Invalid flow value: {}", e)))?; + + FlowModule::traverse_modules(&flow_value.modules, &mut |module| { + if let Some(ref retry) = module.retry { + validate_retry(retry, &module.id)?; + } + return Ok(()); + }) + .map_err(|e| serde::de::Error::custom(e.to_string()))?; + + if let Some(ref _failure_module) = flow_value.failure_module { + //add validation logic here for failure module + } + + if let Some(ref _preprocessor_module) = flow_value.preprocessor_module { + //add validation logic here for preprocessor module + } + + Ok(raw_value) +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct FlowValue { + pub modules: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub failure_module: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub preprocessor_module: Option>, + #[serde(default)] + #[serde(skip_serializing_if = "is_default")] + pub same_worker: bool, + #[serde(flatten)] + pub concurrency_settings: ConcurrencySettings, + #[serde(flatten)] + pub debouncing_settings: DebouncingSettings, + #[serde(skip_serializing_if = "Option::is_none")] + pub skip_expr: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ttl: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ignore_s3_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub early_return: Option, + #[serde(skip_serializing_if = "Option::is_none")] + // Priority at the flow level + pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_input_enabled: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_env: Option>>, +} + +impl FlowValue { + pub fn get_flow_module_at_step(&self, step: Step) -> anyhow::Result<&FlowModule> { + use anyhow::Context; + let flow_module = match step { + Step::PreprocessorStep => self + .preprocessor_module + .as_deref() + .with_context(|| format!("no preprocessor module")), + Step::Step { idx, .. } => self + .modules + .get(idx) + .with_context(|| format!("no module found at index: {idx}")), + Step::FailureStep => self + .failure_module + .as_deref() + .with_context(|| format!("no failure module")), + }; + + flow_module + } + + pub fn traverse_leafs anyhow::Result<()>>( + modules: Vec<&FlowModule>, + cb: &mut C, + ) -> anyhow::Result<()> { + use FlowModuleValue::*; + for module in modules { + match serde_json::from_str::(module.value.get())? { + s @ (Script { .. } + | RawScript { .. } + | Flow { .. } + | FlowScript { .. } + | Identity) => cb(&s, &module.id)?, + ForloopFlow { modules, .. } | WhileloopFlow { modules, .. } => { + Self::traverse_leafs(modules.iter().collect(), cb)? + } + AIAgent { tools, .. } => { + for tool in tools { + match &tool.value { + ToolValue::FlowModule(module_value) => cb(module_value, &tool.id)?, + ToolValue::Mcp(_) => {} + ToolValue::Websearch(_) => {} + } + } + } + BranchOne { default, branches, .. } => { + Self::traverse_leafs(default.iter().collect(), cb)?; + for branch in branches { + Self::traverse_leafs(branch.modules.iter().collect(), cb)?; + } + } + BranchAll { branches, .. } => { + for branch in branches { + Self::traverse_leafs(branch.modules.iter().collect(), cb)?; + } + } + } + } + Ok(()) + } +} + +#[derive(Debug, Copy, Clone)] +pub enum Step { + Step { idx: usize, len: usize }, + PreprocessorStep, + FailureStep, +} + +impl Step { + pub fn from_i32_and_len(step: i32, len: usize) -> Self { + if step < 0 { + Step::PreprocessorStep + } else if (step as usize) < len { + Step::Step { idx: step as usize, len } + } else { + Step::FailureStep + } + } + + pub fn get_step_index(&self) -> Option { + match self { + Step::Step { idx, .. } => Some(*idx), + _ => None, + } + } + + pub fn is_index_step(&self) -> bool { + matches!(self, Step::Step { .. }) + } + + pub fn is_preprocessor_step(&self) -> bool { + matches!(self, Step::PreprocessorStep) + } + + pub fn is_failure_step(&self) -> bool { + matches!(self, Step::FailureStep) + } + + pub fn is_last_step(&self) -> bool { + matches!(self, Step::Step { idx, len } if *idx == len - 1) + } +} + +#[derive(Default, Deserialize, Serialize, Debug, Clone)] +pub struct StopAfterIf { + pub expr: String, + pub skip_if_stopped: bool, + pub error_message: Option, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] +pub struct RetryIf { + pub expr: String, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] +#[serde(default)] +pub struct Retry { + pub constant: ConstantDelay, + pub exponential: ExponentialDelay, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry_if: Option, +} + +impl Retry { + pub fn interval(&self, previous_attempts: u32, silent: bool) -> Option { + let Self { constant, exponential, .. } = self; + + if previous_attempts < constant.attempts { + Some(Duration::from_secs(constant.seconds as u64)) + } else if previous_attempts - constant.attempts < exponential.attempts { + let exp = previous_attempts.saturating_add(1) as u32; + let mut secs = exponential.multiplier * exponential.seconds.saturating_pow(exp); + if let Some(random_factor) = exponential.random_factor { + if random_factor > 0 { + let random_component = + rand::rng().random_range(0..(std::cmp::min(random_factor, 100) as u16)); + secs = match rand::rng().random_bool(1.0 / 2.0) { + true => secs.saturating_add(secs * random_component / 100), + false => secs.saturating_sub(secs * random_component / 100), + }; + } + } + if !silent { + tracing::warn!("Rescheduling job in {} seconds due to failure", secs); + } + Some(Duration::from_secs(secs as u64)) + } else { + None + } + } + + pub fn has_attempts(&self) -> bool { + self.constant.attempts != 0 || self.exponential.attempts != 0 + } + + pub fn max_attempts(&self) -> u32 { + self.constant + .attempts + .saturating_add(self.exponential.attempts) + } + + pub fn max_interval(&self) -> Option { + self.max_attempts() + .checked_sub(1) + .and_then(|p| self.interval(p, true)) + } +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] +#[serde(default)] +pub struct ConstantDelay { + pub attempts: u32, + pub seconds: u16, +} + +/// multiplier * seconds ^ failures (+/- jitter of the previous value, if any) +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[serde(default)] +pub struct ExponentialDelay { + pub attempts: u32, + pub multiplier: u16, + pub seconds: u16, + pub random_factor: Option, +} + +impl Default for ExponentialDelay { + fn default() -> Self { + Self { attempts: 0, multiplier: 1, seconds: 0, random_factor: None } + } +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct Suspend { + #[serde(skip_serializing_if = "Option::is_none")] + pub required_events: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub resume_form: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub user_auth_required: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub user_groups_required: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub self_approval_disabled: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub hide_cancel: Option, + #[serde(skip_serializing_if = "false_or_empty")] + pub continue_on_disapprove_timeout: Option, +} + +fn false_or_empty(v: &Option) -> bool { + v.is_none() || v.as_ref().is_some_and(|x| !x) +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct Mock { + pub enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub return_value: Option, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct FlowModule { + #[serde(default = "default_id")] + pub id: String, + pub value: Box, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_after_if: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_after_all_iters_if: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub suspend: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mock: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub sleep: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ttl: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ignore_s3_path: Option, + #[serde( + default, + deserialize_with = "raw_value_to_input_transform::<_, i32>", + skip_serializing_if = "Option::is_none" + )] + pub timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub delete_after_use: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub continue_on_error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub skip_if: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub apply_preprocessor: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub pass_flow_input_directly: Option, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct SkipIf { + pub expr: String, +} + +#[derive(Deserialize)] +pub struct FlowModuleValueWithParallel { + #[serde(rename = "type")] + pub type_: String, + pub parallel: Option, + #[serde( + default, + deserialize_with = "raw_value_to_input_transform::<_, u16>", + skip_serializing_if = "Option::is_none" + )] + pub parallelism: Option, +} + +#[derive(Deserialize)] +pub struct FlowModuleValueWithSkipFailures { + pub skip_failures: Option, + pub parallel: Option, + #[serde( + default, + deserialize_with = "raw_value_to_input_transform::<_, u16>", + skip_serializing_if = "Option::is_none" + )] + pub parallelism: Option, +} + +#[derive(Deserialize)] +pub struct BranchWithSkipFailures { + pub skip_failure: Option, +} + +#[derive(Deserialize)] +pub struct FlowModuleWithBranches { + pub branches: Vec, +} + +impl FlowModule { + pub fn id_append(&mut self, s: &str) { + self.id = format!("{}-{}", self.id, s); + } + pub fn get_value(&self) -> anyhow::Result { + serde_json::from_str::(self.value.get()) + .map_err(|e| anyhow::anyhow!("{}", e)) + } + + pub fn get_value_with_skip_failures(&self) -> anyhow::Result { + serde_json::from_str::(self.value.get()) + .map_err(|e| anyhow::anyhow!("{}", e)) + } + + pub fn get_branches_skip_failures(&self) -> anyhow::Result { + serde_json::from_str::(self.value.get()) + .map_err(|e| anyhow::anyhow!("{}", e)) + } + + pub fn is_flow(&self) -> bool { + self.get_type().is_ok_and(|x| x == "flow") + } + + pub fn get_value_with_parallel(&self) -> anyhow::Result { + serde_json::from_str::(self.value.get()) + .map_err(|e| anyhow::anyhow!("{}", e)) + } + + pub fn is_ai_agent(&self) -> bool { + self.get_type().is_ok_and(|x| x == "aiagent") + } + + pub fn is_simple(&self) -> bool { + self.get_type() + .is_ok_and(|x| x == "script" || x == "rawscript" || x == "flowscript") + } + + pub fn get_type(&self) -> anyhow::Result<&str> { + #[derive(Deserialize)] + pub struct FlowModuleValueType<'a> { + pub r#type: &'a str, + } + + serde_json::from_str::(self.value.get()) + .map_err(|e| anyhow::anyhow!("{}", e)) + .map(|x| x.r#type) + } + + pub fn traverse_modules anyhow::Result<()>>( + modules: &Vec, + cb: &mut C, + ) -> anyhow::Result<()> { + for module in modules { + cb(module)?; + match module + .get_value() + .map_err(|e| anyhow::anyhow!("Module '{}': {}", module.id, e))? + { + FlowModuleValue::ForloopFlow { modules, .. } + | FlowModuleValue::WhileloopFlow { modules, .. } => { + Self::traverse_modules(&modules, cb)?; + } + FlowModuleValue::BranchOne { branches, default, .. } => { + for branch in branches { + Self::traverse_modules(&branch.modules, cb)?; + } + Self::traverse_modules(&default, cb)?; + } + FlowModuleValue::BranchAll { branches, .. } => { + for branch in branches { + Self::traverse_modules(&branch.modules, cb)?; + } + } + FlowModuleValue::AIAgent { tools, .. } => { + for tool in tools { + match &tool.value { + ToolValue::FlowModule(module_value) => match module_value { + FlowModuleValue::ForloopFlow { modules, .. } + | FlowModuleValue::WhileloopFlow { modules, .. } => { + Self::traverse_modules(&modules, cb)?; + } + FlowModuleValue::BranchOne { branches, default, .. } => { + for branch in branches { + Self::traverse_modules(&branch.modules, cb)?; + } + Self::traverse_modules(&default, cb)?; + } + FlowModuleValue::BranchAll { branches, .. } => { + for branch in branches { + Self::traverse_modules(&branch.modules, cb)?; + } + } + _ => {} + }, + ToolValue::Mcp(_) => {} + ToolValue::Websearch(_) => {} + } + } + } + _ => {} + } + } + Ok(()) + } +} + +#[derive(Deserialize)] +pub struct UntaggedInputTransform { + #[serde(rename = "type")] + pub type_: String, + pub value: Option>, + pub expr: Option, +} + +impl<'de> Deserialize<'de> for InputTransform { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let untagged: UntaggedInputTransform = UntaggedInputTransform::deserialize(deserializer)?; + + let input_transform = TryInto::::try_into(untagged) + .map_err(|e| serde::de::Error::custom(e))?; + + Ok(input_transform) + } +} + +#[derive(Serialize, Debug, Clone)] +#[serde( + tag = "type", + rename_all(serialize = "lowercase", deserialize = "lowercase") +)] +pub enum InputTransform { + Static { + #[serde(default = "default_null")] + value: Box, + }, + Javascript { + #[serde(default = "default_empty_string")] + expr: String, + }, + Ai, +} + +impl InputTransform { + pub fn new_static_value(value: Box) -> InputTransform { + InputTransform::Static { value } + } + + pub fn new_javascript_expr(expr: &str) -> InputTransform { + InputTransform::Javascript { expr: expr.to_owned() } + } +} + +impl TryFrom for InputTransform { + type Error = anyhow::Error; + fn try_from(value: UntaggedInputTransform) -> Result { + let input_transform = match value.type_.as_str() { + "static" => InputTransform::new_static_value(value.value.unwrap_or_else(default_null)), + "javascript" => InputTransform::new_javascript_expr(&value.expr.unwrap_or_default()), + "ai" => InputTransform::Ai, + other => { + return Err(anyhow::anyhow!( + "got value: {other} for field `type`, expected value: `static` or `javascript`" + )) + } + }; + + Ok(input_transform) + } +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum RawValueOrFormatted { + RawValue(T), + Formatted { r#type: String, value: Option, expr: Option }, +} + +fn raw_value_to_input_transform<'de, D, T>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, + T: DeserializeOwned + Serialize, +{ + let val = Option::>::deserialize(deserializer)?; + let input_tranform = match val { + Some(RawValueOrFormatted::RawValue(v)) => { + Some(InputTransform::new_static_value(to_raw_value(&v))) + } + Some(RawValueOrFormatted::Formatted { r#type, expr, value }) => { + let untaged_input_transform = UntaggedInputTransform { + type_: r#type, + expr, + value: value.map(|val| to_raw_value(&val)), + }; + let input_transform = TryInto::::try_into(untaged_input_transform) + .map_err(|e| serde::de::Error::custom(e))?; + Some(input_transform) + } + _ => None, + }; + Ok(input_tranform) +} + +/// Id in the `flow_node` table. +#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash, Eq, PartialEq)] +#[serde(transparent)] +pub struct FlowNodeId(pub i64); + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Branch { + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, + #[serde(default = "default_empty_string")] + pub expr: String, + pub modules: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub modules_node: Option, + #[serde(default = "default_true")] + pub skip_failure: bool, + #[serde(default = "default_true")] + pub parallel: bool, +} + +// Tool types for AI Agent +#[derive(Serialize, Debug, Clone, Deserialize)] +pub struct AgentTool { + pub id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, + pub value: ToolValue, +} + +// Convert FlowModule -> AgentTool +impl From for AgentTool { + fn from(flow_module: FlowModule) -> Self { + let module_value = serde_json::from_str::(flow_module.value.get()) + .unwrap_or(FlowModuleValue::Identity); + + AgentTool { + id: flow_module.id, + summary: flow_module.summary, + value: ToolValue::FlowModule(module_value), + } + } +} + +// Convert AgentTool -> FlowModule (only for FlowModule type tools) +impl From<&AgentTool> for Option { + fn from(tool: &AgentTool) -> Self { + match &tool.value { + ToolValue::FlowModule(module_value) => Some(FlowModule { + id: tool.id.clone(), + value: to_raw_value(module_value), + summary: tool.summary.clone(), + ..Default::default() + }), + ToolValue::Mcp(_) => None, + ToolValue::Websearch(_) => None, + } + } +} + +#[derive(Serialize, Debug, Clone)] +#[serde(tag = "tool_type", rename_all = "lowercase")] +pub enum ToolValue { + FlowModule(FlowModuleValue), + Mcp(McpToolValue), + Websearch(WebsearchToolValue), +} + +// Custom deserializer for backward compatibility with old flows +impl<'de> Deserialize<'de> for ToolValue { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + use serde::de::Error; + + let content = serde_json::Value::deserialize(deserializer)?; + + #[derive(Deserialize)] + #[serde(tag = "tool_type", rename_all = "lowercase")] + enum TaggedToolValue { + FlowModule(FlowModuleValue), + Mcp(McpToolValue), + Websearch(WebsearchToolValue), + } + + if let Ok(tagged) = TaggedToolValue::deserialize(&content) { + return Ok(match tagged { + TaggedToolValue::FlowModule(v) => ToolValue::FlowModule(v), + TaggedToolValue::Mcp(v) => ToolValue::Mcp(v), + TaggedToolValue::Websearch(v) => ToolValue::Websearch(v), + }); + } + + FlowModuleValue::deserialize(&content) + .map(ToolValue::FlowModule) + .map_err(|_| { + D::Error::custom( + "expected ToolValue with tool_type field or legacy FlowModuleValue", + ) + }) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct McpToolValue { + pub resource_path: String, + #[serde(default)] + pub include_tools: Vec, + #[serde(default)] + pub exclude_tools: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +pub struct WebsearchToolValue {} + +fn is_none_or_empty_vec(expr: &Option>) -> bool { + expr.is_none() || expr.as_ref().unwrap().is_empty() +} + +#[derive(Serialize, Debug, Clone)] +#[serde( + tag = "type", + rename_all(serialize = "lowercase", deserialize = "lowercase") +)] +pub enum FlowModuleValue { + Script { + #[serde(default)] + #[serde(alias = "input_transform")] + input_transforms: HashMap, + path: String, + #[serde(skip_serializing_if = "Option::is_none")] + hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] + tag_override: Option, + #[serde(skip_serializing_if = "Option::is_none")] + is_trigger: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pass_flow_input_directly: Option, + }, + Flow { + #[serde(default)] + #[serde(alias = "input_transform")] + input_transforms: HashMap, + path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pass_flow_input_directly: Option, + }, + ForloopFlow { + iterator: InputTransform, + modules: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + modules_node: Option, + #[serde(default = "default_true")] + skip_failures: bool, + parallel: bool, + #[serde(skip_serializing_if = "Option::is_none")] + parallelism: Option, + #[serde(skip_serializing_if = "Option::is_none")] + squash: Option, + }, + WhileloopFlow { + modules: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + modules_node: Option, + #[serde(default = "default_false")] + skip_failures: bool, + #[serde(skip_serializing_if = "Option::is_none")] + squash: Option, + }, + BranchOne { + branches: Vec, + default: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + default_node: Option, + }, + BranchAll { + branches: Vec, + #[serde(default = "default_true")] + parallel: bool, + }, + RawScript { + #[serde(default)] + #[serde(alias = "input_transform", serialize_with = "ordered_map")] + input_transforms: HashMap, + content: String, + #[serde(skip_serializing_if = "Option::is_none")] + lock: Option, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "is_none_or_empty")] + tag: Option, + language: ScriptLang, + #[serde(flatten)] + concurrency_settings: ConcurrencySettingsWithCustom, + #[serde(skip_serializing_if = "Option::is_none")] + is_trigger: Option, + #[serde(skip_serializing_if = "is_none_or_empty_vec")] + assets: Option>, + }, + Identity, + FlowScript { + #[serde(default)] + #[serde(alias = "input_transform", serialize_with = "ordered_map")] + input_transforms: HashMap, + id: FlowNodeId, + #[serde(skip_serializing_if = "is_none_or_empty")] + tag: Option, + language: ScriptLang, + #[serde(flatten)] + concurrency_settings: ConcurrencySettingsWithCustom, + #[serde(skip_serializing_if = "Option::is_none")] + is_trigger: Option, + #[serde(skip_serializing_if = "is_none_or_empty_vec")] + assets: Option>, + }, + AIAgent { + input_transforms: HashMap, + tools: Vec, + }, +} + +fn is_none_or_empty(expr: &Option) -> bool { + expr.is_none() || expr.as_ref().unwrap().is_empty() +} + +#[derive(Deserialize)] +struct UntaggedFlowModuleValue { + #[serde(rename = "type")] + type_: String, + #[serde(alias = "input_transform")] + input_transforms: Option>, + path: Option, + hash: Option, + tag_override: Option, + iterator: Option, + modules: Option>, + skip_failures: Option, + parallel: Option, + #[serde(default, deserialize_with = "raw_value_to_input_transform::<_, u16>")] + parallelism: Option, + branches: Option>, + default: Option>, + content: Option, + lock: Option, + tag: Option, + language: Option, + is_trigger: Option, + id: Option, + default_node: Option, + modules_node: Option, + assets: Option>, + tools: Option>, + pass_flow_input_directly: Option, + squash: Option, + #[serde(flatten)] + concurrency_settings: ConcurrencySettingsWithCustom, +} + +impl<'de> Deserialize<'de> for FlowModuleValue { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let untagged: UntaggedFlowModuleValue = UntaggedFlowModuleValue::deserialize(deserializer)?; + + match untagged.type_.as_str() { + "script" => Ok(FlowModuleValue::Script { + input_transforms: untagged.input_transforms.unwrap_or_default(), + path: untagged + .path + .ok_or_else(|| serde::de::Error::missing_field("path"))?, + hash: untagged.hash, + tag_override: untagged.tag_override, + is_trigger: untagged.is_trigger, + pass_flow_input_directly: untagged.pass_flow_input_directly, + }), + "flow" => Ok(FlowModuleValue::Flow { + input_transforms: untagged.input_transforms.unwrap_or_default(), + path: untagged + .path + .ok_or_else(|| serde::de::Error::missing_field("path"))?, + pass_flow_input_directly: untagged.pass_flow_input_directly, + }), + "forloopflow" => Ok(FlowModuleValue::ForloopFlow { + iterator: untagged + .iterator + .ok_or_else(|| serde::de::Error::missing_field("iterator"))?, + modules: untagged + .modules + .ok_or_else(|| serde::de::Error::missing_field("modules"))?, + modules_node: untagged.modules_node, + skip_failures: untagged.skip_failures.unwrap_or(true), + parallel: untagged.parallel.unwrap_or(false), + parallelism: untagged.parallelism, + squash: untagged.squash, + }), + "whileloopflow" => Ok(FlowModuleValue::WhileloopFlow { + modules: untagged + .modules + .ok_or_else(|| serde::de::Error::missing_field("modules"))?, + modules_node: untagged.modules_node, + skip_failures: untagged.skip_failures.unwrap_or(false), + squash: untagged.squash, + }), + "branchone" => Ok(FlowModuleValue::BranchOne { + branches: untagged + .branches + .ok_or_else(|| serde::de::Error::missing_field("branches"))?, + default: untagged + .default + .ok_or_else(|| serde::de::Error::missing_field("default"))?, + default_node: untagged.default_node, + }), + "branchall" => Ok(FlowModuleValue::BranchAll { + branches: untagged + .branches + .ok_or_else(|| serde::de::Error::missing_field("branches"))?, + parallel: untagged.parallel.unwrap_or(true), + }), + "rawscript" => Ok(FlowModuleValue::RawScript { + input_transforms: untagged.input_transforms.unwrap_or_default(), + content: untagged + .content + .ok_or_else(|| serde::de::Error::missing_field("content"))?, + lock: untagged.lock, + path: untagged.path, + tag: untagged.tag, + language: untagged + .language + .ok_or_else(|| serde::de::Error::missing_field("language"))?, + concurrency_settings: untagged.concurrency_settings, + is_trigger: untagged.is_trigger, + assets: untagged.assets, + }), + "flowscript" => Ok(FlowModuleValue::FlowScript { + input_transforms: untagged.input_transforms.unwrap_or_default(), + id: untagged + .id + .ok_or_else(|| serde::de::Error::missing_field("id"))?, + tag: untagged.tag, + language: untagged + .language + .ok_or_else(|| serde::de::Error::missing_field("language"))?, + concurrency_settings: untagged.concurrency_settings, + is_trigger: untagged.is_trigger, + assets: untagged.assets, + }), + "identity" => Ok(FlowModuleValue::Identity), + "aiagent" => Ok(FlowModuleValue::AIAgent { + input_transforms: untagged.input_transforms.unwrap_or_default(), + tools: untagged + .tools + .ok_or_else(|| serde::de::Error::missing_field("tools"))?, + }), + other => Err(serde::de::Error::unknown_variant( + other, + &[ + "script", + "flow", + "forloopflow", + "whileloopflow", + "branchone", + "branchall", + "rawscript", + "identity", + "aiagent", + ], + )), + } + } +} + +impl Into> for FlowModuleValue { + fn into(self) -> Box { + to_raw_value(&self) + } +} + +pub fn ordered_map(value: &HashMap, serializer: S) -> Result +where + S: Serializer, +{ + let ordered: BTreeMap<_, _> = value.iter().collect(); + ordered.serialize(serializer) +} + +#[derive(Deserialize)] +pub struct ListFlowQuery { + pub without_description: Option, + pub path_start: Option, + pub path_exact: Option, + pub edited_by: Option, + pub show_archived: Option, + pub order_by: Option, + pub order_desc: Option, + pub starred_only: Option, + pub include_draft_only: Option, + pub with_deployment_msg: Option, + pub dedicated_worker: Option, +} + +pub fn add_virtual_items_if_necessary(modules: &mut Vec) { + if modules.len() > 0 + && (modules[modules.len() - 1].sleep.is_some() + || modules[modules.len() - 1].suspend.is_some()) + { + modules.push(FlowModule { + id: format!("{}-v", modules[modules.len() - 1].id), + value: to_raw_value(&FlowModuleValue::Identity), + stop_after_if: None, + stop_after_all_iters_if: None, + summary: Some("Virtual module needed for suspend/sleep when last module".to_string()), + mock: None, + retry: None, + sleep: None, + suspend: None, + cache_ttl: None, + cache_ignore_s3_path: None, + timeout: None, + priority: None, + delete_after_use: None, + continue_on_error: None, + skip_if: None, + apply_preprocessor: None, + pass_flow_input_directly: None, + }); + } +} diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs new file mode 100644 index 0000000000..3c54b7956e --- /dev/null +++ b/backend/windmill-types/src/jobs.rs @@ -0,0 +1,510 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sqlx::types::Json; +use uuid::Uuid; + +use crate::{ + apps::AppScriptId, + flow_status::{FlowStatus, RestartedFrom}, + flows::{FlowNodeId, FlowValue, Retry}, + runnable_settings::{ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings}, + scripts::{ScriptHash, ScriptLang}, +}; + +#[derive(Debug, Deserialize, Clone)] +pub struct DynamicInput { + #[serde(rename = "x-windmill-dyn-select-code")] + pub x_windmill_dyn_select_code: String, + #[serde(rename = "x-windmill-dyn-select-lang")] + pub x_windmill_dyn_select_lang: ScriptLang, +} + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone)] +#[sqlx(type_name = "JOB_TRIGGER_KIND", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum JobTriggerKind { + Webhook, + Http, + Websocket, + Kafka, + Email, + Nats, + Mqtt, + Sqs, + Postgres, + Schedule, + Gcp, + Nextcloud, +} + +impl std::fmt::Display for JobTriggerKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let kind = match self { + JobTriggerKind::Webhook => "webhook", + JobTriggerKind::Http => "http", + JobTriggerKind::Websocket => "websocket", + JobTriggerKind::Kafka => "kafka", + JobTriggerKind::Email => "email", + JobTriggerKind::Nats => "nats", + JobTriggerKind::Mqtt => "mqtt", + JobTriggerKind::Sqs => "sqs", + JobTriggerKind::Postgres => "postgres", + JobTriggerKind::Schedule => "schedule", + JobTriggerKind::Gcp => "gcp", + JobTriggerKind::Nextcloud => "nextcloud", + }; + write!(f, "{}", kind) + } +} + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Default)] +#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum JobKind { + Script, + #[allow(non_camel_case_types)] + Script_Hub, + Preview, + Dependencies, + Flow, + FlowPreview, + SingleStepFlow, + Identity, + FlowDependencies, + AppDependencies, + #[default] + Noop, + DeploymentCallback, + FlowScript, + FlowNode, + AppScript, + AIAgent, + #[serde(rename = "unassigned_script")] + #[sqlx(rename = "unassigned_script")] + UnassignedScript, + #[serde(rename = "unassigned_flow")] + #[sqlx(rename = "unassigned_flow")] + UnassignedFlow, + #[serde(rename = "unassigned_singlestepflow")] + #[sqlx(rename = "unassigned_singlestepflow")] + UnassignedSinglestepFlow, +} + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone)] +#[sqlx(type_name = "JOB_STATUS", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum JobStatus { + Success, + Failure, + Canceled, + Skipped, +} + +impl JobKind { + pub fn is_flow(&self) -> bool { + matches!( + self, + JobKind::Flow | JobKind::FlowPreview | JobKind::SingleStepFlow | JobKind::FlowNode + ) + } + + pub fn is_dependency(&self) -> bool { + matches!( + self, + JobKind::FlowDependencies | JobKind::AppDependencies | JobKind::Dependencies + ) + } +} + +#[derive(sqlx::FromRow, Debug, Serialize, Clone)] +pub struct QueuedJob { + pub workspace_id: String, + pub id: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_job: Option, + pub created_by: String, + pub created_at: chrono::DateTime, + #[serde(skip_serializing_if = "Option::is_none")] + pub started_at: Option>, + pub scheduled_for: chrono::DateTime, + pub running: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub script_hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub script_path: Option, + pub script_entrypoint_override: Option, + pub args: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub logs: Option, + pub canceled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub canceled_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub canceled_reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_ping: Option>, + pub job_kind: JobKind, + #[serde(skip_serializing_if = "Option::is_none")] + pub schedule_path: Option, + pub permissioned_as: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_status: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub workflow_as_code_status: Option>>, + pub is_flow_step: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + pub same_worker: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub pre_run_error: Option, + pub email: String, + pub visible_to_owner: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub suspend: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mem_peak: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub root_job: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub leaf_jobs: Option, + pub tag: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrent_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_time_window_s: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_step_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ttl: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ignore_s3_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub preprocessed: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub runnable_settings_handle: Option, +} + +impl QueuedJob { + pub fn script_path(&self) -> &str { + self.script_path + .as_ref() + .map(String::as_str) + .unwrap_or("tmp/main") + } + pub fn is_flow(&self) -> bool { + self.job_kind.is_flow() + } + + pub fn full_path_with_workspace(&self) -> String { + format!( + "{}/{}/{}", + self.workspace_id, + if self.is_flow() { "flow" } else { "script" }, + self.script_path() + ) + } + + pub fn parse_flow_status(&self) -> Option { + self.flow_status + .as_ref() + .and_then(|v| serde_json::from_str::((**v).get()).ok()) + } +} + +impl Default for QueuedJob { + fn default() -> Self { + Self { + workspace_id: "".to_string(), + id: Uuid::default(), + parent_job: None, + created_by: "".to_string(), + created_at: chrono::Utc::now(), + started_at: None, + scheduled_for: chrono::Utc::now(), + running: false, + script_hash: None, + script_path: None, + args: None, + logs: None, + canceled: false, + canceled_by: None, + canceled_reason: None, + last_ping: None, + job_kind: JobKind::Identity, + schedule_path: None, + permissioned_as: "".to_string(), + workflow_as_code_status: None, + flow_status: None, + is_flow_step: false, + language: None, + script_entrypoint_override: None, + same_worker: false, + pre_run_error: None, + email: "".to_string(), + visible_to_owner: false, + suspend: None, + mem_peak: None, + root_job: None, + leaf_jobs: None, + tag: "deno".to_string(), + concurrent_limit: None, + concurrency_time_window_s: None, + timeout: None, + flow_step_id: None, + cache_ttl: None, + cache_ignore_s3_path: None, + priority: None, + preprocessed: None, + runnable_settings_handle: None, + } + } +} + +#[derive(Debug, sqlx::FromRow, Serialize, Clone)] +pub struct CompletedJob { + pub workspace_id: String, + pub id: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_job: Option, + pub created_by: String, + pub created_at: chrono::DateTime, + pub started_at: Option>, + pub completed_at: Option>, + pub duration_ms: i64, + pub success: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub script_hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub script_path: Option, + pub args: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub result: Option>>, + pub result_columns: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub logs: Option, + pub deleted: bool, + pub canceled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub canceled_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub canceled_reason: Option, + pub job_kind: JobKind, + #[serde(skip_serializing_if = "Option::is_none")] + pub schedule_path: Option, + pub permissioned_as: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_status: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub workflow_as_code_status: Option>>, + pub is_flow_step: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + pub is_skipped: bool, + pub email: String, + pub visible_to_owner: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub mem_peak: Option, + pub tag: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub labels: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub preprocessed: Option, +} + +impl CompletedJob { + pub fn json_result(&self) -> Option { + self.result + .as_ref() + .map(|r| serde_json::from_str(r.get()).ok()) + .flatten() + } + + pub fn parse_flow_status(&self) -> Option { + self.flow_status + .as_ref() + .and_then(|v| serde_json::from_str::((**v).get()).ok()) + } +} + +#[derive(Debug, Clone)] +pub enum JobPayload { + ScriptHub { + path: String, + apply_preprocessor: bool, + }, + ScriptHash { + hash: ScriptHash, + path: String, + cache_ttl: Option, + cache_ignore_s3_path: Option, + dedicated_worker: Option, + language: ScriptLang, + priority: Option, + apply_preprocessor: bool, + concurrency_settings: ConcurrencySettings, + debouncing_settings: DebouncingSettings, + }, + FlowNode { + id: FlowNodeId, + path: String, + }, + FlowScript { + id: FlowNodeId, + path: String, + language: ScriptLang, + cache_ttl: Option, + cache_ignore_s3_path: Option, + dedicated_worker: Option, + concurrency_settings: ConcurrencySettings, + }, + AppScript { + id: AppScriptId, + path: Option, + language: ScriptLang, + cache_ttl: Option, + }, + Code(RawCode), + Dependencies { + path: String, + hash: ScriptHash, + language: ScriptLang, + dedicated_worker: Option, + debouncing_settings: DebouncingSettings, + }, + FlowDependencies { + path: String, + dedicated_worker: Option, + version: i64, + debouncing_settings: DebouncingSettings, + }, + AppDependencies { + path: String, + version: i64, + debouncing_settings: DebouncingSettings, + }, + RawFlowDependencies { + path: String, + flow_value: FlowValue, + }, + RawScriptDependencies { + script_path: String, + content: String, + language: ScriptLang, + }, + Flow { + path: String, + dedicated_worker: Option, + apply_preprocessor: bool, + version: i64, + }, + RestartedFlow { + completed_job_id: Uuid, + step_id: String, + branch_or_iteration_n: Option, + flow_version: Option, + }, + RawFlow { + value: FlowValue, + path: Option, + restarted_from: Option, + }, + SingleStepFlow { + path: String, + hash: Option, + flow_version: Option, + args: HashMap>, + retry: Option, + error_handler_path: Option, + error_handler_args: Option>>, + skip_handler: Option, + cache_ttl: Option, + cache_ignore_s3_path: Option, + priority: Option, + tag_override: Option, + trigger_path: Option, + apply_preprocessor: bool, + concurrency_settings: ConcurrencySettings, + debouncing_settings: DebouncingSettings, + }, + DeploymentCallback { + path: String, + debouncing_settings: DebouncingSettings, + }, + Identity, + Noop, + AIAgent { + path: String, + }, +} + +#[derive(Clone, Serialize, Deserialize, Debug)] +pub struct SkipHandler { + pub path: String, + pub args: HashMap>, + pub stop_condition: String, + pub stop_message: String, +} + +#[derive(Clone, Deserialize, Debug, Default)] +pub struct RawCode { + pub content: String, + pub path: Option, + pub hash: Option, + pub language: ScriptLang, + pub lock: Option, + pub cache_ttl: Option, + pub cache_ignore_s3_path: Option, + pub dedicated_worker: Option, + #[serde(flatten)] + pub concurrency_settings: ConcurrencySettingsWithCustom, + #[serde(flatten)] + pub debouncing_settings: DebouncingSettings, +} + +impl JobPayload { + pub fn job_kind(&self) -> JobKind { + match self { + JobPayload::Noop => JobKind::Noop, + JobPayload::Identity => JobKind::Identity, + JobPayload::Code { .. } => JobKind::Preview, + JobPayload::AIAgent { .. } => JobKind::AIAgent, + JobPayload::FlowNode { .. } => JobKind::FlowNode, + JobPayload::ScriptHash { .. } => JobKind::Script, + JobPayload::AppScript { .. } => JobKind::AppScript, + JobPayload::RawFlow { .. } => JobKind::FlowPreview, + JobPayload::ScriptHub { .. } => JobKind::Script_Hub, + JobPayload::FlowScript { .. } => JobKind::FlowScript, + JobPayload::Dependencies { .. } => JobKind::Dependencies, + JobPayload::SingleStepFlow { .. } => JobKind::SingleStepFlow, + JobPayload::AppDependencies { .. } => JobKind::AppDependencies, + JobPayload::FlowDependencies { .. } => JobKind::FlowDependencies, + JobPayload::RawScriptDependencies { .. } => JobKind::Dependencies, + JobPayload::RawFlowDependencies { .. } => JobKind::FlowDependencies, + JobPayload::DeploymentCallback { .. } => JobKind::DeploymentCallback, + JobPayload::Flow { .. } | JobPayload::RestartedFlow { .. } => JobKind::Flow, + } + } +} + +#[derive(Clone, Debug)] +pub struct OnBehalfOf { + pub email: String, + pub permissioned_as: String, +} + +pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE"; +pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000; +pub const EMAIL_ERROR_HANDLER_USER_EMAIL: &str = "email_error_handler@windmill.dev"; + +#[inline(always)] +pub fn generate_dynamic_input_key(workspace_id: &str, path: &str) -> String { + format!("{workspace_id}:{path}") +} diff --git a/backend/windmill-types/src/lib.rs b/backend/windmill-types/src/lib.rs new file mode 100644 index 0000000000..8b7e68d75c --- /dev/null +++ b/backend/windmill-types/src/lib.rs @@ -0,0 +1,17 @@ +pub mod apps; +pub mod assets; +pub mod flow_status; +pub mod flows; +pub mod jobs; +pub mod more_serde; +pub mod runnable_settings; +pub mod schedule; +pub mod scripts; +pub mod triggers; + +/// Duplicated from windmill-common::worker::to_raw_value. +/// windmill-types cannot depend on windmill-common (it would be circular). +pub fn to_raw_value(result: &T) -> Box { + serde_json::value::to_raw_value(result) + .unwrap_or_else(|_| serde_json::value::RawValue::from_string("{}".to_string()).unwrap()) +} diff --git a/backend/windmill-types/src/more_serde.rs b/backend/windmill-types/src/more_serde.rs new file mode 100644 index 0000000000..d4b648d2d5 --- /dev/null +++ b/backend/windmill-types/src/more_serde.rs @@ -0,0 +1,68 @@ +//! helpers for serde + serde derive attributes + +use rand::distr::Alphanumeric; +use rand::Rng; +use serde::{Deserialize, Deserializer}; +use serde_json::value::RawValue; +use std::{fmt::Display, str::FromStr}; + +pub fn default_true() -> bool { + true +} + +pub fn default_false() -> bool { + false +} + +pub fn default_null() -> Box { + RawValue::from_string("null".to_string()).unwrap() +} + +pub fn default_empty_string() -> String { + String::new() +} + +pub fn default_id() -> String { + rd_string(6) +} + +fn rd_string(len: usize) -> String { + rand::rng() + .sample_iter(&Alphanumeric) + .take(len) + .map(char::from) + .collect() +} + +pub fn is_default(t: &T) -> bool { + &T::default() == t +} + +pub fn maybe_number_opt<'de, T, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: FromStr + serde::Deserialize<'de>, + ::Err: Display, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum NumericOrNull<'a, T> { + String(String), + Str(&'a str), + RawT(T), + Null, + } + + match NumericOrNull::::deserialize(deserializer)? { + NumericOrNull::String(s) => match s.as_str() { + "" => Ok(None), + _ => T::from_str(&s).map(Some).map_err(serde::de::Error::custom), + }, + NumericOrNull::Str(s) => match s { + "" => Ok(None), + _ => T::from_str(s).map(Some).map_err(serde::de::Error::custom), + }, + NumericOrNull::RawT(i) => Ok(Some(i)), + NumericOrNull::Null => Ok(None), + } +} diff --git a/backend/windmill-types/src/runnable_settings.rs b/backend/windmill-types/src/runnable_settings.rs new file mode 100644 index 0000000000..dfff0df79e --- /dev/null +++ b/backend/windmill-types/src/runnable_settings.rs @@ -0,0 +1,113 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize, Clone, Copy, Serialize, Default, Hash)] +pub struct RunnableSettings { + pub debouncing_settings: Option, + pub concurrency_settings: Option, +} + +// TODO: Add validation logic. +#[derive( + Debug, Clone, Serialize, Deserialize, Default, Hash, PartialEq, sqlx::FromRow, sqlx::Type, +)] +pub struct DebouncingSettings { + #[serde(skip_serializing_if = "Option::is_none", alias = "custom_debounce_key")] + pub debounce_key: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub debounce_delay_s: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub max_total_debouncing_time: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub max_total_debounces_amount: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub debounce_args_to_accumulate: Option>, +} + +#[derive( + Debug, Default, Clone, Serialize, Deserialize, Hash, PartialEq, sqlx::FromRow, sqlx::Decode, +)] +pub struct ConcurrencySettings { + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrent_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_time_window_s: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow, Default)] +pub struct ConcurrencySettingsWithCustom { + #[serde(skip_serializing_if = "Option::is_none")] + pub custom_concurrency_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrent_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_time_window_s: Option, +} + +impl DebouncingSettings { + pub fn maybe_fallback( + self, + debounce_key: Option, + debounce_delay_s: Option, + ) -> Self { + Self { + debounce_key: self.debounce_key.or(debounce_key), + debounce_delay_s: self.debounce_delay_s.or(debounce_delay_s), + ..self + } + } + + pub fn is_legacy_compatible(&self) -> bool { + self.max_total_debouncing_time.is_none() + && self.max_total_debounces_amount.is_none() + && self.debounce_args_to_accumulate.is_none() + } +} + +impl ConcurrencySettings { + pub fn maybe_fallback( + self, + concurrency_key: Option, + concurrent_limit: Option, + concurrency_time_window_s: Option, + ) -> Self { + Self { + concurrency_key: self.concurrency_key.or(concurrency_key), + concurrent_limit: self.concurrent_limit.or(concurrent_limit), + concurrency_time_window_s: self.concurrency_time_window_s.or(concurrency_time_window_s), + } + } +} + +impl From for ConcurrencySettingsWithCustom { + fn from( + ConcurrencySettings { concurrency_key, concurrent_limit, concurrency_time_window_s }: ConcurrencySettings, + ) -> Self { + ConcurrencySettingsWithCustom { + custom_concurrency_key: concurrency_key, + concurrency_time_window_s, + concurrent_limit, + } + } +} + +impl From for ConcurrencySettings { + fn from( + ConcurrencySettingsWithCustom { + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + }: ConcurrencySettingsWithCustom, + ) -> Self { + ConcurrencySettings { + concurrency_key: custom_concurrency_key, + concurrency_time_window_s, + concurrent_limit, + } + } +} diff --git a/backend/windmill-types/src/schedule.rs b/backend/windmill-types/src/schedule.rs new file mode 100644 index 0000000000..a3a61b5765 --- /dev/null +++ b/backend/windmill-types/src/schedule.rs @@ -0,0 +1,67 @@ +use chrono::DateTime; +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; + +use crate::flows::Retry; + +#[derive(FromRow, Serialize, Deserialize, Debug, Clone)] +pub struct Schedule { + pub workspace_id: String, + pub path: String, + pub edited_by: String, + pub edited_at: DateTime, + pub schedule: String, + pub timezone: String, + pub enabled: bool, + pub script_path: String, + pub is_flow: bool, + pub args: Option>>, + pub extra_perms: serde_json::Value, + pub email: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_failure: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_failure_times: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_failure_exact: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_failure_extra_args: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_recovery: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_recovery_times: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_recovery_extra_args: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_success: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_success_extra_args: Option>>, + pub ws_error_handler_muted: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry: Option, + pub no_flow_overlap: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tag: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub paused_until: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub cron_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub dynamic_skip: Option, +} + +impl Schedule { + pub fn parse_retry(self) -> Option { + self.retry.map(|r| serde_json::from_value(r).ok()).flatten() + } +} + +pub fn schedule_to_user(path: &str) -> String { + format!("schedule-{}", path.replace('/', "-")) +} diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs new file mode 100644 index 0000000000..a925bd2031 --- /dev/null +++ b/backend/windmill-types/src/scripts.rs @@ -0,0 +1,613 @@ +use std::{ + fmt::{self, Display}, + hash::{Hash, Hasher}, + ops::Deref, + str::FromStr, +}; + +use itertools::Itertools; +use serde::de::Error as _; +use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize}; + +use crate::{ + assets::AssetWithAltAccessType, + runnable_settings::{ConcurrencySettings, DebouncingSettings}, +}; + +#[derive( + Serialize, + Deserialize, + Debug, + PartialEq, + Copy, + Clone, + Hash, + Eq, + sqlx::Type, + Default, + Ord, + PartialOrd, +)] +#[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum ScriptLang { + Nativets, + #[default] + Deno, + Python3, + Go, + Bash, + Powershell, + Postgresql, + Bun, + Bunnative, + Mysql, + Bigquery, + Snowflake, + Graphql, + Mssql, + OracleDB, + DuckDb, + Php, + Rust, + Ansible, + CSharp, + Nu, + Java, + Ruby, + // for related places search: ADD_NEW_LANG +} + +impl ScriptLang { + pub fn as_str(&self) -> &'static str { + match self { + ScriptLang::Bun => "bun", + ScriptLang::Bunnative => "bunnative", + ScriptLang::Nativets => "nativets", + ScriptLang::Deno => "deno", + ScriptLang::Python3 => "python3", + ScriptLang::Go => "go", + ScriptLang::Bash => "bash", + ScriptLang::Powershell => "powershell", + ScriptLang::Postgresql => "postgresql", + ScriptLang::Mysql => "mysql", + ScriptLang::Bigquery => "bigquery", + ScriptLang::Snowflake => "snowflake", + ScriptLang::Mssql => "mssql", + ScriptLang::Graphql => "graphql", + ScriptLang::OracleDB => "oracledb", + ScriptLang::DuckDb => "duckdb", + ScriptLang::Php => "php", + ScriptLang::Rust => "rust", + ScriptLang::Ansible => "ansible", + ScriptLang::CSharp => "csharp", + ScriptLang::Nu => "nu", + ScriptLang::Java => "java", + ScriptLang::Ruby => "ruby", + // for related places search: ADD_NEW_LANG + } + } + + pub fn as_dependencies_filename(&self) -> Option { + use ScriptLang::*; + Some( + match self { + Bun | Bunnative | Nativets => "package.json", + Python3 => "requirements.in", + // Go => "go.mod", + Php => "composer.json", + _ => return None, + } + .to_owned(), + ) + } + + pub fn as_comment_lit(&self) -> String { + use ScriptLang::*; + match self { + Nativets | Bun | Bunnative | Deno | Go | Php | CSharp | Java => "//", + Python3 | Bash | Powershell | Graphql | Ansible | Nu | Ruby => "#", + Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--", + Rust => "//!", + // for related places search: ADD_NEW_LANG + } + .to_owned() + } +} + +impl FromStr for ScriptLang { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + let language = match s.to_lowercase().as_str() { + "bun" => ScriptLang::Bun, + "bunnative" => ScriptLang::Bunnative, + "nativets" => ScriptLang::Nativets, + "deno" => ScriptLang::Deno, + "python3" => ScriptLang::Python3, + "go" => ScriptLang::Go, + "bash" => ScriptLang::Bash, + "powershell" => ScriptLang::Powershell, + "postgresql" => ScriptLang::Postgresql, + "mysql" => ScriptLang::Mysql, + "bigquery" => ScriptLang::Bigquery, + "snowflake" => ScriptLang::Snowflake, + "mssql" => ScriptLang::Mssql, + "graphql" => ScriptLang::Graphql, + "oracledb" => ScriptLang::OracleDB, + "php" => ScriptLang::Php, + "rust" => ScriptLang::Rust, + "ansible" => ScriptLang::Ansible, + "csharp" => ScriptLang::CSharp, + "nu" => ScriptLang::Nu, + "java" => ScriptLang::Java, + "ruby" => ScriptLang::Ruby, + // for related places search: ADD_NEW_LANG + language => { + return Err(anyhow::anyhow!("{} is currently not supported", language)) + } + }; + + Ok(language) + } +} + +#[derive(Eq, PartialEq, Debug, Hash, Clone, Copy, sqlx::Type)] +#[sqlx(transparent)] +pub struct ScriptHash(pub i64); + +impl Deref for ScriptHash { + type Target = i64; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Into for ScriptHash { + fn into(self) -> u64 { + self.0 as u64 + } +} + +impl From for ScriptHash { + fn from(value: i64) -> Self { + Self(value) + } +} + +#[derive(PartialEq, sqlx::Type, Debug)] +#[sqlx(transparent, no_pg_array)] +pub struct ScriptHashes(pub Vec); + +impl Display for ScriptHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", to_hex_string(&self.0)) + } +} +impl Serialize for ScriptHash { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + serializer.serialize_str(to_hex_string(&self.0).as_str()) + } +} +impl<'de> Deserialize<'de> for ScriptHash { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let i = to_i64(&s).map_err(|e| { + tracing::error!("Could not deserialize ScriptHash. Note, input should be in Hex and digit amount should be divisible by 16 (can be padded). err: {}", &e); + D::Error::custom(format!("{}", e)) + })?; + Ok(ScriptHash(i)) + } +} + +impl Serialize for ScriptHashes { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + let mut seq = serializer.serialize_seq(Some(self.0.len()))?; + for element in &self.0 { + seq.serialize_element(&ScriptHash(*element))?; + } + seq.end() + } +} + +#[derive(Serialize, Deserialize, Debug, Hash, sqlx::Type)] +#[sqlx(type_name = "SCRIPT_KIND", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum ScriptKind { + Trigger, + Failure, + Script, + Approval, + Preprocessor, +} + +impl Display for ScriptKind { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.write_str(match self { + ScriptKind::Trigger => "trigger", + ScriptKind::Failure => "failure", + ScriptKind::Script => "script", + ScriptKind::Approval => "approval", + ScriptKind::Preprocessor => "preprocessor", + })?; + Ok(()) + } +} + +const PREVIEW_IS_CODEBASE_HASH: i64 = -42; +const PREVIEW_IS_TAR_CODEBASE_HASH: i64 = -43; +const PREVIEW_IS_ESM_CODEBASE_HASH: i64 = -44; +const PREVIEW_IS_TAR_ESM_CODEBASE_HASH: i64 = -45; + +pub fn is_special_codebase_hash(hash: i64) -> bool { + hash == PREVIEW_IS_CODEBASE_HASH + || hash == PREVIEW_IS_TAR_CODEBASE_HASH + || hash == PREVIEW_IS_ESM_CODEBASE_HASH + || hash == PREVIEW_IS_TAR_ESM_CODEBASE_HASH +} + +pub fn codebase_to_hash(is_tar: bool, is_esm: bool) -> i64 { + if is_tar { + if is_esm { + PREVIEW_IS_TAR_ESM_CODEBASE_HASH + } else { + PREVIEW_IS_TAR_CODEBASE_HASH + } + } else { + if is_esm { + PREVIEW_IS_ESM_CODEBASE_HASH + } else { + PREVIEW_IS_CODEBASE_HASH + } + } +} + +pub fn hash_to_codebase_id(job_id: &str, hash: i64) -> Option { + match hash { + PREVIEW_IS_CODEBASE_HASH => Some(job_id.to_string()), + PREVIEW_IS_TAR_CODEBASE_HASH => Some(format!("{}.tar", job_id)), + PREVIEW_IS_ESM_CODEBASE_HASH => Some(format!("{}.esm", job_id)), + PREVIEW_IS_TAR_ESM_CODEBASE_HASH => Some(format!("{}.esm.tar", job_id)), + _ => None, + } +} + +pub struct CodebaseInfo { + pub is_tar: bool, + pub is_esm: bool, +} + +pub fn id_to_codebase_info(id: &str) -> CodebaseInfo { + let is_tar = id.ends_with(".tar"); + let is_esm = id.contains(".esm"); + CodebaseInfo { is_tar, is_esm } +} + +#[derive(Serialize, sqlx::FromRow, Debug)] +pub struct Script { + pub workspace_id: String, + pub hash: ScriptHash, + pub path: String, + pub parent_hashes: Option, + pub summary: String, + pub description: String, + pub content: String, + pub created_by: String, + pub created_at: chrono::DateTime, + pub archived: bool, + pub schema: Option, + pub deleted: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub is_template: Option, + pub extra_perms: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub lock: Option, + pub lock_error_logs: Option, + pub language: ScriptLang, + pub kind: ScriptKind, + pub tag: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub envs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub dedicated_worker: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ws_error_handler_muted: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ttl: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ignore_s3_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub delete_after_use: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub restart_unless_cancelled: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub visible_to_runner_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub no_main_func: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub codebase: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub has_preprocessor: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_behalf_of_email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[sqlx(json(nullable))] + pub assets: Option>, + #[serde(flatten)] + #[sqlx(flatten)] + pub runnable_settings: SR, +} + +// Not serializable +#[derive(sqlx::FromRow, Debug, Clone)] +pub struct ScriptRunnableSettingsHandle { + // legacy - for backwards compatibility + // don't add new values. + pub concurrency_key: Option, + pub concurrent_limit: Option, + pub concurrency_time_window_s: Option, + pub debounce_key: Option, + pub debounce_delay_s: Option, + + // add here as well. + pub runnable_settings_handle: Option, +} + +// Not sqlx queriable +#[derive(Serialize, Debug, Clone, Default)] +pub struct ScriptRunnableSettingsInline { + #[serde(flatten)] + pub concurrency_settings: ConcurrencySettings, + #[serde(flatten)] + pub debouncing_settings: DebouncingSettings, +} + +#[derive(Serialize, sqlx::FromRow)] +pub struct ScriptWithStarred { + #[sqlx(flatten)] + #[serde(flatten)] + pub script: Script, + #[serde(skip_serializing_if = "Option::is_none")] + pub starred: Option, +} + +#[derive(Serialize, sqlx::FromRow)] +pub struct ListableScript { + pub hash: ScriptHash, + pub path: String, + pub summary: String, + pub created_at: chrono::DateTime, + pub archived: bool, + pub extra_perms: serde_json::Value, + pub language: ScriptLang, + pub starred: bool, + pub tag: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub has_draft: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_only: Option, + pub has_deploy_errors: bool, + pub ws_error_handler_muted: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub no_main_func: Option, + #[serde(skip_serializing_if = "is_false")] + pub use_codebase: bool, + #[sqlx(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub deployment_msg: Option, + pub kind: ScriptKind, +} + +fn is_false(x: &bool) -> bool { + return !x; +} + +#[derive(Serialize)] +pub struct ScriptHistory { + pub script_hash: ScriptHash, + #[serde(skip_serializing_if = "Option::is_none")] + pub deployment_msg: Option, +} + +#[derive(Deserialize)] +pub struct ScriptHistoryUpdate { + pub deployment_msg: Option, +} + +#[derive(Serialize, Deserialize, Debug, sqlx::Type, Clone)] +#[sqlx(transparent)] +#[serde(transparent)] +pub struct Schema(pub sqlx::types::Json>); + +impl Hash for Schema { + fn hash(&self, state: &mut H) { + self.0.get().hash(state); + } +} + +#[derive(Serialize, Deserialize, Hash, Debug)] +pub struct NewScript { + pub path: String, + pub parent_hash: Option, + pub summary: String, + pub description: String, + pub content: String, + pub schema: Option, + pub is_template: Option, + #[serde(default = "Option::default")] + #[serde(deserialize_with = "lock_deserialize")] + pub lock: Option, + pub language: ScriptLang, + pub kind: Option, + pub tag: Option, + pub draft_only: Option, + pub envs: Option>, + #[serde(flatten)] + pub concurrency_settings: ConcurrencySettings, + #[serde(flatten)] + pub debouncing_settings: DebouncingSettings, + pub cache_ttl: Option, + pub cache_ignore_s3_path: Option, + pub dedicated_worker: Option, + pub ws_error_handler_muted: Option, + pub priority: Option, + pub timeout: Option, + pub delete_after_use: Option, + pub restart_unless_cancelled: Option, + pub deployment_message: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub visible_to_runner_only: Option, + pub no_main_func: Option, + pub codebase: Option, + pub has_preprocessor: Option, + pub on_behalf_of_email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub assets: Option>, +} + +fn lock_deserialize<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::de::Deserializer<'de>, +{ + struct StringOrArrayVisitor; + + impl<'de> serde::de::Visitor<'de> for StringOrArrayVisitor { + type Value = Option; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("either a string or an array of strings") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Ok(Some(v.to_string())) + } + + fn visit_none(self) -> Result + where + E: serde::de::Error, + { + Ok(None) + } + + fn visit_unit(self) -> Result + where + E: serde::de::Error, + { + Ok(None) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut split_lock: Vec = vec![]; + loop { + if let Ok(Some(elem)) = seq.next_element::() { + split_lock.push(elem); + } else { + break; + } + } + let lock = split_lock.join("\n"); + return Ok(Some(lock)); + } + } + deserializer.deserialize_any(StringOrArrayVisitor) +} + +#[derive(Debug, Deserialize)] +pub struct ListScriptQuery { + pub without_description: Option, + pub path_start: Option, + pub path_exact: Option, + pub created_by: Option, + pub first_parent_hash: Option, + pub last_parent_hash: Option, + pub parent_hash: Option, + pub show_archived: Option, + pub order_by: Option, + pub order_desc: Option, + pub is_template: Option, + pub kinds: Option, + pub starred_only: Option, + pub include_without_main: Option, + pub include_draft_only: Option, + pub with_deployment_msg: Option, + #[serde(default, deserialize_with = "from_seq")] + pub languages: Option>, + pub dedicated_worker: Option, +} + +fn from_seq<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + let s = ::deserialize(deserializer)?; + + let languages: Vec = s + .split(",") + .map(ScriptLang::from_str) + .try_collect() + .map_err(|e: anyhow::Error| serde::de::Error::custom(e.to_string()))?; + + let languages = if languages.is_empty() { + None + } else { + Some(languages) + }; + + Ok(languages) +} + +pub fn to_i64(s: &str) -> anyhow::Result { + let v = hex::decode(s)?; + if v.len() < 8 { + return Err(anyhow::anyhow!( + "hex string did not decode to an u64: {s}", + )); + } + let nb: u64 = u64::from_be_bytes( + v[0..8] + .try_into() + .map_err(|_| hex::FromHexError::InvalidStringLength)?, + ); + Ok(nb as i64) +} + +pub fn to_hex_string(i: &i64) -> String { + hex::encode(i.to_be_bytes()) +} + +#[derive(Deserialize, Serialize)] +pub struct HubScript { + pub content: String, + pub lockfile: Option, + pub language: ScriptLang, + pub schema: Box, + pub summary: Option, +} + +pub fn hash_script(ns: &NewScript) -> i64 { + let mut dh = std::hash::DefaultHasher::new(); + ns.hash(&mut dh); + dh.finish() as i64 +} diff --git a/backend/windmill-types/src/triggers.rs b/backend/windmill-types/src/triggers.rs new file mode 100644 index 0000000000..64d9a2ce80 --- /dev/null +++ b/backend/windmill-types/src/triggers.rs @@ -0,0 +1,94 @@ +use serde::{Deserialize, Serialize}; +use std::fmt; +use strum::EnumIter; + +use crate::jobs::JobTriggerKind; + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, EnumIter)] +#[sqlx(type_name = "TRIGGER_KIND", rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum TriggerKind { + Webhook, + Http, + Websocket, + Kafka, + DefaultEmail, + Email, + Nats, + Mqtt, + Sqs, + Postgres, + Gcp, + Nextcloud, +} + +impl TriggerKind { + pub fn to_key(&self) -> String { + match self { + TriggerKind::Webhook => "webhook".to_string(), + TriggerKind::Http => "http".to_string(), + TriggerKind::Websocket => "websocket".to_string(), + TriggerKind::Kafka => "kafka".to_string(), + TriggerKind::Email => "email".to_string(), + TriggerKind::DefaultEmail => "email".to_string(), + TriggerKind::Nats => "nats".to_string(), + TriggerKind::Mqtt => "mqtt".to_string(), + TriggerKind::Sqs => "sqs".to_string(), + TriggerKind::Postgres => "postgres".to_string(), + TriggerKind::Gcp => "gcp".to_string(), + TriggerKind::Nextcloud => "nextcloud".to_string(), + } + } +} + +impl fmt::Display for TriggerKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = match self { + TriggerKind::Webhook => "webhook", + TriggerKind::Http => "http", + TriggerKind::Websocket => "websocket", + TriggerKind::Kafka => "kafka", + TriggerKind::Email => "email", + TriggerKind::DefaultEmail => "default_email", + TriggerKind::Nats => "nats", + TriggerKind::Mqtt => "mqtt", + TriggerKind::Sqs => "sqs", + TriggerKind::Postgres => "postgres", + TriggerKind::Gcp => "gcp", + TriggerKind::Nextcloud => "nextcloud", + }; + write!(f, "{}", s) + } +} + +#[derive(Eq, PartialEq, Hash)] +pub enum HubOrWorkspaceId { + Hub, + WorkspaceId(String), +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] +pub struct RunnableFormat { + pub version: RunnableFormatVersion, + pub has_preprocessor: bool, +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] +pub enum RunnableFormatVersion { + V1, + V2, +} + +pub type RunnableFormatCacheKey = (HubOrWorkspaceId, i64, TriggerKind); + +#[derive(Debug, Clone)] +pub struct TriggerMetadata { + pub trigger_path: Option, + pub trigger_kind: JobTriggerKind, +} + +impl TriggerMetadata { + pub fn new(trigger_path: Option, trigger_kind: JobTriggerKind) -> TriggerMetadata { + TriggerMetadata { trigger_path, trigger_kind } + } +} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 8991df34d7..608ae1c517 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -3025,7 +3025,7 @@ pub async fn handle_queued_job( #[cfg(not(feature = "enterprise"))] if let Connection::Sql(db) = conn { if (job.concurrent_limit.is_some() || - windmill_common::runnable_settings::RunnableSettings::prefetch_cached_from_handle(job.runnable_settings_handle, db).await?.1.concurrent_limit.is_some()) + windmill_common::runnable_settings::prefetch_cached_from_handle(job.runnable_settings_handle, db).await?.1.concurrent_limit.is_some()) && !job.kind.is_dependency() { logs.push_str("---\n"); logs.push_str("WARNING: This job has concurrency limits enabled. Concurrency limits are an EE feature and the setting is ignored.\n");