diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 9ab8158f33..16accc7d0f 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -405,14 +405,24 @@ pub struct FlowModuleValueWithParallel { #[serde(rename = "type")] pub type_: String, pub parallel: Option, - pub parallelism: Option, + #[serde( + default, + deserialize_with = "raw_value_to_input_transform::<_, u16>", + skip_serializing_if = "Option::is_none" + )] + pub parallelism: Option, } #[derive(Deserialize)] pub struct FlowModuleValueWithSkipFailures { pub skip_failures: Option, pub parallel: Option, - pub parallelism: Option, + #[serde( + default, + deserialize_with = "raw_value_to_input_transform::<_, u16>", + skip_serializing_if = "Option::is_none" + )] + pub parallelism: Option, } #[derive(Deserialize)] @@ -627,7 +637,7 @@ pub enum FlowModuleValue { skip_failures: bool, parallel: bool, #[serde(skip_serializing_if = "Option::is_none")] - parallelism: Option, + parallelism: Option, }, /// While loop node @@ -730,7 +740,8 @@ struct UntaggedFlowModuleValue { modules: Option>, skip_failures: Option, parallel: Option, - parallelism: Option, + #[serde(default, deserialize_with = "raw_value_to_input_transform::<_, u16>")] + parallelism: Option, branches: Option>, default: Option>, content: Option, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 1e6a7befe2..2b8081651d 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -372,7 +372,7 @@ pub async fn update_flow_status_after_job_completion_internal( .as_ref() .and_then(|x| x.skip_failures) .unwrap_or(false), - value.as_ref().and_then(|x| x.parallelism), + value.and_then(|x| x.parallelism), *parallel, ) } else { @@ -3172,17 +3172,32 @@ async fn push_next_flow_job( tracing::debug!(id = %flow_job.id, root_id = %job_root, "pushed next flow job: {uuid}"); if value_with_parallel.type_ == "forloopflow" { - if let Some(p) = value_with_parallel.parallelism { - tracing::debug!(id = %flow_job.id, root_id = %job_root, "updating suspend for forloopflow job {uuid}"); + if let Some(parallelism_transform) = &value_with_parallel.parallelism { + tracing::debug!(id = %flow_job.id, root_id = %job_root, "evaluating parallelism expression for forloopflow job {uuid}"); - if i as u16 >= p { + let ctx = get_transform_context(&flow_job, &previous_id, &status) + .warn_after_seconds(3) + .await?; + + let evaluated_parallelism = evaluate_input_transform::( + parallelism_transform, + arc_last_job_result.clone(), + Some(arc_flow_job_args.clone()), + Some(client), + Some(&ctx), + ) + .await?; + + tracing::debug!(id = %flow_job.id, root_id = %job_root, "updating suspend for forloopflow job {uuid} with parallelism {evaluated_parallelism}"); + + if i as u16 >= evaluated_parallelism { sqlx::query!( "UPDATE v2_job_queue SET suspend = $1, suspend_until = now() + interval '14 day', running = true WHERE id = $2", - (i as u16 - p + 1) as i32, + (i as u16 - evaluated_parallelism + 1) as i32, uuid, ) .execute(&mut *inner_tx) diff --git a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte index c81665cf60..5dd76d2ba0 100644 --- a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte @@ -532,8 +532,11 @@ module.value.parallelism = undefined } else if (module.value.parallel || opts.parallel === true) { // Only set parallelism if parallel is enabled - const n = Math.max(1, Math.floor(Math.abs(opts.parallelism))) - module.value.parallelism = n + const n = Math.max(1, Math.floor(Math.abs(opts.parallelism))); + module.value.parallelism = { + type: 'static', + value: n + } } } diff --git a/frontend/src/lib/components/flows/content/FlowLoop.svelte b/frontend/src/lib/components/flows/content/FlowLoop.svelte index bb6b4cd820..643e4681f5 100644 --- a/frontend/src/lib/components/flows/content/FlowLoop.svelte +++ b/frontend/src/lib/components/flows/content/FlowLoop.svelte @@ -16,8 +16,8 @@ import FlowModuleSleep from './FlowModuleSleep.svelte' import FlowModuleMock from './FlowModuleMock.svelte' - import { Play } from 'lucide-svelte' - import type { FlowModule, Job } from '$lib/gen' + import { Play, FunctionSquare } from 'lucide-svelte' + import type { FlowModule, ForloopFlow, Job } from '$lib/gen' import FlowLoopIterationPreview from '$lib/components/FlowLoopIterationPreview.svelte' import FlowModuleDeleteAfterUse from './FlowModuleDeleteAfterUse.svelte' import IteratorGen from '$lib/components/copilot/IteratorGen.svelte' @@ -28,6 +28,10 @@ import type { PropPickerContext } from '$lib/components/prop_picker' import TabsV2 from '$lib/components/common/tabs/TabsV2.svelte' import { useUiIntent } from '$lib/components/copilot/chat/flow/useUiIntent' + import { emptySchema } from '$lib/utils' + import { slide } from 'svelte/transition' + import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' const { previewArgs, flowStateStore, flowStore, currentEditor } = getContext('FlowEditorContext') @@ -49,7 +53,30 @@ }: Props = $props() let editor: SimpleEditor | undefined = $state(undefined) + let parallelismEditor: SimpleEditor | undefined = $state(undefined) let selected: string = $state('early-stop') + let parallelismType: 'static' | 'javascript' | undefined = $state( + mod.value.type === 'forloopflow' + ? mod.value.parallelism?.type === 'javascript' + ? 'javascript' + : 'static' + : undefined + ) + + let parallelismSchema = $state(emptySchema()) + parallelismSchema.properties['parallelism'] = { + type: 'number' + } + + if (mod.value.type === 'forloopflow') { + const forloopValue = mod.value as ForloopFlow + if (typeof forloopValue.parallelism === 'number') { + forloopValue.parallelism = { + type: 'static', + value: forloopValue.parallelism + } + } + } // UI Intent handling for AI tool control useUiIntent(`forloopflow-${mod.id}`, { @@ -139,8 +166,8 @@ {#if mod.value.type === 'forloopflow'} -
-
+
+
Skip failures
-
+
Run in parallel
-
+
Parallelism Assign a maximum number of branches run in parallel to control huge for-loops.
- +
+ { + const parallelismExpr = (mod.value as ForloopFlow).parallelism + + return parallelismExpr && parallelismExpr.type === 'static' + ? parallelismExpr.value + : '' + }, + (value) => { + ;(mod.value as ForloopFlow).parallelism = { + type: 'static', + value + } + } + } + /> + { + const forLoopFlow = mod.value as ForloopFlow + if (e.detail == parallelismType) return + if (e.detail === 'javascript') { + if (!forLoopFlow.parallelism || forLoopFlow.parallelism.type !== 'javascript') { + ;(mod.value as ForloopFlow).parallelism = { + type: 'javascript', + expr: '' + } + } + } else { + if (!forLoopFlow.parallelism || forLoopFlow.parallelism.type !== 'static') { + ;(mod.value as ForloopFlow).parallelism = { + type: 'static', + value: 0 + } + } + } + }} + class="h-6" + > + {#snippet children({ item })} + + + + {/snippet} + +
+ + {#if mod.value.type === 'forloopflow' && mod.value.parallel && mod.value.parallelism?.type == 'javascript'} +
+
+ Parallelism expression + + JavaScript expression that defines the maximum number of parallel executions. + Example: flow_input.max_parallel || 3 + +
+
+ +
+ { + parallelismEditor?.insertAtCursor(detail) + parallelismEditor?.focus() + }} + > + { + const parallelismExpr = (mod.value as ForloopFlow).parallelism + + return parallelismExpr && parallelismExpr.type === 'javascript' + ? parallelismExpr.expr + : '' + }, + (expr) => { + ;(mod.value as ForloopFlow).parallelism = { + type: 'javascript', + expr + } + } + } + class="small-editor" + shouldBindKey={false} + extraLib={stepPropPicker.extraLib} + /> + +
+ {/if}
Iterator expression diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 755f41569b..2f96993b09 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -373,7 +373,7 @@ components: parallel: type: boolean parallelism: - type: integer + $ref: "#/components/schemas/InputTransform" required: - modules - iterator @@ -396,7 +396,7 @@ components: parallel: type: boolean parallelism: - type: integer + $ref: "#/components/schemas/InputTransform" required: - modules - skip_failures