mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-17 00:20:51 +00:00
feat: cap concurrent branches of a parallel branch-all step (#12211)
* feat: cap concurrent branches of a parallel branch-all step Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: reject a zero parallelism cap and regenerate flow guidance Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
This commit is contained in:
co-authored by
Claude Opus 5.5
Ruben Fiszel
parent
0ab7290f67
commit
b52eb8b6c2
File diff suppressed because one or more lines are too long
@@ -1127,6 +1127,8 @@ pub enum FlowModuleValue {
|
||||
branches: Vec<Branch>,
|
||||
#[serde(default = "default_true")]
|
||||
parallel: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
parallelism: Option<InputTransform>,
|
||||
},
|
||||
RawScript {
|
||||
#[serde(default)]
|
||||
@@ -1295,6 +1297,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue {
|
||||
.branches
|
||||
.ok_or_else(|| serde::de::Error::missing_field("branches"))?,
|
||||
parallel: untagged.parallel.unwrap_or(true),
|
||||
parallelism: untagged.parallelism,
|
||||
}),
|
||||
"rawscript" => Ok(FlowModuleValue::RawScript {
|
||||
input_transforms: untagged.input_transforms.unwrap_or_default(),
|
||||
@@ -1754,6 +1757,19 @@ mod tests {
|
||||
assert_eq!(skips, vec![false, true]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn branchall_parallelism_round_trips() {
|
||||
// Deploying a flow re-serializes every module value; a dropped cap would start
|
||||
// every branch at once.
|
||||
let input = json!({ "type": "branchall", "branches": [], "parallelism": 2 });
|
||||
let val: FlowModuleValue = serde_json::from_value(input).unwrap();
|
||||
let output = serde_json::to_value(&val).unwrap();
|
||||
assert_eq!(
|
||||
output["parallelism"],
|
||||
json!({ "type": "static", "value": 2 })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_after_if_omits_null_error_message() {
|
||||
let input = json!({ "expr": "result == 404", "skip_if_stopped": true });
|
||||
|
||||
@@ -634,6 +634,14 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
FlowStatusModule::InProgress { branchall: Some(_), parallel, .. } => (true, *parallel),
|
||||
_ => (false, false),
|
||||
};
|
||||
let parallelism = if parallel_branchall {
|
||||
current_module
|
||||
.as_ref()
|
||||
.and_then(|x| x.get_value_with_skip_failures().ok())
|
||||
.and_then(|x| x.parallelism)
|
||||
} else {
|
||||
parallelism
|
||||
};
|
||||
|
||||
// 0 length flows are not failure steps
|
||||
let is_failure_step =
|
||||
@@ -5244,11 +5252,15 @@ async fn push_next_flow_job(
|
||||
}
|
||||
}
|
||||
|
||||
if value_with_parallel.type_ == "forloopflow"
|
||||
&& value_with_parallel.parallel.unwrap_or(false)
|
||||
{
|
||||
// A branch-all is parallel unless `parallel: false`, a for-loop only with `parallel: true`.
|
||||
let is_parallel_step = match value_with_parallel.type_.as_str() {
|
||||
"forloopflow" => value_with_parallel.parallel.unwrap_or(false),
|
||||
"branchall" => value_with_parallel.parallel.unwrap_or(true),
|
||||
_ => false,
|
||||
};
|
||||
if is_parallel_step {
|
||||
if let Some(parallelism_transform) = &value_with_parallel.parallelism {
|
||||
tracing::debug!(id = %flow_job.id, root_id = %job_root, "evaluating parallelism expression for forloopflow job {uuid}");
|
||||
tracing::debug!(id = %flow_job.id, root_id = %job_root, "evaluating parallelism expression for {} job {uuid}", value_with_parallel.type_);
|
||||
|
||||
let ctx = get_transform_context(&flow_job, &previous_id, &status);
|
||||
|
||||
@@ -5261,8 +5273,15 @@ async fn push_next_flow_job(
|
||||
Some(&ctx),
|
||||
)
|
||||
.await?;
|
||||
// A cap of 0 suspends every child, and only a finishing child releases one.
|
||||
if evaluated_parallelism == 0 {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"parallelism of step {} must be at least 1",
|
||||
module.id
|
||||
)));
|
||||
}
|
||||
|
||||
tracing::debug!(id = %flow_job.id, root_id = %job_root, "updating suspend for forloopflow job {uuid} with parallelism {evaluated_parallelism}");
|
||||
tracing::debug!(id = %flow_job.id, root_id = %job_root, "updating suspend for parallel job {uuid} with parallelism {evaluated_parallelism}");
|
||||
|
||||
if i as u16 >= evaluated_parallelism {
|
||||
sqlx::query!(
|
||||
|
||||
@@ -1710,7 +1710,7 @@ async fn lock_modules(
|
||||
}
|
||||
.into()
|
||||
}
|
||||
FlowModuleValue::BranchAll { branches, parallel } => {
|
||||
FlowModuleValue::BranchAll { branches, parallel, parallelism } => {
|
||||
let mut nbranches = vec![];
|
||||
for mut b in branches {
|
||||
let nmodules;
|
||||
@@ -1743,7 +1743,9 @@ async fn lock_modules(
|
||||
b.modules = nmodules;
|
||||
nbranches.push(b)
|
||||
}
|
||||
e.value = FlowModuleValue::BranchAll { branches: nbranches, parallel }.into()
|
||||
e.value =
|
||||
FlowModuleValue::BranchAll { branches: nbranches, parallel, parallelism }
|
||||
.into()
|
||||
}
|
||||
FlowModuleValue::WhileloopFlow {
|
||||
modules,
|
||||
|
||||
Generated
+1
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -22,6 +22,12 @@
|
||||
import FlowCard from '../common/FlowCard.svelte'
|
||||
import FlowRunSettings from './FlowRunSettings.svelte'
|
||||
import { useUiIntent } from '$lib/components/copilot/chat/flow/useUiIntent'
|
||||
import SimpleEditor from '$lib/components/SimpleEditor.svelte'
|
||||
import InputTransformForm from '$lib/components/InputTransformForm.svelte'
|
||||
import PropPickerWrapper from '../propPicker/PropPickerWrapper.svelte'
|
||||
import { getStepPropPicker } from '../previousResults'
|
||||
import { emptySchema } from '$lib/utils'
|
||||
import { slideDynamic } from '$lib/transitions'
|
||||
|
||||
interface Props {
|
||||
noEditor: boolean
|
||||
@@ -79,7 +85,29 @@
|
||||
dragging = false
|
||||
}
|
||||
|
||||
const { flowStore, flowStateStore, history } = getContext<FlowEditorContext>('FlowEditorContext')
|
||||
const { flowStore, flowStateStore, history, previewArgs } =
|
||||
getContext<FlowEditorContext>('FlowEditorContext')
|
||||
|
||||
if (typeof value.parallelism === 'number') {
|
||||
value.parallelism = { type: 'static', value: value.parallelism }
|
||||
}
|
||||
|
||||
const DEFAULT_PARALLELISM = 2
|
||||
let parallelismEditor: SimpleEditor | undefined = $state(undefined)
|
||||
let parallelismSchema = $state(emptySchema())
|
||||
parallelismSchema.properties['parallelism'] = { type: 'integer' }
|
||||
const parallelismCapped = $derived(value.parallelism != undefined)
|
||||
let stepPropPicker = $derived(
|
||||
getStepPropPicker(
|
||||
flowStateStore.val,
|
||||
parentModule,
|
||||
previousModule,
|
||||
flowModule.id,
|
||||
flowStore.val,
|
||||
previewArgs.val,
|
||||
false
|
||||
)
|
||||
)
|
||||
|
||||
function addBranch() {
|
||||
addBranchOp(flowModule.id, { flowStore, history })
|
||||
@@ -109,6 +137,23 @@
|
||||
})
|
||||
</script>
|
||||
|
||||
{#snippet parallelismToggle()}
|
||||
<Toggle
|
||||
size="xs"
|
||||
textClass="text-xs font-normal text-primary"
|
||||
checked={parallelismCapped}
|
||||
on:change={({ detail }) => {
|
||||
value.parallelism = detail ? { type: 'static', value: DEFAULT_PARALLELISM } : undefined
|
||||
}}
|
||||
options={{
|
||||
right: 'Limit concurrent branches',
|
||||
rightTooltip:
|
||||
'Cap how many branches run at once. The others wait and start as running ones finish. Without a cap every branch starts at once.',
|
||||
rightDocumentationLink: 'https://www.windmill.dev/docs/flows/flow_branches#branch-all'
|
||||
}}
|
||||
/>
|
||||
{/snippet}
|
||||
|
||||
<div class="h-full flex flex-col w-full" id="flow-editor-branch-all-wrapper">
|
||||
<FlowCard
|
||||
{noEditor}
|
||||
@@ -206,10 +251,43 @@
|
||||
<Toggle
|
||||
id="branchall-parallel-{flowModule.id}"
|
||||
bind:checked={value.parallel}
|
||||
on:change={({ detail }) => {
|
||||
// An absent `parallelism` means "no cap" to the worker.
|
||||
if (!detail) value.parallelism = undefined
|
||||
}}
|
||||
options={{
|
||||
right: 'All branches run in parallel'
|
||||
}}
|
||||
/>
|
||||
{#if value.parallel}
|
||||
<div class="mt-2 pl-9" transition:slideDynamic>
|
||||
<PropPickerWrapper
|
||||
sidePane
|
||||
flow_input={stepPropPicker.pickableProperties.flow_input}
|
||||
notSelectable
|
||||
pickableProperties={stepPropPicker.pickableProperties}
|
||||
on:select={({ detail }) => {
|
||||
parallelismEditor?.insertAtCursor(detail)
|
||||
parallelismEditor?.focus()
|
||||
}}
|
||||
>
|
||||
<!-- Keyed on the toggle: the form seeds its static/expression mode once. -->
|
||||
{#key parallelismCapped}
|
||||
<InputTransformForm
|
||||
bind:arg={() => value.parallelism, (v) => (value.parallelism = v)}
|
||||
argName="parallelism"
|
||||
collapsed={!parallelismCapped}
|
||||
header={parallelismToggle}
|
||||
schema={parallelismSchema}
|
||||
argExtra={{ min: 1, step: 1 }}
|
||||
animateAppear
|
||||
previousModuleId={previousModule?.id}
|
||||
bind:editor={parallelismEditor}
|
||||
/>
|
||||
{/key}
|
||||
</PropPickerWrapper>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
</section>
|
||||
{:else}
|
||||
|
||||
@@ -1015,6 +1015,9 @@ components:
|
||||
parallel:
|
||||
type: boolean
|
||||
description: If true, all branches execute concurrently. If false, they execute sequentially
|
||||
parallelism:
|
||||
description: Maximum number of branches running at once when parallel=true. Can be static number or expression
|
||||
$ref: '#/components/schemas/InputTransform'
|
||||
required:
|
||||
- branches
|
||||
- type
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user