mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-17 00:20:51 +00:00
feat: add max iterations and max duration bounds to while loops (#12245)
* feat: add max iterations and max duration bounds to while loops Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: fail exhausted while loops regardless of skip_failures and prior recovery Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * docs: state that a while loop's duration limit can overrun by one iteration Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: let the while loop iterations field be cleared and retyped Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
This commit is contained in:
co-authored by
Claude Opus 5.5
Ruben Fiszel
parent
afc6e9d3ca
commit
71beff7438
@@ -7122,3 +7122,39 @@ async fn test_results_length_in_input_transform(db: Pool<Postgres>) -> anyhow::R
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "quickjs")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_whileloop_max_iterations_fails_with_loop_exhausted(
|
||||
db: Pool<Postgres>,
|
||||
) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
|
||||
let flow: FlowValue = serde_json::from_value(serde_json::json!({
|
||||
"modules": [{
|
||||
"id": "loop",
|
||||
"value": {
|
||||
"type": "whileloopflow",
|
||||
// skip_failures absorbs failed iterations, never the loop's own exhaustion
|
||||
"skip_failures": true,
|
||||
"max_iterations": 3,
|
||||
"modules": [{ "id": "inner", "value": { "type": "identity" } }],
|
||||
},
|
||||
}],
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
let job = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None })
|
||||
.run_until_complete(&db, false, server.addr.port())
|
||||
.await;
|
||||
|
||||
assert!(!job.success);
|
||||
let error = job.json_result().unwrap()["error"].clone();
|
||||
assert_eq!(error["name"], "LoopExhausted", "{error}");
|
||||
let iterations = job.flow_status.unwrap()["modules"][0]["flow_jobs"]
|
||||
.as_array()
|
||||
.map(|a| a.len());
|
||||
assert_eq!(iterations, Some(3));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -667,6 +667,8 @@ pub struct FlowModuleValueWithParallel {
|
||||
#[derive(Deserialize)]
|
||||
pub struct FlowModuleValueWithSkipFailures {
|
||||
pub skip_failures: Option<bool>,
|
||||
pub max_iterations: Option<u32>,
|
||||
pub max_duration_secs: Option<u32>,
|
||||
pub parallel: Option<bool>,
|
||||
#[serde(
|
||||
default,
|
||||
@@ -1107,6 +1109,10 @@ pub enum FlowModuleValue {
|
||||
skip_failures: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
squash: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
max_iterations: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
max_duration_secs: Option<u32>,
|
||||
},
|
||||
BranchOne {
|
||||
branches: Vec<Branch>,
|
||||
@@ -1218,6 +1224,8 @@ struct UntaggedFlowModuleValue {
|
||||
tool_inputs: Option<HashMap<String, HashMap<String, InputTransform>>>,
|
||||
pass_flow_input_directly: Option<bool>,
|
||||
squash: Option<bool>,
|
||||
max_iterations: Option<u32>,
|
||||
max_duration_secs: Option<u32>,
|
||||
#[serde(flatten)]
|
||||
concurrency_settings: ConcurrencySettingsWithCustom,
|
||||
}
|
||||
@@ -1267,6 +1275,8 @@ impl<'de> Deserialize<'de> for FlowModuleValue {
|
||||
modules_node: untagged.modules_node,
|
||||
skip_failures: untagged.skip_failures.unwrap_or(false),
|
||||
squash: untagged.squash,
|
||||
max_iterations: untagged.max_iterations,
|
||||
max_duration_secs: untagged.max_duration_secs,
|
||||
}),
|
||||
"branchone" => Ok(FlowModuleValue::BranchOne {
|
||||
branches: untagged
|
||||
@@ -1733,8 +1743,11 @@ mod tests {
|
||||
let val: FlowModuleValue = serde_json::from_value(input).unwrap();
|
||||
let output = serde_json::to_value(&val).unwrap();
|
||||
let reparsed: FlowModuleWithBranches = serde_json::from_value(output).unwrap();
|
||||
let skips: Vec<bool> =
|
||||
reparsed.branches.iter().map(|b| b.skip_failure.unwrap_or(false)).collect();
|
||||
let skips: Vec<bool> = reparsed
|
||||
.branches
|
||||
.iter()
|
||||
.map(|b| b.skip_failure.unwrap_or(false))
|
||||
.collect();
|
||||
assert_eq!(skips, vec![false, true]);
|
||||
}
|
||||
|
||||
|
||||
@@ -605,7 +605,7 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
// "UPDATE FLOW STATUS 2: {module_step:#?} {module_status:#?} {old_status:#?} "
|
||||
// );
|
||||
|
||||
let (is_loop, skip_loop_failures, parallelism, parallel_loop) =
|
||||
let (is_loop, skip_loop_failures, parallelism, parallel_loop, loop_bounds) =
|
||||
if let FlowStatusModule::InProgress { iterator: Some(_), parallel, .. } = module_status
|
||||
{
|
||||
let value = current_module
|
||||
@@ -617,11 +617,17 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
.as_ref()
|
||||
.and_then(|x| x.skip_failures)
|
||||
.unwrap_or(false),
|
||||
value.and_then(|x| x.parallelism),
|
||||
value.as_ref().and_then(|x| x.parallelism.clone()),
|
||||
*parallel,
|
||||
value.map(|x| {
|
||||
(
|
||||
x.max_iterations.filter(|m| *m > 0),
|
||||
x.max_duration_secs.filter(|m| *m > 0),
|
||||
)
|
||||
}),
|
||||
)
|
||||
} else {
|
||||
(false, false, None, false)
|
||||
(false, false, None, false, None)
|
||||
};
|
||||
|
||||
let (is_branch_all, parallel_branchall) = match module_status {
|
||||
@@ -767,6 +773,53 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
(false, None, false, false, false)
|
||||
};
|
||||
|
||||
// A while loop that reached its iteration or duration bound without its stop condition
|
||||
// being met fails. The loop's start is the first iteration's start, which the module
|
||||
// status persists, so the duration bound holds across worker restarts.
|
||||
let loop_exhausted_msg = match (module_status, loop_bounds) {
|
||||
(
|
||||
FlowStatusModule::InProgress {
|
||||
iterator: Some(FlowIterator { index, .. }),
|
||||
while_loop: true,
|
||||
parallel: false,
|
||||
flow_jobs_duration,
|
||||
..
|
||||
},
|
||||
Some((max_iterations, max_duration_secs)),
|
||||
) if (success || skip_loop_failures) && !stop_early => {
|
||||
let iterations = index + 1;
|
||||
let started_at = flow_jobs_duration
|
||||
.as_ref()
|
||||
.and_then(|d| d.started_at.first().copied().flatten())
|
||||
.or_else(|| flow_job_duration.as_ref().map(|d| d.started_at));
|
||||
let elapsed_secs =
|
||||
started_at.map(|s| (chrono::Utc::now() - s).num_milliseconds() as f64 / 1000.0);
|
||||
if max_iterations.is_some_and(|m| iterations >= m as usize) {
|
||||
Some(format!(
|
||||
"While loop reached its maximum of {iterations} iterations without its stop condition being met"
|
||||
))
|
||||
} else {
|
||||
match (max_duration_secs, elapsed_secs) {
|
||||
(Some(m), Some(e)) if e >= m as f64 => Some(format!(
|
||||
"While loop reached its maximum duration of {m}s ({e:.1}s elapsed, {iterations} iterations) without its stop condition being met"
|
||||
)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let loop_exhausted_err = loop_exhausted_msg.as_ref().map(|message| {
|
||||
Arc::new(to_raw_value(&serde_json::json!({
|
||||
"error": { "name": "LoopExhausted", "message": message }
|
||||
})))
|
||||
});
|
||||
// skip_failures tolerates failed iterations, not the loop failing as a whole.
|
||||
let skip_loop_failures = skip_loop_failures && loop_exhausted_err.is_none();
|
||||
if loop_exhausted_err.is_some() {
|
||||
has_triggered_error_handler = false;
|
||||
}
|
||||
|
||||
let skip_seq_branch_failure = match module_status {
|
||||
FlowStatusModule::InProgress {
|
||||
branchall: Some(BranchAllStatus { branch, .. }),
|
||||
@@ -1172,6 +1225,7 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
(*while_loop || *index + 1 < itered_len)
|
||||
&& (success || skip_loop_failures)
|
||||
&& !stop_early
|
||||
&& loop_exhausted_err.is_none()
|
||||
} =>
|
||||
{
|
||||
if let Some(jobs) = flow_jobs {
|
||||
@@ -1299,6 +1353,7 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
if (success
|
||||
|| (flow_jobs.is_some() && (skip_loop_failures || skip_seq_branch_failure)))
|
||||
&& !(stop_early && stop_early_err_msg.is_some() && !skip_if_stop_early)
|
||||
&& loop_exhausted_err.is_none()
|
||||
{
|
||||
let is_skipped = (stop_early && skip_if_stop_early)
|
||||
|| if current_module.as_ref().is_some_and(|m| m.skip_if.is_some()) {
|
||||
@@ -1340,7 +1395,7 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
evaluate_retry(
|
||||
&retry,
|
||||
&old_status.retry,
|
||||
result.clone(),
|
||||
loop_exhausted_err.clone().unwrap_or_else(|| result.clone()),
|
||||
Marc::new(args),
|
||||
resolved_flow_env.as_deref(),
|
||||
Some(client),
|
||||
@@ -1350,7 +1405,8 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
} else {
|
||||
false
|
||||
};
|
||||
if stop_early && stop_early_err_msg.is_some() {
|
||||
if (stop_early && stop_early_err_msg.is_some()) || loop_exhausted_err.is_some()
|
||||
{
|
||||
// if stop_early with error message, we need to set explictely success to false as this function was called with success = true as the job succeeded
|
||||
success = false;
|
||||
}
|
||||
@@ -1391,6 +1447,10 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
)));
|
||||
}
|
||||
|
||||
if loop_exhausted_err.is_some() {
|
||||
nresult = loop_exhausted_err.clone();
|
||||
}
|
||||
|
||||
let step_counter = if inc_step_counter {
|
||||
old_status.step + 1
|
||||
} else {
|
||||
@@ -1781,7 +1841,7 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
let should_retry = evaluate_retry(
|
||||
retry_config,
|
||||
&old_status.retry,
|
||||
result.clone(),
|
||||
loop_exhausted_err.clone().unwrap_or_else(|| result.clone()),
|
||||
Marc::new(args),
|
||||
resolved_flow_env.as_deref(),
|
||||
Some(client),
|
||||
@@ -1881,9 +1941,9 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
};
|
||||
|
||||
let failure_step_recovers = is_failure_step && result_has_recover_true(nresult.clone());
|
||||
// An error this flow raises on its own (a stop predicate's error message) is a new
|
||||
// failure that no error handler recovered.
|
||||
if stop_early_err_msg.is_some() {
|
||||
// An error this flow raises on its own (a stop predicate's error message, a while loop
|
||||
// reaching its bound) is a new failure that no error handler recovered or saw yet.
|
||||
if stop_early_err_msg.is_some() || loop_exhausted_msg.is_some() {
|
||||
recovered_by_child = false;
|
||||
}
|
||||
|
||||
|
||||
@@ -1745,7 +1745,14 @@ async fn lock_modules(
|
||||
}
|
||||
e.value = FlowModuleValue::BranchAll { branches: nbranches, parallel }.into()
|
||||
}
|
||||
FlowModuleValue::WhileloopFlow { modules, modules_node, skip_failures, squash } => {
|
||||
FlowModuleValue::WhileloopFlow {
|
||||
modules,
|
||||
modules_node,
|
||||
skip_failures,
|
||||
squash,
|
||||
max_iterations,
|
||||
max_duration_secs,
|
||||
} => {
|
||||
let nmodules;
|
||||
(nmodules, nmodified_ids, nerrors) = Box::pin(lock_modules(
|
||||
modules,
|
||||
@@ -1774,6 +1781,8 @@ async fn lock_modules(
|
||||
modules_node,
|
||||
skip_failures,
|
||||
squash,
|
||||
max_iterations,
|
||||
max_duration_secs,
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
Generated
+2
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -12,6 +12,10 @@
|
||||
import StepSettingsBadges from './StepSettingsBadges.svelte'
|
||||
import FlowModuleEarlyStop from './FlowModuleEarlyStop.svelte'
|
||||
import { useUiIntent } from '$lib/components/copilot/chat/flow/useUiIntent'
|
||||
import Label from '$lib/components/Label.svelte'
|
||||
import TextInput from '$lib/components/text_input/TextInput.svelte'
|
||||
import { SecondsInput } from '$lib/components/common'
|
||||
import { slideDynamic } from '$lib/transitions'
|
||||
|
||||
const { flowStateStore } = getContext<FlowEditorContext>('FlowEditorContext')
|
||||
|
||||
@@ -33,6 +37,13 @@
|
||||
let previewIterationArgs = $derived(flowStateStore.val[mod.id]?.previewArgs ?? {})
|
||||
|
||||
let selectedTab = $state('loop')
|
||||
// Keeps "Limit iterations" open while its field is emptied mid-edit; the model holds no
|
||||
// limit meanwhile, and leaving the field empty turns the limit off.
|
||||
let iterationsFieldEmpty = $state(false)
|
||||
let isIterationsLimited = $derived(
|
||||
(mod.value.type === 'whileloopflow' && mod.value.max_iterations !== undefined) ||
|
||||
iterationsFieldEmpty
|
||||
)
|
||||
|
||||
useUiIntent(`whileloopflow-${mod.id}`, {
|
||||
openTab: async (tab) => {
|
||||
@@ -103,14 +114,81 @@
|
||||
>
|
||||
Add steps inside the while loop but have one of them use early stop/break in their Run
|
||||
settings (or do it at the loop level that will watch the last step) to break out of
|
||||
the while loop (otherwise it will loop forever and you will have to cancel the flow
|
||||
manually).
|
||||
the while loop. Without a break or a limit below, it loops until you cancel the flow.
|
||||
</Alert>
|
||||
{/if}
|
||||
|
||||
<section class="flex flex-col gap-6">
|
||||
<FlowModuleEarlyStop blocks="stop-after" bind:flowModule={mod} />
|
||||
|
||||
<div class="flex flex-col gap-2">
|
||||
<Toggle
|
||||
size="xs"
|
||||
textClass="text-xs font-normal text-primary"
|
||||
checked={isIterationsLimited}
|
||||
on:change={({ detail }) => {
|
||||
iterationsFieldEmpty = false
|
||||
;(mod.value as WhileloopFlow).max_iterations = detail ? 10 : undefined
|
||||
}}
|
||||
options={{
|
||||
right: 'Limit iterations',
|
||||
rightTooltip:
|
||||
'Checked after each iteration. When the limit is reached and the break condition did not stop the loop, the loop fails with an error named LoopExhausted, handled like any step failure (retries, error handler, or Continue on error then a branch on the error).'
|
||||
}}
|
||||
/>
|
||||
{#if isIterationsLimited}
|
||||
<div class="flex flex-col gap-2 pl-9" transition:slideDynamic>
|
||||
<Label label="Maximum iterations">
|
||||
<TextInput
|
||||
class="w-24"
|
||||
bind:value={
|
||||
() => (mod.value as WhileloopFlow).max_iterations,
|
||||
(v) => {
|
||||
const n = Math.floor(Number(v))
|
||||
iterationsFieldEmpty = !(n >= 1)
|
||||
;(mod.value as WhileloopFlow).max_iterations = n >= 1 ? n : undefined
|
||||
}
|
||||
}
|
||||
inputProps={{
|
||||
type: 'number',
|
||||
min: 1,
|
||||
onblur: () => (iterationsFieldEmpty = false)
|
||||
}}
|
||||
/>
|
||||
</Label>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<div class="flex flex-col gap-2">
|
||||
<Toggle
|
||||
size="xs"
|
||||
textClass="text-xs font-normal text-primary"
|
||||
checked={mod.value.max_duration_secs !== undefined}
|
||||
on:change={({ detail }) => {
|
||||
;(mod.value as WhileloopFlow).max_duration_secs = detail ? 600 : undefined
|
||||
}}
|
||||
options={{
|
||||
right: 'Limit duration',
|
||||
rightTooltip:
|
||||
'Checked after each iteration, so the loop can run up to one iteration past the limit. Set a timeout on the steps inside to cap an iteration. When the limit is reached before the break condition, the loop fails with LoopExhausted, handled like any step failure.'
|
||||
}}
|
||||
/>
|
||||
{#if mod.value.max_duration_secs !== undefined}
|
||||
<div class="flex flex-col gap-2 pl-9" transition:slideDynamic>
|
||||
<Label label="Maximum duration">
|
||||
<SecondsInput
|
||||
bind:seconds={
|
||||
() => (mod.value as WhileloopFlow).max_duration_secs,
|
||||
(v) =>
|
||||
((mod.value as WhileloopFlow).max_duration_secs = Math.max(1, v ?? 1))
|
||||
}
|
||||
/>
|
||||
</Label>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<Toggle
|
||||
size="xs"
|
||||
textClass="text-xs font-normal text-primary"
|
||||
|
||||
@@ -931,6 +931,14 @@ components:
|
||||
$ref: '#/components/schemas/InputTransform'
|
||||
squash:
|
||||
type: boolean
|
||||
max_iterations:
|
||||
type: integer
|
||||
minimum: 1
|
||||
description: Maximum number of iterations. Checked after each iteration; when reached and stop_after_if did not stop the loop, the loop fails with an error named 'LoopExhausted'
|
||||
max_duration_secs:
|
||||
type: integer
|
||||
minimum: 1
|
||||
description: Maximum wall-clock seconds since the first iteration started, checked after each iteration. A running iteration is not interrupted, so the loop can exceed the limit by up to one iteration; the timeout of the body's steps caps an iteration. When reached and stop_after_if did not stop the loop, the loop fails with an error named 'LoopExhausted'
|
||||
required:
|
||||
- modules
|
||||
- skip_failures
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -316,6 +316,7 @@ can call one, not an agent used as a tool.
|
||||
- `stop_after_if` is evaluated after each iteration: on the loop module, `result` is that iteration's result (what its last step returned); on an inner step, it is that step's result
|
||||
- `flow_input.iter.value` in a `whileloopflow` is just the iteration index (same number as `flow_input.iter.index`) — it never carries state, so `flow_input.iter.value.<field>` is always undefined and a loop whose stop condition depends on it never terminates
|
||||
- To carry state across iterations, a step reads its own previous-iteration result via `results.<its_own_id>` with a first-iteration fallback (e.g. `results.b ?? flow_input.start`) — but then the loop's `stop_after_if` MUST sit on that inner step, not on the loop module: a body that is exactly one plain step with the stop condition on the loop module runs on a fast path where `results.<step_id>` is null on every iteration and the loop never terminates (bodies with 2+ steps, or whose single step has its own `stop_after_if`, retry or similar, resolve `results` across iterations regardless of stop placement)
|
||||
- To cap a `whileloopflow`, set `max_iterations` and/or `max_duration_secs` (seconds since the first iteration started) on the loop's `value`, rather than encoding the bound into `stop_after_if`. Both are checked after each iteration, so a running iteration is never interrupted and the loop can exceed `max_duration_secs` by at most one iteration (bound an iteration with its steps' `timeout`); whichever is reached first, unless `stop_after_if` stopped the loop at that iteration, fails the loop with `{ error: { name: "LoopExhausted", message } }`. To handle it, set `continue_on_error: true` on the loop and branch in the next step on `results.<loop_id>.error?.name === "LoopExhausted"` (only the step right after the loop sees the error there; later steps see the iterations' results); retries and the failure module apply as for any step failure. For a pause between iterations, set `sleep` on the body's step
|
||||
- For state that is just a counter, derive it from the index instead (e.g. `flow_input.iter.index + 1`) — that works in every configuration, including with `stop_after_if` on the loop module
|
||||
- If the user asks for a final scalar/object after a loop, add a normal step after the loop that extracts the final value from the loop result instead of returning the whole loop result array
|
||||
|
||||
|
||||
Reference in New Issue
Block a user