starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-06 16:02:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: improve early stop (#4257)

Browse Source
This commit is contained in:
HugoCasa
2024-08-21 00:47:34 +02:00
committed by GitHub
parent 776e978752
commit bcde2e62d7
13 changed files with 314 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/worker.rs
+8
View File
@@ -1113,6 +1113,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
}
.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1152,6 +1153,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
}
.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1166,6 +1168,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
}
.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1270,6 +1273,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrency_time_window_s: None,
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1319,6 +1323,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrency_time_window_s: None,
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1354,6 +1359,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrency_time_window_s: None,
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1369,6 +1375,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
],
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1411,6 +1418,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrency_time_window_s: None,
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
backend/windmill-api/src/flows.rs
+4
View File
@@ -1144,6 +1144,7 @@ mod tests {
tag_override: None,
}),
stop_after_if: None,
stop_after_all_iters_if: None,
summary: None,
suspend: Default::default(),
retry: None,
@@ -1172,6 +1173,7 @@ mod tests {
expr: "foo = 'bar'".to_string(),
skip_if_stopped: false,
}),
stop_after_all_iters_if: None,
summary: None,
suspend: Default::default(),
retry: None,
@@ -1198,6 +1200,7 @@ mod tests {
expr: "previous.isEmpty()".to_string(),
skip_if_stopped: false,
}),
stop_after_all_iters_if: None,
summary: None,
suspend: Default::default(),
retry: None,
@@ -1223,6 +1226,7 @@ mod tests {
expr: "previous.isEmpty()".to_string(),
skip_if_stopped: false,
}),
stop_after_all_iters_if: None,
summary: None,
suspend: Default::default(),
retry: None,
backend/windmill-common/src/flows.rs
+3
View File
@@ -236,6 +236,8 @@ pub struct FlowModule {
#[serde(skip_serializing_if = "Option::is_none")]
pub stop_after_if: Option<StopAfterIf>,
#[serde(skip_serializing_if = "Option::is_none")]
pub stop_after_all_iters_if: Option<StopAfterIf>,
#[serde(skip_serializing_if = "Option::is_none")]
pub summary: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub suspend: Option<Suspend>,
@@ -580,6 +582,7 @@ pub fn add_virtual_items_if_necessary(modules: &mut Vec<FlowModule>) {
id: format!("{}-v", modules[modules.len() - 1].id),
value: crate::worker::to_raw_value(&FlowModuleValue::Identity),
stop_after_if: None,
stop_after_all_iters_if: None,
summary: Some("Virtual module needed for suspend/sleep when last module".to_string()),
mock: None,
retry: None,
backend/windmill-queue/src/jobs.rs
+1
View File
@@ -3349,6 +3349,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
},
),
stop_after_if: None,
stop_after_all_iters_if: None,
summary: None,
suspend: None,
mock: None,
backend/windmill-worker/src/worker_flow.rs
+72 -4
View File
@@ -232,22 +232,27 @@ pub async fn update_flow_status_after_job_completion_internal<
// "UPDATE FLOW STATUS 2: {module_index:#?} {module_status:#?} {old_status:#?} "
// );
let (skip_loop_failures, parallelism) = if matches!(
let (is_loop, skip_loop_failures, parallelism) = if matches!(
module_status,
FlowStatusModule::InProgress { iterator: Some(_), .. }
) {
let (loop_failures, parallelism) =
compute_skip_loop_failures_and_parallelism(flow, old_status.step, db).await?;
(loop_failures.unwrap_or(false), parallelism)
(true, loop_failures.unwrap_or(false), parallelism)
} else {
(false, None)
(false, false, None)
};
let is_branch_all = matches!(
module_status,
FlowStatusModule::InProgress { branchall: Some(_), .. }
);
// 0 length flows are not failure steps
let is_failure_step =
old_status.step >= old_status.modules.len() as i32 && old_status.modules.len() > 0;
let (mut stop_early, skip_if_stop_early, continue_on_error) = if let Some(se) =
let (mut stop_early, mut skip_if_stop_early, continue_on_error) = if let Some(se) =
stop_early_override
{
//do not stop early if module is a flow step
@@ -280,7 +285,18 @@ pub async fn update_flow_status_after_job_completion_internal<
.map_err(|e| Error::InternalErr(format!("retrieval of stop_early_expr from state: {e:#}")))?;
let stop_early = success
&& !is_branch_all
&& if let Some(expr) = r.stop_early_expr.clone() {
let all_iters = match &module_status {
FlowStatusModule::InProgress { flow_jobs: Some(flow_jobs), .. }
if expr.contains("all_iters") =>
{
Some(Arc::new(
retrieve_flow_jobs_results(db, w_id, flow_jobs).await?,
))
}
_ => None,
};
compute_bool_from_expr(
expr,
Marc::new(
@@ -290,6 +306,7 @@ pub async fn update_flow_status_after_job_completion_internal<
.to_owned(),
),
result.clone(),
all_iters,
None,
Some(client),
None,
@@ -735,6 +752,50 @@ pub async fn update_flow_status_after_job_completion_internal<
_ => result.clone(),
};
match &new_status {
Some(FlowStatusModule::Success { .. }) if is_loop || is_branch_all => {
let r_after_all_iters = sqlx::query_as::<_, SkipIfStopped>(
"SELECT
raw_flow->'modules'->$1::int->'stop_after_all_iters_if'->>'expr' as stop_early_expr,
(raw_flow->'modules'->$1::int->'stop_after_all_iters_if'->>'skip_if_stopped')::bool as skip_if_stopped,
NULL as continue_on_error,
args
FROM queue
WHERE id = $2"
)
.bind(old_status.step)
.bind(flow)
.fetch_one(db)
.await
.map_err(|e| Error::InternalErr(format!("retrieval of stop_early_expr from state: {e:#}")))?;
if let Some(expr) = r_after_all_iters.stop_early_expr {
let should_stop = compute_bool_from_expr(
expr,
Marc::new(
r_after_all_iters
.args
.map(|x| x.0)
.unwrap_or_else(|| serde_json::from_str("{}").unwrap())
.to_owned(),
),
nresult.clone(),
None,
None,
Some(client),
None,
None,
)
.await?;
if should_stop {
stop_early = should_stop;
skip_if_stop_early = r_after_all_iters.skip_if_stopped.unwrap_or(false);
}
}
}
_ => {}
}
if old_status.retry.fail_count > 0
&& matches!(&new_status, Some(FlowStatusModule::Success { .. }))
{
@@ -1186,6 +1247,7 @@ async fn compute_bool_from_expr(
expr: String,
flow_args: Marc<HashMap<String, Box<RawValue>>>,
result: Arc<Box<RawValue>>,
all_iters: Option<Arc<Box<RawValue>>>,
by_id: Option<IdContext>,
client: Option<&AuthedClient>,
resumes: Option<(Arc<Box<RawValue>>, Arc<Box<RawValue>>, Arc<Box<RawValue>>)>,
@@ -1193,6 +1255,9 @@ async fn compute_bool_from_expr(
) -> error::Result<bool> {
let mut context = HashMap::with_capacity(if resumes.is_some() { 7 } else { 3 });
context.insert("result".to_string(), result.clone());
if let Some(all_iters) = all_iters {
context.insert("all_iters".to_string(), all_iters);
}
context.insert("previous_result".to_string(), result.clone());
if let Some(resumes) = resumes {
@@ -1568,6 +1633,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
arc_flow_job_args.clone(),
Arc::new(to_raw_value(&json!("{}"))),
None,
None,
Some(client),
None,
Some(vec![(
@@ -2982,6 +3048,7 @@ async fn compute_next_flow_transform(
b.expr.to_string(),
arc_flow_job_args.clone(),
arc_last_job_result.clone(),
None,
Some(idcontext.clone()),
Some(client),
Some((resumes.clone(), resume.clone(), approvers.clone())),
@@ -3258,6 +3325,7 @@ fn is_simple_modules(modules: &Vec<FlowModule>, flow: &FlowValue) -> bool {
&& modules[0].cache_ttl.is_none()
&& modules[0].retry.is_none()
&& modules[0].stop_after_if.is_none()
&& modules[0].stop_after_all_iters_if.is_none()
&& (modules[0].mock.is_none() || modules[0].mock.as_ref().is_some_and(|m| !m.enabled))
&& flow.failure_module.is_none();
is_simple
frontend/src/lib/components/flows/content/FlowModuleComponent.svelte
+6 -1
View File
@@ -402,7 +402,12 @@
{#if !$selectedId.includes('failure')}
<Tab value="runtime">Runtime</Tab>
<Tab value="cache" active={Boolean(flowModule.cache_ttl)}>Cache</Tab>
<Tab value="early-stop" active={Boolean(flowModule.stop_after_if)}>
<Tab
value="early-stop"
active={Boolean(
flowModule.stop_after_if || flowModule.stop_after_all_iters_if
)}
>
Early Stop
</Tab>
<Tab value="suspend" active={Boolean(flowModule.suspend)}>Suspend</Tab>
frontend/src/lib/components/flows/content/FlowModuleEarlyStop.svelte
+198 -70
View File
@@ -2,13 +2,14 @@
import SimpleEditor from '$lib/components/SimpleEditor.svelte'
import Toggle from '$lib/components/Toggle.svelte'
import PropPickerWrapper from '$lib/components/flows/propPicker/PropPickerWrapper.svelte'
import type { FlowModule } from '$lib/gen'
import type { Flow, FlowModule } from '$lib/gen'
import Tooltip from '$lib/components/Tooltip.svelte'
import type { FlowEditorContext } from '../types'
import { getContext } from 'svelte'
import { NEVER_TESTED_THIS_FAR } from '../models'
import Section from '$lib/components/Section.svelte'
import { getStepPropPicker } from '../previousResults'
import { dfs } from '../previousResults'
const { flowStateStore, flowStore, previewArgs } =
getContext<FlowEditorContext>('FlowEditorContext')
@@ -26,80 +27,207 @@
false
)
function checkIfParentLoop(flowStore: typeof $flowStore): string | null {
const flow: Flow = JSON.parse(JSON.stringify(flowStore))
const parents = dfs(flowModule.id, flow, true)
for (const parent of parents.slice(1)) {
if (parent.value.type === 'forloopflow' || parent.value.type === 'whileloopflow') {
return parent.id
}
}
return null
}
$: isLoop = flowModule.value.type === 'forloopflow' || flowModule.value.type === 'whileloopflow'
$: isBranchAll = flowModule.value.type === 'branchall'
$: isStopAfterIfEnabled = Boolean(flowModule.stop_after_if)
$: isStopAfterAllIterationsEnabled = Boolean(flowModule.stop_after_all_iters_if)
$: result = $flowStateStore[flowModule.id]?.previewResult ?? NEVER_TESTED_THIS_FAR
$: parentLoopId = checkIfParentLoop($flowStore)
</script>
<div class="flex flex-col items-start space-y-2 {$$props.class}">
<Section label="Early stop/Break" class="w-full">
<svelte:fragment slot="header">
<Tooltip documentationLink="https://www.windmill.dev/docs/flows/early_stop">
If defined, at the end of the step, the predicate expression will be evaluated to decide if
the flow should stop early.
</Tooltip>
</svelte:fragment>
<Toggle
checked={isStopAfterIfEnabled}
on:change={() => {
if (isStopAfterIfEnabled && flowModule.stop_after_if) {
flowModule.stop_after_if = undefined
} else {
flowModule.stop_after_if = {
expr: 'result == undefined',
skip_if_stopped: false
}
}
}}
options={{
right: 'Early stop or Break if condition met'
}}
/>
<div
class="w-full border p-2 flex flex-col {flowModule.stop_after_if
? ''
: 'bg-surface-secondary'}"
{#if !isBranchAll}
<Section
label={(isLoop
? 'Break loop'
: parentLoopId
? 'Break parent loop module ' + parentLoopId
: 'Stop flow early') + (isLoop ? ' (evaluated after each iteration)' : '')}
class="w-full"
>
{#if flowModule.stop_after_if}
<Toggle
size="xs"
bind:checked={flowModule.stop_after_if.skip_if_stopped}
options={{
right: 'Label flow as "skipped" if stopped'
}}
/>
<span class="mt-2 text-xs font-bold">Stop condition expression</span>
<div class="border w-full">
<PropPickerWrapper
notSelectable
flow_input={stepPropPicker.pickableProperties.flow_input}
pickableProperties={undefined}
{result}
on:select={({ detail }) => {
editor?.insertAtCursor(detail)
editor?.focus()
}}
>
<SimpleEditor
bind:this={editor}
lang="javascript"
bind:code={flowModule.stop_after_if.expr}
class="few-lines-editor"
extraLib={`declare const result = ${JSON.stringify(result)};`}
<svelte:fragment slot="header">
<Tooltip documentationLink="https://www.windmill.dev/docs/flows/early_stop">
If defined, at the end of the step, the predicate expression will be evaluated to decide
if the flow should stop early or break if inside a for/while loop.
</Tooltip>
</svelte:fragment>
<Toggle
checked={isStopAfterIfEnabled}
on:change={() => {
if (isStopAfterIfEnabled && flowModule.stop_after_if) {
flowModule.stop_after_if = undefined
} else {
flowModule.stop_after_if = {
expr: 'result == undefined',
skip_if_stopped: false
}
}
}}
options={{
right: isLoop
? 'Break loop'
: parentLoopId
? 'Break parent loop module'
: 'Stop flow' + ' if condition met'
}}
/>
<div
class="w-full border p-2 flex flex-col {flowModule.stop_after_if
? ''
: 'bg-surface-secondary'}"
>
{#if flowModule.stop_after_if}
{@const earlyStopResult = isLoop
? Array.isArray(result) && result.length > 0
? result[result.length - 1]
: result === NEVER_TESTED_THIS_FAR
? result
: undefined
: result}
{#if !parentLoopId && !isLoop}
<Toggle
size="xs"
bind:checked={flowModule.stop_after_if.skip_if_stopped}
options={{
right: 'Label flow as "skipped" if stopped'
}}
/>
</PropPickerWrapper>
</div>
{:else}
<Toggle
disabled
size="xs"
options={{
right: 'Label flow as "skipped" if stopped'
}}
/> <span class="mt-2 text-xs font-bold">Stop condition expression</span>
<textarea disabled rows="3" class="min-h-[80px]" />
{/if}
</div>
</Section>
{/if}
<span class="mt-2 text-xs font-bold">Stop condition expression</span>
<div class="border w-full">
<PropPickerWrapper
notSelectable
flow_input={stepPropPicker.pickableProperties.flow_input}
pickableProperties={undefined}
result={earlyStopResult}
extraResults={isLoop ? { all_iters: result } : undefined}
on:select={({ detail }) => {
editor?.insertAtCursor(detail)
editor?.focus()
}}
>
<SimpleEditor
bind:this={editor}
lang="javascript"
bind:code={flowModule.stop_after_if.expr}
class="few-lines-editor"
extraLib={`declare const result = ${JSON.stringify(earlyStopResult)};` +
(isLoop ? `\ndeclare const all_iters = ${JSON.stringify(result)};` : '')}
/>
</PropPickerWrapper>
</div>
{:else}
{#if !parentLoopId && !isLoop}
<Toggle
disabled
size="xs"
options={{
right: 'Label flow as "skipped" if stopped'
}}
/>
{/if}
<span class="mt-2 text-xs font-bold">Stop condition expression</span>
<textarea disabled rows="3" class="min-h-[80px]" />
{/if}
</div>
</Section>
{/if}
{#if isLoop || isBranchAll}
<Section
label={(parentLoopId ? 'Break parent loop module ' + parentLoopId : 'Stop flow early') +
(isBranchAll
? ' (evaluated after all branches have been run)'
: ' (evaluated after all iterations)')}
class="w-full"
>
<svelte:fragment slot="header">
<Tooltip documentationLink="https://www.windmill.dev/docs/flows/early_stop">
If defined, at the end of the step, the predicate expression will be evaluated to decide
if the flow should stop early or break if inside a for/while loop.
</Tooltip>
</svelte:fragment>
<Toggle
checked={isStopAfterAllIterationsEnabled}
on:change={() => {
if (isStopAfterAllIterationsEnabled && flowModule.stop_after_all_iters_if) {
flowModule.stop_after_all_iters_if = undefined
} else {
flowModule.stop_after_all_iters_if = {
expr: 'result == undefined',
skip_if_stopped: false
}
}
}}
options={{
right: (parentLoopId ? 'Break parent loop module' : 'Stop flow') + ' if condition met'
}}
/>
<div
class="w-full border p-2 flex flex-col {flowModule.stop_after_all_iters_if
? ''
: 'bg-surface-secondary'}"
>
{#if flowModule.stop_after_all_iters_if}
{#if !parentLoopId}
<Toggle
size="xs"
bind:checked={flowModule.stop_after_all_iters_if.skip_if_stopped}
options={{
right: 'Label flow as "skipped" if stopped'
}}
/>
{/if}
<span class="mt-2 text-xs font-bold">Stop condition expression</span>
<div class="border w-full">
<PropPickerWrapper
notSelectable
flow_input={stepPropPicker.pickableProperties.flow_input}
pickableProperties={undefined}
{result}
on:select={({ detail }) => {
editor?.insertAtCursor(detail)
editor?.focus()
}}
>
<SimpleEditor
bind:this={editor}
lang="javascript"
bind:code={flowModule.stop_after_all_iters_if.expr}
class="few-lines-editor"
extraLib={`declare const result = ${JSON.stringify(result)};`}
/>
</PropPickerWrapper>
</div>
{:else}
{#if !parentLoopId}
<Toggle
disabled
size="xs"
options={{
right: 'Label flow as "skipped" if stopped'
}}
/>
{/if}
<span class="mt-2 text-xs font-bold">Stop condition expression</span>
<textarea disabled rows="3" class="min-h-[80px]" />
{/if}
</div>
</Section>
{/if}
</div>
frontend/src/lib/components/flows/content/FlowModuleHeader.svelte
+1 -1
View File
@@ -71,7 +71,7 @@
<svelte:fragment slot="text">Cache</svelte:fragment>
</Popover>
{/if}
{#if module.stop_after_if}
{#if module.stop_after_if || module.stop_after_all_iters_if}
<Popover
placement="bottom"
class="center-center rounded p-2 bg-blue-100 text-blue-800 border border-blue-300 hover:bg-blue-200 dark:bg-frost-700 dark:text-frost-100 dark:border-frost-600"
frontend/src/lib/components/flows/flowExplorer.ts
+3
View File
@@ -85,6 +85,9 @@ function getModuleExprs(x: FlowModule): string[] {
if (x.stop_after_if?.expr) {
exprs.push(x.stop_after_if.expr)
}
if (x.stop_after_all_iters_if?.expr) {
exprs.push(x.stop_after_all_iters_if.expr)
}
exprs.push(...getExpr(x.sleep))
}
return exprs
frontend/src/lib/components/flows/map/MapItem.svelte
+1 -1
View File
@@ -45,7 +45,7 @@
$: itemProps = {
selected: $selectedId === mod.id,
retry: mod.retry?.constant != undefined || mod.retry?.exponential != undefined,
earlyStop: mod.stop_after_if != undefined,
earlyStop: mod.stop_after_if != undefined || mod.stop_after_all_iters_if != undefined,
suspend: Boolean(mod.suspend),
sleep: Boolean(mod.sleep),
cache: Boolean(mod.cache_ttl),
frontend/src/lib/components/flows/propPicker/PropPickerWrapper.svelte
+2
View File
@@ -28,6 +28,7 @@
export let pickableProperties: PickableProperties | undefined
export let result: any = undefined
export let extraResults: any = undefined
export let flow_input: any = undefined
export let error: boolean = false
export let displayContext = true
@@ -73,6 +74,7 @@
{#if result}
<PropPickerResult
{result}
{extraResults}
{flow_input}
on:select={({ detail }) => {
if (!notSelectable && !$propPickerConfig) {
frontend/src/lib/components/propertyPicker/PropPickerResult.svelte
+6 -1
View File
@@ -3,6 +3,7 @@
import ObjectViewer from './ObjectViewer.svelte'
export let result: any
export let extraResults: any = undefined
export let flow_input: any = undefined
const dispatch = createEventDispatcher()
@@ -11,7 +12,11 @@
<div class="w-full px-2">
<span class="font-bold text-sm">Result</span>
<div class="overflow-y-auto mb-2 w-full">
<ObjectViewer allowCopy={false} json={{ result }} on:select />
<ObjectViewer
allowCopy={false}
json={{ result, ...(extraResults ? extraResults : {}) }}
on:select
/>
</div>
{#if flow_input}
<span class="font-bold text-sm">Flow Input</span>
openflow.openapi.yaml
+9
View File
@@ -103,6 +103,15 @@ components:
type: string
required:
- expr
stop_after_all_iters_if:
type: object
properties:
skip_if_stopped:
type: boolean
expr:
type: string
required:
- expr
sleep:
$ref: "#/components/schemas/InputTransform"
cache_ttl: