mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-20 00:07:01 +00:00
feat: label for-loop iterations with a per-iteration expression (#12526)
* feat: label for-loop iterations with a per-iteration expression Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: evaluate iteration labels before the push transaction, under a timeout Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore: point the EE ref at the iteration_label feature usage registration Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: stop QuickJS at its deadline and bound labelling a push as a whole Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: report an interrupted expression as timed out so eval retries it Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * feat: put the iteration label behind a custom iteration labels toggle Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore: update ee-repo-ref to 3389335a8f58de7a06ae0402ec1c4fdad3bf6122 This commit updates the EE repository reference after PR #994 was merged in windmill-ee-private. Previous ee-repo-ref: dde2f1656939c8fc1fc8c75f3ddffcd637cce8a3 New ee-repo-ref: 3389335a8f58de7a06ae0402ec1c4fdad3bf6122 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
windmill-internal-app[bot]
parent
815db30c39
commit
91de7252b8
@@ -1 +1 @@
|
||||
1eb95bcdc7624b90074e8c3a4eaba011b3b5100d
|
||||
3389335a8f58de7a06ae0402ec1c4fdad3bf6122
|
||||
|
||||
@@ -188,6 +188,7 @@ export function main() {
|
||||
skip_failures: false,
|
||||
parallel: false,
|
||||
squash: None,
|
||||
iteration_label: None,
|
||||
parallelism: None,
|
||||
modules: vec![flow_module(
|
||||
"c",
|
||||
@@ -531,6 +532,7 @@ export function main() {
|
||||
skip_failures: false,
|
||||
parallel: false,
|
||||
squash: None,
|
||||
iteration_label: None,
|
||||
parallelism: None,
|
||||
modules: vec![
|
||||
// For each category, compute results using base_values
|
||||
@@ -980,6 +982,7 @@ export function main() {
|
||||
skip_failures: false,
|
||||
parallel: true,
|
||||
squash: None,
|
||||
iteration_label: None,
|
||||
parallelism: None,
|
||||
modules: vec![flow_module(
|
||||
"process",
|
||||
@@ -1897,6 +1900,7 @@ export function main() {
|
||||
skip_failures: false,
|
||||
parallel: false,
|
||||
squash: None,
|
||||
iteration_label: None,
|
||||
parallelism: None,
|
||||
modules: vec![
|
||||
// Step b: generate inner items based on category
|
||||
@@ -2539,6 +2543,7 @@ export function main() {
|
||||
parallel: true,
|
||||
parallelism: None,
|
||||
squash: None,
|
||||
iteration_label: None,
|
||||
},
|
||||
)],
|
||||
flow_env: Some(flow_env),
|
||||
@@ -3301,6 +3306,7 @@ export function main(i: number) {
|
||||
parallel: false,
|
||||
parallelism: None,
|
||||
squash: None,
|
||||
iteration_label: None,
|
||||
},
|
||||
);
|
||||
m.stop_after_all_iters_if = Some(windmill_common::flows::StopAfterIf {
|
||||
@@ -3402,6 +3408,7 @@ export function main(i: number) {
|
||||
parallel: false,
|
||||
parallelism: None,
|
||||
squash: None,
|
||||
iteration_label: None,
|
||||
},
|
||||
);
|
||||
m.stop_after_all_iters_if = Some(windmill_common::flows::StopAfterIf {
|
||||
|
||||
@@ -164,6 +164,69 @@ async fn test_iteration_parallel(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "deno_core")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_iteration_labels(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
|
||||
let label_loop = |id: &str, parallel: bool, label: &str| {
|
||||
json!({
|
||||
"id": id,
|
||||
"value": {
|
||||
"type": "forloopflow",
|
||||
"iterator": { "type": "javascript", "expr": "flow_input.items" },
|
||||
"skip_failures": false,
|
||||
"parallel": parallel,
|
||||
"iteration_label": { "type": "javascript", "expr": label },
|
||||
"modules": [
|
||||
{ "id": format!("{id}_a"), "value": { "type": "identity" } },
|
||||
{ "id": format!("{id}_b"), "value": { "type": "identity" } },
|
||||
],
|
||||
},
|
||||
})
|
||||
};
|
||||
let flow: FlowValue = serde_json::from_value(json!({
|
||||
"modules": [
|
||||
label_loop("seq", false, "flow_input.iter.value.name"),
|
||||
label_loop("par", true, "`${flow_input.iter.index}:${flow_input.iter.value.name}`"),
|
||||
label_loop("bad", false, "flow_input.iter.value.missing.name"),
|
||||
],
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
let job = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None })
|
||||
.arg(
|
||||
"items",
|
||||
json!([{ "name": "acme" }, { "name": null }, { "name": "globex" }]),
|
||||
)
|
||||
.run_until_complete(&db, false, server.addr.port())
|
||||
.await;
|
||||
assert!(job.success, "{:?}", job.json_result());
|
||||
|
||||
let labels = |id: &str| get_module(&job, id).and_then(|m| m.flow_jobs_labels());
|
||||
assert_eq!(
|
||||
labels("seq"),
|
||||
Some(vec![
|
||||
Some("acme".to_string()),
|
||||
None,
|
||||
Some("globex".to_string())
|
||||
])
|
||||
);
|
||||
assert_eq!(
|
||||
labels("par"),
|
||||
Some(vec![
|
||||
Some("0:acme".to_string()),
|
||||
Some("1:null".to_string()),
|
||||
Some("2:globex".to_string())
|
||||
])
|
||||
);
|
||||
// A label that fails to evaluate leaves the iteration unlabelled and the loop running.
|
||||
assert_eq!(labels("bad"), Some(vec![None, None, None]));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "deno_core")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_bun_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
@@ -221,6 +284,7 @@ async fn test_bun_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
skip_failures: false,
|
||||
parallel: false,
|
||||
squash: None,
|
||||
iteration_label: None,
|
||||
parallelism: None,
|
||||
modules: vec![FlowModule {
|
||||
id: "c".to_string(),
|
||||
@@ -577,6 +641,7 @@ async fn test_bun_flow_same_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
skip_failures: false,
|
||||
parallel: false,
|
||||
squash: None,
|
||||
iteration_label: None,
|
||||
parallelism: None,
|
||||
modules: vec![
|
||||
FlowModule {
|
||||
|
||||
@@ -2435,6 +2435,7 @@ mod tests {
|
||||
parallel: false,
|
||||
parallelism: None,
|
||||
squash: None,
|
||||
iteration_label: None,
|
||||
}),
|
||||
stop_after_if: Some(StopAfterIf {
|
||||
expr: "previous.isEmpty()".to_string(),
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -253,6 +253,32 @@ pub async fn eval_timeout_quickjs(
|
||||
authed_client: Option<&AuthedClient>,
|
||||
by_id: Option<&IdContext>,
|
||||
ctx: Option<Vec<(String, String)>>,
|
||||
) -> anyhow::Result<Box<RawValue>> {
|
||||
eval_timeout_quickjs_within(
|
||||
std::time::Duration::from_millis(EVAL_TIMEOUT_MS),
|
||||
expr,
|
||||
transform_context,
|
||||
flow_input,
|
||||
flow_env,
|
||||
authed_client,
|
||||
by_id,
|
||||
ctx,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`eval_timeout_quickjs`] with its own time limit. QuickJS is interrupted at the limit, so a
|
||||
/// script that never returns does not keep its blocking thread busy after the caller gave up.
|
||||
#[cfg(feature = "quickjs")]
|
||||
pub async fn eval_timeout_quickjs_within(
|
||||
timeout: std::time::Duration,
|
||||
expr: String,
|
||||
transform_context: HashMap<String, Arc<Box<RawValue>>>,
|
||||
flow_input: Option<mappable_rc::Marc<HashMap<String, Box<RawValue>>>>,
|
||||
flow_env: Option<&HashMap<String, Box<RawValue>>>,
|
||||
authed_client: Option<&AuthedClient>,
|
||||
by_id: Option<&IdContext>,
|
||||
ctx: Option<Vec<(String, String)>>,
|
||||
) -> anyhow::Result<Box<RawValue>> {
|
||||
let expr = expr.trim().to_string();
|
||||
|
||||
@@ -315,8 +341,9 @@ pub async fn eval_timeout_quickjs(
|
||||
// and producing spurious "DispatchGone" / "runtime dropped the dispatch
|
||||
// task" errors on subsequent requests from the main runtime.
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
let deadline = std::time::Instant::now() + timeout;
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_millis(EVAL_TIMEOUT_MS),
|
||||
timeout,
|
||||
tokio::task::spawn_blocking(move || {
|
||||
handle.block_on(async move {
|
||||
eval_quickjs_inner(
|
||||
@@ -329,6 +356,7 @@ pub async fn eval_timeout_quickjs(
|
||||
ctx,
|
||||
context_keys,
|
||||
memory_limit,
|
||||
Some(deadline),
|
||||
)
|
||||
.await
|
||||
})
|
||||
@@ -337,7 +365,8 @@ pub async fn eval_timeout_quickjs(
|
||||
.await
|
||||
.map_err(|_| {
|
||||
anyhow::anyhow!(
|
||||
"The expression evaluation `{expr}` took too long to execute (>{EVAL_TIMEOUT_MS}ms)"
|
||||
"The expression evaluation `{expr}` took too long to execute (>{}ms)",
|
||||
timeout.as_millis()
|
||||
)
|
||||
})??
|
||||
}
|
||||
@@ -445,9 +474,17 @@ async fn eval_quickjs_inner(
|
||||
extra_ctx: Option<Vec<(String, String)>>,
|
||||
context_keys: Vec<String>,
|
||||
memory_limit: usize,
|
||||
deadline: Option<std::time::Instant>,
|
||||
) -> anyhow::Result<Box<RawValue>> {
|
||||
let runtime = AsyncRuntime::new()?;
|
||||
runtime.set_memory_limit(memory_limit).await;
|
||||
if let Some(deadline) = deadline {
|
||||
runtime
|
||||
.set_interrupt_handler(Some(Box::new(move || {
|
||||
std::time::Instant::now() >= deadline
|
||||
})))
|
||||
.await;
|
||||
}
|
||||
let context = AsyncContext::full(&runtime).await?;
|
||||
|
||||
// Create shared state for async ops if we have a client
|
||||
@@ -469,7 +506,7 @@ async fn eval_quickjs_inner(
|
||||
format!("__loadResults({})", serde_json::to_string(&step_ids)?)
|
||||
};
|
||||
|
||||
async_with!(context => |ctx| {
|
||||
let result: anyhow::Result<Box<RawValue>> = async_with!(context => |ctx| {
|
||||
let globals = ctx.globals();
|
||||
|
||||
// Set up context variables
|
||||
@@ -584,15 +621,29 @@ async fn eval_quickjs_inner(
|
||||
// Evaluate the expression (returns a Promise that resolves to a JSON string)
|
||||
let promise: rquickjs::Promise = ctx.eval(code).catch(&ctx).map_err(|e| map_quickjs_error(e, memory_limit, Some("QUICKJS_MEMORY_LIMIT_MB")))?;
|
||||
|
||||
// Await the promise
|
||||
let result: Value = promise.into_future().await.catch(&ctx).map_err(|e| map_quickjs_error(e, memory_limit, Some("QUICKJS_MEMORY_LIMIT_MB")))?;
|
||||
// An interrupted job never settles its promise, so the deadline also bounds the wait.
|
||||
let settled = promise.into_future::<Value>();
|
||||
let settled = match deadline {
|
||||
Some(deadline) => tokio::time::timeout_at(deadline.into(), settled)
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("The expression evaluation `{expr}` took too long to execute"))?,
|
||||
None => settled.await,
|
||||
};
|
||||
let result: Value = settled.catch(&ctx).map_err(|e| map_quickjs_error(e, memory_limit, Some("QUICKJS_MEMORY_LIMIT_MB")))?;
|
||||
|
||||
let json_str = String::from_js(&ctx, result)
|
||||
.unwrap_or_else(|_| "null".to_string());
|
||||
|
||||
Ok(unsafe_raw(json_str))
|
||||
})
|
||||
.await
|
||||
.await;
|
||||
// Whatever the interrupt surfaced as, report it the way callers recognize (and retry) a timeout.
|
||||
match result {
|
||||
Err(_) if deadline.is_some_and(|d| std::time::Instant::now() >= d) => Err(anyhow::anyhow!(
|
||||
"The expression evaluation `{expr}` took too long to execute"
|
||||
)),
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "quickjs")]
|
||||
@@ -1053,6 +1104,20 @@ pub async fn eval_timeout_quickjs(
|
||||
anyhow::bail!("JavaScript expression evaluation requires the `quickjs` feature. Enable it with: cargo build --features quickjs")
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "quickjs"))]
|
||||
pub async fn eval_timeout_quickjs_within(
|
||||
_timeout: std::time::Duration,
|
||||
_expr: String,
|
||||
_transform_context: HashMap<String, Arc<Box<RawValue>>>,
|
||||
_flow_input: Option<mappable_rc::Marc<HashMap<String, Box<RawValue>>>>,
|
||||
_flow_env: Option<&HashMap<String, Box<RawValue>>>,
|
||||
_authed_client: Option<&windmill_common::client::AuthedClient>,
|
||||
_by_id: Option<&IdContext>,
|
||||
_ctx: Option<Vec<(String, String)>>,
|
||||
) -> anyhow::Result<Box<RawValue>> {
|
||||
anyhow::bail!("JavaScript expression evaluation requires the `quickjs` feature. Enable it with: cargo build --features quickjs")
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "quickjs"))]
|
||||
pub async fn eval_simple_js(
|
||||
_expr: String,
|
||||
@@ -2987,6 +3052,7 @@ mod tests {
|
||||
None,
|
||||
Vec::new(),
|
||||
4 * 1024 * 1024,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("expected OOM to fail")
|
||||
@@ -3061,6 +3127,28 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_eval_stops_at_its_deadline() {
|
||||
// Without the interrupt the script never returns, and a caller's timeout only stops
|
||||
// waiting for it.
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_millis(200);
|
||||
let err = eval_quickjs_inner(
|
||||
"(() => { while (true) {} })()",
|
||||
HashMap::new(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Vec::new(),
|
||||
4 * 1024 * 1024,
|
||||
Some(deadline),
|
||||
)
|
||||
.await;
|
||||
let err = err.expect_err("an endless script must be interrupted").to_string();
|
||||
assert!(err.contains("took too long"), "unexpected error: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_eval_issue_payload_succeeds_with_raised_cap() {
|
||||
// The exact #8073 payload exceeds the conservative default but succeeds
|
||||
@@ -3077,6 +3165,7 @@ mod tests {
|
||||
None,
|
||||
Vec::new(),
|
||||
256 * 1024 * 1024,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("issue payload should evaluate once the cap is raised");
|
||||
|
||||
@@ -8835,6 +8835,7 @@ fn create_restarted_module(
|
||||
flow_jobs: Some(new_flow_jobs),
|
||||
flow_jobs_success: new_flow_jobs_success,
|
||||
flow_jobs_duration: new_flow_jobs_timeline,
|
||||
flow_jobs_labels: None,
|
||||
branch_chosen: None,
|
||||
branchall: Some(BranchAllStatus {
|
||||
branch: branch_or_iteration_n - 1,
|
||||
@@ -8873,6 +8874,10 @@ fn create_restarted_module(
|
||||
if let Some(new_flow_jobs_timeline) = new_flow_jobs_timeline.as_mut() {
|
||||
new_flow_jobs_timeline.truncate(branch_or_iteration_n);
|
||||
}
|
||||
let mut new_flow_jobs_labels = module.flow_jobs_labels();
|
||||
if let Some(new_flow_jobs_labels) = new_flow_jobs_labels.as_mut() {
|
||||
new_flow_jobs_labels.truncate(branch_or_iteration_n);
|
||||
}
|
||||
Ok(FlowStatusModule::InProgress {
|
||||
id: module.id(),
|
||||
job: new_flow_jobs[new_flow_jobs.len() - 1],
|
||||
@@ -8884,6 +8889,7 @@ fn create_restarted_module(
|
||||
flow_jobs: Some(new_flow_jobs),
|
||||
flow_jobs_success: new_flow_jobs_success,
|
||||
flow_jobs_duration: new_flow_jobs_timeline,
|
||||
flow_jobs_labels: new_flow_jobs_labels,
|
||||
branch_chosen: None,
|
||||
branchall: None,
|
||||
parallel,
|
||||
@@ -8948,6 +8954,7 @@ fn reuse_completed_zombie_module(module: FlowStatusModule) -> FlowStatusModule {
|
||||
flow_jobs,
|
||||
flow_jobs_success,
|
||||
flow_jobs_duration,
|
||||
flow_jobs_labels,
|
||||
branch_chosen,
|
||||
agent_actions,
|
||||
agent_actions_success,
|
||||
@@ -8961,6 +8968,7 @@ fn reuse_completed_zombie_module(module: FlowStatusModule) -> FlowStatusModule {
|
||||
.map(|v| v.into_iter().map(|_| Some(true)).collect()),
|
||||
flow_jobs,
|
||||
flow_jobs_duration,
|
||||
flow_jobs_labels,
|
||||
branch_chosen,
|
||||
approvers: vec![],
|
||||
failed_retries: vec![],
|
||||
|
||||
@@ -190,6 +190,7 @@ struct UntaggedFlowStatusModule {
|
||||
flow_jobs: Option<Vec<Uuid>>,
|
||||
flow_jobs_success: Option<Vec<Option<bool>>>,
|
||||
flow_jobs_duration: Option<FlowJobsDuration>,
|
||||
flow_jobs_labels: Option<Vec<Option<String>>>,
|
||||
branch_chosen: Option<BranchChosen>,
|
||||
branchall: Option<BranchAllStatus>,
|
||||
parallel: Option<bool>,
|
||||
@@ -274,6 +275,8 @@ pub enum FlowStatusModule {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
flow_jobs_duration: Option<FlowJobsDuration>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
flow_jobs_labels: Option<Vec<Option<String>>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
branch_chosen: Option<BranchChosen>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
branchall: Option<BranchAllStatus>,
|
||||
@@ -296,6 +299,8 @@ pub enum FlowStatusModule {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
flow_jobs_duration: Option<FlowJobsDuration>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
flow_jobs_labels: Option<Vec<Option<String>>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
branch_chosen: Option<BranchChosen>,
|
||||
#[serde(default)]
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
@@ -318,6 +323,8 @@ pub enum FlowStatusModule {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
flow_jobs_duration: Option<FlowJobsDuration>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
flow_jobs_labels: Option<Vec<Option<String>>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
branch_chosen: Option<BranchChosen>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
failed_retries: Vec<Uuid>,
|
||||
@@ -372,6 +379,7 @@ impl<'de> Deserialize<'de> for FlowStatusModule {
|
||||
flow_jobs: untagged.flow_jobs,
|
||||
flow_jobs_success: untagged.flow_jobs_success,
|
||||
flow_jobs_duration: untagged.flow_jobs_duration,
|
||||
flow_jobs_labels: untagged.flow_jobs_labels,
|
||||
branch_chosen: untagged.branch_chosen,
|
||||
branchall: untagged.branchall,
|
||||
parallel: untagged.parallel.unwrap_or(false),
|
||||
@@ -390,6 +398,7 @@ impl<'de> Deserialize<'de> for FlowStatusModule {
|
||||
flow_jobs: untagged.flow_jobs,
|
||||
flow_jobs_success: untagged.flow_jobs_success,
|
||||
flow_jobs_duration: untagged.flow_jobs_duration,
|
||||
flow_jobs_labels: untagged.flow_jobs_labels,
|
||||
branch_chosen: untagged.branch_chosen,
|
||||
approvers: untagged.approvers.unwrap_or_default(),
|
||||
failed_retries: untagged.failed_retries.unwrap_or_default(),
|
||||
@@ -407,6 +416,7 @@ impl<'de> Deserialize<'de> for FlowStatusModule {
|
||||
flow_jobs: untagged.flow_jobs,
|
||||
flow_jobs_success: untagged.flow_jobs_success,
|
||||
flow_jobs_duration: untagged.flow_jobs_duration,
|
||||
flow_jobs_labels: untagged.flow_jobs_labels,
|
||||
branch_chosen: untagged.branch_chosen,
|
||||
failed_retries: untagged.failed_retries.unwrap_or_default(),
|
||||
agent_actions: untagged.agent_actions,
|
||||
@@ -487,6 +497,15 @@ impl FlowStatusModule {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn flow_jobs_labels(&self) -> Option<Vec<Option<String>>> {
|
||||
match self {
|
||||
FlowStatusModule::InProgress { flow_jobs_labels, .. } => flow_jobs_labels.clone(),
|
||||
FlowStatusModule::Success { flow_jobs_labels, .. } => flow_jobs_labels.clone(),
|
||||
FlowStatusModule::Failure { flow_jobs_labels, .. } => flow_jobs_labels.clone(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn flow_jobs_duration(&self) -> Option<FlowJobsDuration> {
|
||||
match self {
|
||||
FlowStatusModule::InProgress { flow_jobs_duration, .. } => flow_jobs_duration.clone(),
|
||||
|
||||
@@ -1113,6 +1113,10 @@ pub enum FlowModuleValue {
|
||||
parallelism: Option<InputTransform>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
squash: Option<bool>,
|
||||
/// Evaluated once per iteration, with `flow_input.iter` set, into the text the run
|
||||
/// viewers show for that iteration. Never fails the iteration.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
iteration_label: Option<InputTransform>,
|
||||
},
|
||||
WhileloopFlow {
|
||||
modules: Vec<FlowModule>,
|
||||
@@ -1239,6 +1243,7 @@ struct UntaggedFlowModuleValue {
|
||||
tool_inputs: Option<HashMap<String, HashMap<String, InputTransform>>>,
|
||||
pass_flow_input_directly: Option<bool>,
|
||||
squash: Option<bool>,
|
||||
iteration_label: Option<InputTransform>,
|
||||
max_iterations: Option<u32>,
|
||||
max_duration_secs: Option<u32>,
|
||||
#[serde(flatten)]
|
||||
@@ -1282,6 +1287,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue {
|
||||
parallel: untagged.parallel.unwrap_or(false),
|
||||
parallelism: untagged.parallelism,
|
||||
squash: untagged.squash,
|
||||
iteration_label: untagged.iteration_label,
|
||||
}),
|
||||
"whileloopflow" => Ok(FlowModuleValue::WhileloopFlow {
|
||||
modules: untagged
|
||||
|
||||
@@ -107,6 +107,43 @@ pub async fn eval_timeout(
|
||||
}
|
||||
}
|
||||
|
||||
/// [`eval_timeout`] for an expression that only reads `flow_input`, `flow_env` and `results`,
|
||||
/// bounded by `timeout` in total: no retry, and QuickJS is interrupted at the limit.
|
||||
pub async fn eval_within(
|
||||
timeout: std::time::Duration,
|
||||
expr: &str,
|
||||
flow_input: mappable_rc::Marc<HashMap<String, Box<RawValue>>>,
|
||||
flow_env: Option<&HashMap<String, Box<RawValue>>>,
|
||||
authed_client: &AuthedClient,
|
||||
by_id: &IdContext,
|
||||
) -> anyhow::Result<Box<RawValue>> {
|
||||
let expr = expr.trim();
|
||||
if let Some(value) =
|
||||
windmill_jseval::try_exact_property_access(expr, Some(&flow_input), flow_env)
|
||||
{
|
||||
return Ok(value);
|
||||
}
|
||||
let eval = async {
|
||||
if let Some(result) = windmill_jseval::handle_full_regex(expr, authed_client, by_id).await {
|
||||
return result;
|
||||
}
|
||||
windmill_jseval::eval_timeout_quickjs_within(
|
||||
timeout,
|
||||
expr.to_string(),
|
||||
HashMap::new(),
|
||||
Some(flow_input),
|
||||
flow_env,
|
||||
Some(authed_client),
|
||||
Some(by_id),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
};
|
||||
tokio::time::timeout(timeout, eval)
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("`{expr}` took longer than {}ms", timeout.as_millis()))?
|
||||
}
|
||||
|
||||
/// Adds `step_id`, `job_id` and `flow_path` to a native script's thrown error, as the
|
||||
/// subprocess wrappers do from the same env vars.
|
||||
pub fn with_step_context(
|
||||
|
||||
@@ -12,7 +12,7 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::common::{cached_result_path, get_root_job_id, save_in_cache, transform_json};
|
||||
use crate::js_eval::{eval_timeout, IdContext};
|
||||
use crate::js_eval::{eval_timeout, eval_within, IdContext};
|
||||
use crate::worker_utils::get_tag_and_concurrency;
|
||||
use crate::{
|
||||
JobCompletedSender, PreviousResult, SameWorkerSender, SendResultPayload, StepFailureKind,
|
||||
@@ -1123,6 +1123,7 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
job: job_id_for_status.clone(),
|
||||
flow_jobs: Some(jobs.clone()),
|
||||
flow_jobs_success: flow_jobs_success.clone(),
|
||||
flow_jobs_labels: module_status.flow_jobs_labels(),
|
||||
flow_jobs_duration: flow_jobs_duration.clone(),
|
||||
branch_chosen: None,
|
||||
approvers: vec![],
|
||||
@@ -1138,6 +1139,7 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
job: job_id_for_status.clone(),
|
||||
flow_jobs: Some(jobs.clone()),
|
||||
flow_jobs_success: flow_jobs_success.clone(),
|
||||
flow_jobs_labels: module_status.flow_jobs_labels(),
|
||||
flow_jobs_duration: flow_jobs_duration.clone(),
|
||||
branch_chosen: None,
|
||||
failed_retries: vec![],
|
||||
@@ -1376,6 +1378,7 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
job: job_id_for_status.clone(),
|
||||
flow_jobs,
|
||||
flow_jobs_success,
|
||||
flow_jobs_labels: module_status.flow_jobs_labels(),
|
||||
flow_jobs_duration,
|
||||
branch_chosen,
|
||||
approvers: vec![],
|
||||
@@ -1424,6 +1427,7 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
job: job_id_for_status.clone(),
|
||||
flow_jobs,
|
||||
flow_jobs_success,
|
||||
flow_jobs_labels: module_status.flow_jobs_labels(),
|
||||
flow_jobs_duration,
|
||||
branch_chosen,
|
||||
failed_retries: old_status.retry.failed_jobs.clone(),
|
||||
@@ -3918,6 +3922,7 @@ async fn push_next_flow_job(
|
||||
job: Uuid::nil(),
|
||||
flow_jobs: None,
|
||||
flow_jobs_success: None,
|
||||
flow_jobs_labels: None,
|
||||
flow_jobs_duration: None,
|
||||
branch_chosen: None,
|
||||
failed_retries: vec![],
|
||||
@@ -4780,6 +4785,7 @@ async fn push_next_flow_job(
|
||||
} else {
|
||||
Some(vec![])
|
||||
},
|
||||
flow_jobs_labels: None,
|
||||
flow_jobs_duration: if branch_chosen.is_some() {
|
||||
None
|
||||
} else {
|
||||
@@ -4901,6 +4907,43 @@ async fn push_next_flow_job(
|
||||
.warn_after_seconds(3)
|
||||
.await?;
|
||||
|
||||
// Evaluated before the push transaction opens, so a slow label never holds it.
|
||||
let (has_iteration_label, mut iteration_labels) = match (&next_status, args.as_ref()) {
|
||||
(NextStatus::NextLoopIteration { next, .. }, Ok(flow_args)) => {
|
||||
match &next.iteration_label {
|
||||
Some(label) => {
|
||||
let iters = vec![to_raw_value(&next.new_args)];
|
||||
let ctx = get_transform_context(&flow_job, "", &status);
|
||||
let labels =
|
||||
eval_iteration_labels(label, iters, flow_args, flow_env, client, &ctx)
|
||||
.await;
|
||||
(true, labels)
|
||||
}
|
||||
None => (false, vec![]),
|
||||
}
|
||||
}
|
||||
(
|
||||
NextStatus::AllFlowJobs {
|
||||
iteration_label: Some(label),
|
||||
iterator: Some(FlowIterator { itered: Some(itered), .. }),
|
||||
..
|
||||
},
|
||||
Ok(flow_args),
|
||||
) => {
|
||||
let iters = itered
|
||||
.iter()
|
||||
.take(len)
|
||||
.enumerate()
|
||||
.map(|(i, v)| to_raw_value(&json!({ "index": i as i32, "value": v })))
|
||||
.collect();
|
||||
let ctx = get_transform_context(&flow_job, "", &status);
|
||||
let labels =
|
||||
eval_iteration_labels(label, iters, flow_args, flow_env, client, &ctx).await;
|
||||
(true, labels)
|
||||
}
|
||||
_ => (false, vec![]),
|
||||
};
|
||||
|
||||
let mut tx = db.begin().warn_after_seconds(3).await?;
|
||||
let nargs = args.as_ref();
|
||||
for (i, mut payload_tag) in job_payloads.into_iter().enumerate() {
|
||||
@@ -4963,10 +5006,10 @@ async fn push_next_flow_job(
|
||||
insert_iter_arg(&mut args, "iter".to_string(), to_raw_value(new_args));
|
||||
|
||||
args.insert("iter".to_string(), to_raw_value(new_args));
|
||||
let args = Marc::new(args);
|
||||
if let Some(input_transforms) = simple_input_transforms {
|
||||
//previous id is none because we do not want to use previous id if we are in a for loop
|
||||
let ctx = get_transform_context(&flow_job, "", &status);
|
||||
let args = Marc::new(args);
|
||||
step_flow_input = args.clone();
|
||||
let ti = transform_input(
|
||||
args,
|
||||
@@ -4999,7 +5042,7 @@ async fn push_next_flow_job(
|
||||
Err(&me)
|
||||
}
|
||||
} else {
|
||||
marc = Marc::new(args);
|
||||
marc = args;
|
||||
Ok(&marc)
|
||||
}
|
||||
}
|
||||
@@ -5007,6 +5050,7 @@ async fn push_next_flow_job(
|
||||
branchall: None,
|
||||
iterator: Some(FlowIterator { itered, .. }),
|
||||
simple_input_transforms,
|
||||
..
|
||||
} => {
|
||||
if let Ok(args) = args.as_ref() {
|
||||
let Some(itered) = itered else {
|
||||
@@ -5024,10 +5068,11 @@ async fn push_next_flow_job(
|
||||
"iter".to_string(),
|
||||
to_raw_value(&json!({ "index": i as i32, "value": itered[i]})),
|
||||
);
|
||||
let hm = Marc::new(hm);
|
||||
if let Some(input_transforms) = simple_input_transforms {
|
||||
let ctx = get_transform_context(&flow_job, &previous_id, &status);
|
||||
let ti = transform_input(
|
||||
Marc::new(hm),
|
||||
hm,
|
||||
flow_env,
|
||||
env_refs,
|
||||
false,
|
||||
@@ -5056,7 +5101,7 @@ async fn push_next_flow_job(
|
||||
Err(&me)
|
||||
}
|
||||
} else {
|
||||
marc = Marc::new(hm);
|
||||
marc = hm;
|
||||
Ok(&marc)
|
||||
}
|
||||
} else {
|
||||
@@ -5501,12 +5546,22 @@ async fn push_next_flow_job(
|
||||
while_loop,
|
||||
mut flow_jobs_success,
|
||||
mut flow_jobs_duration,
|
||||
flow_jobs_labels,
|
||||
..
|
||||
},
|
||||
..
|
||||
} => {
|
||||
let uuid = one_uuid?;
|
||||
|
||||
let flow_jobs_labels = if has_iteration_label {
|
||||
let mut labels = flow_jobs_labels.unwrap_or_default();
|
||||
labels.resize(flow_jobs.len(), None);
|
||||
labels.push(iteration_labels.pop().flatten());
|
||||
Some(labels)
|
||||
} else {
|
||||
flow_jobs_labels
|
||||
};
|
||||
|
||||
flow_jobs.push(uuid);
|
||||
|
||||
if let Some(flow_jobs_success) = &mut flow_jobs_success {
|
||||
@@ -5530,6 +5585,7 @@ async fn push_next_flow_job(
|
||||
flow_jobs: Some(flow_jobs),
|
||||
flow_jobs_success,
|
||||
flow_jobs_duration,
|
||||
flow_jobs_labels,
|
||||
branch_chosen: None,
|
||||
branchall: None,
|
||||
id: status_module.id(),
|
||||
@@ -5554,6 +5610,7 @@ async fn push_next_flow_job(
|
||||
flow_jobs_success: Some(vec![None; uuids.len()]),
|
||||
flow_jobs: Some(uuids.clone()),
|
||||
flow_jobs_duration: Some(FlowJobsDuration::new(uuids.len())),
|
||||
flow_jobs_labels: has_iteration_label.then_some(iteration_labels),
|
||||
branch_chosen: None,
|
||||
branchall,
|
||||
id: status_module.id(),
|
||||
@@ -5584,6 +5641,7 @@ async fn push_next_flow_job(
|
||||
iterator: None,
|
||||
flow_jobs: Some(flow_jobs),
|
||||
flow_jobs_success,
|
||||
flow_jobs_labels: None,
|
||||
flow_jobs_duration,
|
||||
branch_chosen: None,
|
||||
branchall: Some(status),
|
||||
@@ -5601,6 +5659,7 @@ async fn push_next_flow_job(
|
||||
iterator: None,
|
||||
flow_jobs: None,
|
||||
flow_jobs_success: None,
|
||||
flow_jobs_labels: None,
|
||||
flow_jobs_duration: None,
|
||||
branch_chosen: Some(branch),
|
||||
branchall: None,
|
||||
@@ -5814,6 +5873,8 @@ struct ForloopNextIteration {
|
||||
flow_jobs: Vec<Uuid>,
|
||||
flow_jobs_success: Option<Vec<Option<bool>>>,
|
||||
flow_jobs_duration: Option<FlowJobsDuration>,
|
||||
flow_jobs_labels: Option<Vec<Option<String>>>,
|
||||
iteration_label: Option<InputTransform>,
|
||||
new_args: Iter,
|
||||
while_loop: bool,
|
||||
}
|
||||
@@ -5846,6 +5907,7 @@ enum NextStatus {
|
||||
branchall: Option<BranchAllStatus>,
|
||||
iterator: Option<FlowIterator>,
|
||||
simple_input_transforms: Option<HashMap<String, InputTransform>>,
|
||||
iteration_label: Option<InputTransform>,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -6173,6 +6235,8 @@ async fn compute_next_flow_transform(
|
||||
flow_jobs: flow_jobs,
|
||||
flow_jobs_success: flow_jobs_success,
|
||||
flow_jobs_duration: flow_jobs_duration,
|
||||
flow_jobs_labels: None,
|
||||
iteration_label: None,
|
||||
new_args: Iter {
|
||||
index: next_loop_idx as i32,
|
||||
value: windmill_common::worker::to_raw_value(&next_loop_idx),
|
||||
@@ -6192,7 +6256,13 @@ async fn compute_next_flow_transform(
|
||||
}
|
||||
/* forloop modules are expected set `iter: { value: Value, index: usize }` as job arguments */
|
||||
FlowModuleValue::ForloopFlow {
|
||||
modules, modules_node, iterator, parallel, squash, ..
|
||||
modules,
|
||||
modules_node,
|
||||
iterator,
|
||||
parallel,
|
||||
squash,
|
||||
iteration_label,
|
||||
..
|
||||
} => {
|
||||
// if it's a simple single step flow, we will collapse it as an optimization and need to pass flow_input as an arg
|
||||
let is_simple = !matches!(flow_job.kind, JobKind::FlowPreview)
|
||||
@@ -6232,7 +6302,8 @@ async fn compute_next_flow_transform(
|
||||
ForLoopStatus::EmptyIterator => {
|
||||
Ok(NextFlowTransform::EmptyInnerFlows { branch_chosen: None })
|
||||
}
|
||||
ForLoopStatus::NextIteration(ns) => {
|
||||
ForLoopStatus::NextIteration(mut ns) => {
|
||||
ns.iteration_label = iteration_label;
|
||||
let start_runners = ns.index == 0 && squash.unwrap_or(false);
|
||||
next_loop_iteration(
|
||||
flow,
|
||||
@@ -6298,6 +6369,7 @@ async fn compute_next_flow_transform(
|
||||
itered_len: Some(itered_len),
|
||||
itered: Some(itered),
|
||||
}),
|
||||
iteration_label,
|
||||
// we removed the is_simple_case for simple_input_transforms
|
||||
// if is_simple {
|
||||
// match value {
|
||||
@@ -6476,6 +6548,7 @@ async fn compute_next_flow_transform(
|
||||
branchall: Some(BranchAllStatus { branch: 0, len }),
|
||||
iterator: None,
|
||||
simple_input_transforms: None,
|
||||
iteration_label: None,
|
||||
},
|
||||
));
|
||||
} else {
|
||||
@@ -6626,6 +6699,95 @@ async fn next_loop_iteration(
|
||||
))
|
||||
}
|
||||
|
||||
const ITERATION_LABEL_MAX_CHARS: usize = 100;
|
||||
const ITERATION_LABEL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2);
|
||||
const ITERATION_LABELS_BUDGET: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
/// One label per entry of `iters` (each an `iter` value). Labels are display-only: one that
|
||||
/// throws or yields nothing is `None`. Labelling one push is bounded as a whole, and the first
|
||||
/// label that runs out of time leaves the rest `None`, so a slow or hanging expression delays a
|
||||
/// parallel loop by at most one budget rather than once per item.
|
||||
async fn eval_iteration_labels(
|
||||
label: &InputTransform,
|
||||
iters: Vec<Box<RawValue>>,
|
||||
flow_args: &HashMap<String, Box<RawValue>>,
|
||||
flow_env: Option<&HashMap<String, Box<RawValue>>>,
|
||||
client: &AuthedClient,
|
||||
by_id: &IdContext,
|
||||
) -> Vec<Option<String>> {
|
||||
let n = iters.len();
|
||||
let deadline = std::time::Instant::now() + ITERATION_LABELS_BUDGET;
|
||||
let mut labels = Vec::with_capacity(n);
|
||||
for iter in iters {
|
||||
let timeout = ITERATION_LABEL_TIMEOUT
|
||||
.min(deadline.saturating_duration_since(std::time::Instant::now()));
|
||||
if timeout.is_zero() {
|
||||
tracing::warn!(
|
||||
"iteration labels ran out of time, leaving the remaining iterations unlabelled"
|
||||
);
|
||||
break;
|
||||
}
|
||||
let mut flow_input = flow_args.clone();
|
||||
insert_iter_arg(&mut flow_input, "iter".to_string(), iter);
|
||||
let started = std::time::Instant::now();
|
||||
labels.push(
|
||||
eval_iteration_label(
|
||||
label,
|
||||
timeout,
|
||||
Marc::new(flow_input),
|
||||
flow_env,
|
||||
client,
|
||||
by_id,
|
||||
)
|
||||
.await,
|
||||
);
|
||||
if started.elapsed() >= timeout {
|
||||
tracing::warn!(
|
||||
"iteration label timed out, leaving the remaining iterations unlabelled"
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
labels.resize(n, None);
|
||||
labels
|
||||
}
|
||||
|
||||
async fn eval_iteration_label(
|
||||
label: &InputTransform,
|
||||
timeout: std::time::Duration,
|
||||
flow_input: Marc<HashMap<String, Box<RawValue>>>,
|
||||
flow_env: Option<&HashMap<String, Box<RawValue>>>,
|
||||
client: &AuthedClient,
|
||||
by_id: &IdContext,
|
||||
) -> Option<String> {
|
||||
let raw = match label {
|
||||
InputTransform::Static { value } => value.clone(),
|
||||
InputTransform::Javascript { expr } if !expr.trim().is_empty() => {
|
||||
match eval_within(timeout, expr, flow_input, flow_env, client, by_id).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!("could not evaluate iteration label `{expr}`: {e:#}");
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
let text = match serde_json::from_str::<serde_json::Value>(raw.get()).ok()? {
|
||||
serde_json::Value::Null => return None,
|
||||
serde_json::Value::String(s) => s,
|
||||
v => v.to_string(),
|
||||
};
|
||||
// jsonb rejects NUL, and the label lands in the flow status.
|
||||
let text: String = text
|
||||
.trim()
|
||||
.chars()
|
||||
.filter(|c| *c != '\0')
|
||||
.take(ITERATION_LABEL_MAX_CHARS)
|
||||
.collect();
|
||||
(!text.is_empty()).then_some(text)
|
||||
}
|
||||
|
||||
pub(super) fn is_simple_modules(
|
||||
modules: &Vec<FlowModule>,
|
||||
failure_module: Option<&Box<FlowModule>>,
|
||||
@@ -6720,6 +6882,8 @@ async fn next_forloop_status(
|
||||
flow_jobs: vec![],
|
||||
flow_jobs_success: Some(vec![]),
|
||||
flow_jobs_duration: Some(FlowJobsDuration::new(0)),
|
||||
flow_jobs_labels: None,
|
||||
iteration_label: None,
|
||||
new_args: iter,
|
||||
while_loop: false,
|
||||
})
|
||||
@@ -6733,6 +6897,7 @@ async fn next_forloop_status(
|
||||
flow_jobs: Some(flow_jobs),
|
||||
flow_jobs_success,
|
||||
flow_jobs_duration,
|
||||
flow_jobs_labels,
|
||||
..
|
||||
} if !*parallel => {
|
||||
// Read itered from separate table or fallback to JSONB
|
||||
@@ -6794,6 +6959,8 @@ async fn next_forloop_status(
|
||||
flow_jobs: flow_jobs.clone(),
|
||||
flow_jobs_success: flow_jobs_success.clone(),
|
||||
flow_jobs_duration: flow_jobs_duration.clone(),
|
||||
flow_jobs_labels: flow_jobs_labels.clone(),
|
||||
iteration_label: None,
|
||||
new_args: Iter { index: index as i32, value: next.to_owned() },
|
||||
while_loop: false,
|
||||
})
|
||||
|
||||
@@ -1696,6 +1696,7 @@ async fn lock_modules(
|
||||
parallel,
|
||||
parallelism,
|
||||
squash,
|
||||
iteration_label,
|
||||
} => {
|
||||
let nmodules;
|
||||
(nmodules, nmodified_ids, nerrors) = Box::pin(lock_modules(
|
||||
@@ -1728,6 +1729,7 @@ async fn lock_modules(
|
||||
parallel,
|
||||
parallelism,
|
||||
squash,
|
||||
iteration_label,
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
Generated
+2
-1
File diff suppressed because one or more lines are too long
@@ -739,6 +739,7 @@
|
||||
id={module.id}
|
||||
{onSelectedIteration}
|
||||
flowJobsSuccess={localModuleStates[module.id]?.flow_jobs_success}
|
||||
flowJobsLabels={localModuleStates[module.id]?.flow_jobs_labels}
|
||||
flowJobs={localModuleStates[module.id]?.flow_jobs}
|
||||
selected={localModuleStates[module.id]?.selectedForloopIndex ?? 0}
|
||||
selectedManually={localModuleStates[module.id]
|
||||
@@ -748,6 +749,16 @@
|
||||
</button>
|
||||
{#if module.value.type === 'forloopflow'}
|
||||
{`/${localModuleStates[module.id]?.iteration_total ?? 0}`}
|
||||
{@const selectedLabel =
|
||||
localModuleStates[module.id]?.flow_jobs_labels?.[
|
||||
localModuleStates[module.id]?.selectedForloopIndex ?? 0
|
||||
]}
|
||||
{#if selectedLabel}
|
||||
<span
|
||||
class="ml-2 font-sans font-normal text-secondary truncate max-w-60"
|
||||
title={selectedLabel}>{selectedLabel}</span
|
||||
>
|
||||
{/if}
|
||||
{/if}
|
||||
</span>
|
||||
{/if}
|
||||
|
||||
@@ -100,6 +100,7 @@
|
||||
moduleId: string
|
||||
flowJobs: string[]
|
||||
flowJobsSuccess: (boolean | undefined)[]
|
||||
flowJobsLabels?: (string | null)[]
|
||||
flowJobsDuration:
|
||||
| {
|
||||
started_at?: (string | undefined)[]
|
||||
@@ -698,6 +699,7 @@
|
||||
if (mod.flow_jobs_success || mod.flow_jobs_duration || mod.flow_jobs) {
|
||||
setModuleState(mod.id ?? '', {
|
||||
flow_jobs_success: mod.flow_jobs_success,
|
||||
flow_jobs_labels: mod.flow_jobs_labels,
|
||||
flow_jobs_duration: mod.flow_jobs_duration,
|
||||
flow_jobs: mod.flow_jobs,
|
||||
iteration_total: mod.iterator?.itered_len ?? mod.flow_jobs?.length
|
||||
@@ -1040,6 +1042,7 @@
|
||||
started_at: started_at,
|
||||
flow_jobs: mod.flow_jobs,
|
||||
flow_jobs_success: mod.flow_jobs_success,
|
||||
flow_jobs_labels: mod.flow_jobs_labels,
|
||||
flow_jobs_duration: mod.flow_jobs_duration,
|
||||
iteration_total: mod.iterator?.itered_len,
|
||||
retries: mod?.failed_retries?.length,
|
||||
@@ -1170,6 +1173,7 @@
|
||||
started_at,
|
||||
flow_jobs: flowJobIds?.flowJobs,
|
||||
flow_jobs_success: flowJobIds?.flowJobsSuccess,
|
||||
flow_jobs_labels: flowJobIds?.flowJobsLabels,
|
||||
flow_jobs_duration: flowJobIds?.flowJobsDuration,
|
||||
iteration_total: flowJobIds?.length,
|
||||
duration_ms: undefined
|
||||
@@ -1640,7 +1644,12 @@
|
||||
}}
|
||||
>
|
||||
<span class="truncate font-mono">
|
||||
#{j + 1}: {loopJobId}
|
||||
#{j + 1}:
|
||||
{#if flowJobIds?.flowJobsLabels?.[j]}
|
||||
{flowJobIds.flowJobsLabels[j]} <span class="text-hint">{loopJobId}</span>
|
||||
{:else}
|
||||
{loopJobId}
|
||||
{/if}
|
||||
</span>
|
||||
</Button>
|
||||
{/if}
|
||||
@@ -1846,6 +1855,7 @@
|
||||
moduleId: mod.id ?? '',
|
||||
flowJobs: mod.flow_jobs,
|
||||
flowJobsSuccess: mod.flow_jobs_success ?? [],
|
||||
flowJobsLabels: mod.flow_jobs_labels,
|
||||
flowJobsDuration: mod.flow_jobs_duration,
|
||||
length: mod.iterator?.itered_len ?? mod.flow_jobs.length,
|
||||
branchall: job?.raw_flow?.modules?.[i]?.value?.type == 'branchall'
|
||||
|
||||
@@ -198,6 +198,7 @@
|
||||
id={k}
|
||||
{onSelectedIteration}
|
||||
flowJobsSuccess={localModuleStates[k]?.flow_jobs_success}
|
||||
flowJobsLabels={localModuleStates[k]?.flow_jobs_labels}
|
||||
flowJobs={localModuleStates[k]?.flow_jobs}
|
||||
selected={localModuleStates[k]?.selectedForloopIndex ?? 0}
|
||||
selectedManually={localModuleStates[k]?.selectedForLoopSetManually ?? false}
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1,5 +1,5 @@
|
||||
<script lang="ts">
|
||||
import { getContext, tick } from 'svelte'
|
||||
import { getContext, tick, untrack } from 'svelte'
|
||||
import FlowCard from '../common/FlowCard.svelte'
|
||||
import type { FlowEditorContext } from '../types'
|
||||
import Toggle from '$lib/components/Toggle.svelte'
|
||||
@@ -10,12 +10,13 @@
|
||||
import { getStepPropPicker } from '../previousResults'
|
||||
|
||||
import { Play } from 'lucide-svelte'
|
||||
import type { FlowModule, ForloopFlow, Job } from '$lib/gen'
|
||||
import type { FlowModule, ForloopFlow, InputTransform, Job } from '$lib/gen'
|
||||
import FlowLoopIterationPreview from '$lib/components/FlowLoopIterationPreview.svelte'
|
||||
import IteratorGen from '$lib/components/copilot/IteratorGen.svelte'
|
||||
import FlowRunSettings from './FlowRunSettings.svelte'
|
||||
import StepSettingsBadges from './StepSettingsBadges.svelte'
|
||||
import FlowModuleEarlyStop from './FlowModuleEarlyStop.svelte'
|
||||
import { logIterationLabel } from '../iterationLabelTelemetry'
|
||||
|
||||
import InputTransformForm from '$lib/components/InputTransformForm.svelte'
|
||||
import PropPickerWrapper from '../propPicker/PropPickerWrapper.svelte'
|
||||
@@ -45,6 +46,7 @@
|
||||
|
||||
let editor: SimpleEditor | undefined = $state(undefined)
|
||||
let parallelismEditor: SimpleEditor | undefined = $state(undefined)
|
||||
let iterationLabelEditor: SimpleEditor | undefined = $state(undefined)
|
||||
let runSettings: FlowRunSettings | undefined = $state(undefined)
|
||||
|
||||
let parallelismSchema = $state(emptySchema())
|
||||
@@ -60,6 +62,40 @@
|
||||
}
|
||||
iteratorSchema.required = ['iterator']
|
||||
|
||||
let iterationLabelSchema = $state(emptySchema())
|
||||
iterationLabelSchema.properties['iteration_label'] = {
|
||||
type: 'string'
|
||||
}
|
||||
|
||||
// The form edits its `arg` in place, so an unset label is backed by this object and moved
|
||||
// onto the step once it holds an expression; an emptied label is removed from the step
|
||||
// rather than stored as an empty expression.
|
||||
let iterationLabelPlaceholder: InputTransform = $state({ type: 'javascript', expr: '' })
|
||||
let showIterationLabel = $state(
|
||||
mod.value.type === 'forloopflow' && mod.value.iteration_label != undefined
|
||||
)
|
||||
$effect(() => {
|
||||
if (mod.value.type !== 'forloopflow') return
|
||||
const label = mod.value.iteration_label
|
||||
if (label == undefined) {
|
||||
// The editor outlives the toggle by its exit transition and writes into the placeholder.
|
||||
if (!showIterationLabel) return
|
||||
const expr =
|
||||
iterationLabelPlaceholder.type === 'javascript' ? iterationLabelPlaceholder.expr : ''
|
||||
if (!emptyString(expr)) {
|
||||
untrack(() => {
|
||||
;(mod.value as ForloopFlow).iteration_label = { type: 'javascript', expr }
|
||||
iterationLabelPlaceholder = { type: 'javascript', expr: '' }
|
||||
logIterationLabel('set')
|
||||
})
|
||||
}
|
||||
} else if (label.type === 'javascript' && emptyString(label.expr)) {
|
||||
untrack(() => {
|
||||
;(mod.value as ForloopFlow).iteration_label = undefined
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
if (mod.value.type === 'forloopflow') {
|
||||
const forloopValue = mod.value as ForloopFlow
|
||||
if (typeof forloopValue.parallelism === 'number') {
|
||||
@@ -140,6 +176,8 @@
|
||||
const ITERATOR_MISSING = 'An iterator expression is required for the loop to run.'
|
||||
const ITERATOR_TOOLTIP =
|
||||
'The JavaScript expression that will be evaluated to get the list of items to iterate over. Example: ["banana", "apple", flow_input.my_fruit].'
|
||||
const ITERATION_LABEL_TOOLTIP =
|
||||
'Evaluated for each iteration to name it in the run viewer and the iteration picker, e.g. flow_input.iter.value.name. A failing expression only leaves the iteration unnamed.'
|
||||
const DEFAULT_PARALLELISM = 4
|
||||
const PARALLELISM_LABEL = 'Limit concurrent iterations'
|
||||
const SQUASH_PARALLEL_CONFLICT =
|
||||
@@ -365,6 +403,56 @@
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<div class="flex flex-col gap-2">
|
||||
<Toggle
|
||||
size="xs"
|
||||
textClass="text-xs font-normal text-primary"
|
||||
bind:checked={showIterationLabel}
|
||||
on:change={({ detail }) => {
|
||||
if (detail) {
|
||||
iterationLabelPlaceholder = { type: 'javascript', expr: '' }
|
||||
} else {
|
||||
;(mod.value as ForloopFlow).iteration_label = undefined
|
||||
}
|
||||
}}
|
||||
options={{
|
||||
right: 'Custom iteration labels',
|
||||
rightTooltip: ITERATION_LABEL_TOOLTIP
|
||||
}}
|
||||
/>
|
||||
{#if showIterationLabel}
|
||||
<div class="pl-9" transition:slideDynamic>
|
||||
<PropPickerWrapper
|
||||
sidePane
|
||||
flow_input={stepPropPicker.pickableProperties.flow_input}
|
||||
notSelectable
|
||||
pickableProperties={stepPropPicker.pickableProperties}
|
||||
on:select={({ detail }) => {
|
||||
iterationLabelEditor?.insertAtCursor(detail)
|
||||
iterationLabelEditor?.focus()
|
||||
}}
|
||||
>
|
||||
<InputTransformForm
|
||||
bind:arg={
|
||||
() => (mod.value as ForloopFlow).iteration_label ?? iterationLabelPlaceholder,
|
||||
(v) => {
|
||||
;(mod.value as ForloopFlow).iteration_label = v
|
||||
}
|
||||
}
|
||||
argName="iteration_label"
|
||||
label="Label expression"
|
||||
schema={iterationLabelSchema}
|
||||
noDynamicToggle
|
||||
extraLib={stepPropPicker.extraLib}
|
||||
pickableProperties={stepPropPicker.pickableProperties}
|
||||
previousModuleId={previousModule?.id}
|
||||
bind:editor={iterationLabelEditor}
|
||||
/>
|
||||
</PropPickerWrapper>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<FlowModuleEarlyStop blocks="stop-after" bind:flowModule={mod} />
|
||||
</section>
|
||||
{:else}
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { logFeatureUsage } from '$lib/utils/featureUsage'
|
||||
|
||||
/**
|
||||
* `set`: a for-loop step gained an iteration label in the editor.
|
||||
* `searched`: an iteration was picked by typing label text into the iteration picker.
|
||||
* Counts only; the label expression and its values never reach here.
|
||||
*/
|
||||
export type IterationLabelAction = 'set' | 'searched'
|
||||
|
||||
export function logIterationLabel(action: IterationLabelAction): void {
|
||||
logFeatureUsage('flow_step', 'iteration_label', { key: action })
|
||||
}
|
||||
@@ -6,12 +6,15 @@
|
||||
import VirtualList from '@tutorlatin/svelte-tiny-virtual-list'
|
||||
import type { OnSelectedIteration } from '$lib/components/graph/graphBuilder.svelte'
|
||||
import { untrack } from 'svelte'
|
||||
import TextInput from '$lib/components/text_input/TextInput.svelte'
|
||||
import { logIterationLabel } from '../iterationLabelTelemetry'
|
||||
|
||||
interface Props {
|
||||
id: string
|
||||
moduleId: string
|
||||
flowJobs: string[] | undefined
|
||||
flowJobsSuccess: (boolean | undefined)[] | undefined
|
||||
flowJobsLabels?: (string | null)[] | undefined
|
||||
selected: number
|
||||
selectedManually: boolean | undefined
|
||||
onSelectedIteration?: OnSelectedIteration
|
||||
@@ -22,6 +25,7 @@
|
||||
id,
|
||||
flowJobs,
|
||||
flowJobsSuccess,
|
||||
flowJobsLabels,
|
||||
selected,
|
||||
selectedManually,
|
||||
onSelectedIteration,
|
||||
@@ -29,41 +33,61 @@
|
||||
showIcon = true
|
||||
}: Props = $props()
|
||||
|
||||
let filter: number | undefined = $state(undefined)
|
||||
let filter: string = $state('')
|
||||
function onKeydown(event: KeyboardEvent) {
|
||||
if (
|
||||
event.key === 'Enter' &&
|
||||
filter != undefined &&
|
||||
flowJobs &&
|
||||
filter < flowJobs.length &&
|
||||
filter > 0
|
||||
) {
|
||||
event.preventDefault()
|
||||
onSelectedIteration?.({
|
||||
index: filter - 1,
|
||||
id: flowJobs[filter - 1],
|
||||
manuallySet: true,
|
||||
moduleId: moduleId
|
||||
})
|
||||
menu?.close()
|
||||
if (event.key !== 'Enter' || !flowJobs) {
|
||||
return
|
||||
}
|
||||
const n = Number(filter.trim())
|
||||
const byNumber = Number.isInteger(n) && n > 0 && n <= flowJobs.length
|
||||
const index = byNumber ? n - 1 : items[0]?.index
|
||||
if (index == undefined) {
|
||||
return
|
||||
}
|
||||
event.preventDefault()
|
||||
logIfSearchedByLabel()
|
||||
onSelectedIteration?.({
|
||||
index,
|
||||
id: flowJobs[index],
|
||||
manuallySet: true,
|
||||
moduleId: moduleId
|
||||
})
|
||||
menu?.close()
|
||||
}
|
||||
|
||||
let hasLabels = $derived(flowJobsLabels?.some((l) => l) ?? false)
|
||||
|
||||
function logIfSearchedByLabel() {
|
||||
const query = filter.trim()
|
||||
if (hasLabels && query != '' && !/^\d+$/.test(query)) logIterationLabel('searched')
|
||||
}
|
||||
|
||||
let buttonHover = $state(false)
|
||||
let menu: Menu | undefined = $state(undefined)
|
||||
|
||||
let items: { id: string; success: boolean | undefined; index: number }[] = $state([])
|
||||
let items: {
|
||||
id: string
|
||||
success: boolean | undefined
|
||||
label: string | undefined
|
||||
index: number
|
||||
}[] = $state([])
|
||||
function updateItems() {
|
||||
const query = filter.trim().toLowerCase()
|
||||
items = (flowJobs ?? [])
|
||||
.map((v, i) => {
|
||||
return {
|
||||
id: v,
|
||||
success: flowJobsSuccess?.[i],
|
||||
label: flowJobsLabels?.[i] ?? undefined,
|
||||
index: i
|
||||
}
|
||||
})
|
||||
.filter((v) => {
|
||||
return filter == undefined || (v.index + 1).toString().includes(filter.toString())
|
||||
return (
|
||||
query == '' ||
|
||||
(v.index + 1).toString().includes(query) ||
|
||||
(v.label?.toLowerCase().includes(query) ?? false)
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -118,7 +142,7 @@
|
||||
>
|
||||
{#snippet triggr({ trigger })}
|
||||
<MeltButton
|
||||
title="Pick an iteration"
|
||||
title={(selected >= 0 && flowJobsLabels?.[selected]) || 'Pick an iteration'}
|
||||
id={`flow-editor-iteration picker-${id}`}
|
||||
type="button"
|
||||
class={twMerge(
|
||||
@@ -136,8 +160,15 @@
|
||||
{/snippet}
|
||||
|
||||
{#snippet children({ item: childrenItem })}
|
||||
<div class="flex flex-col px-1">
|
||||
<input type="number" bind:value={filter} onkeydown={onKeydown} />
|
||||
<div class={twMerge('flex flex-col px-1', hasLabels ? 'w-64' : '')}>
|
||||
<TextInput
|
||||
bind:value={filter}
|
||||
size="xs"
|
||||
inputProps={{
|
||||
onkeydown: onKeydown,
|
||||
placeholder: hasLabels ? 'Iteration # or label' : 'Iteration #'
|
||||
}}
|
||||
/>
|
||||
|
||||
<div class="max-h-[300px]">
|
||||
{#key items}
|
||||
@@ -154,6 +185,7 @@
|
||||
items[idx].index == selected ? 'bg-surface-selected' : ''
|
||||
)}
|
||||
onClick={() => {
|
||||
logIfSearchedByLabel()
|
||||
onSelectedIteration?.({
|
||||
moduleId: moduleId,
|
||||
index: items[idx].index,
|
||||
@@ -173,6 +205,11 @@
|
||||
: 'bg-yellow-400'}"
|
||||
></span>
|
||||
#{items[idx].index + 1}
|
||||
{#if items[idx].label}
|
||||
<span class="truncate text-secondary" title={items[idx].label}
|
||||
>{items[idx].label}</span
|
||||
>
|
||||
{/if}
|
||||
</MenuItem>
|
||||
</div>
|
||||
{/snippet}
|
||||
|
||||
@@ -34,6 +34,7 @@
|
||||
flowJobs: string[]
|
||||
selected: number
|
||||
flowJobsSuccess: (boolean | undefined)[]
|
||||
flowJobsLabels?: (string | null)[]
|
||||
selectedManually: boolean | undefined
|
||||
}
|
||||
| undefined
|
||||
@@ -179,6 +180,7 @@
|
||||
id={mod.id}
|
||||
{onSelectedIteration}
|
||||
flowJobsSuccess={flowJobs.flowJobsSuccess}
|
||||
flowJobsLabels={flowJobs.flowJobsLabels}
|
||||
flowJobs={flowJobs.flowJobs}
|
||||
selected={flowJobs.selected}
|
||||
selectedManually={flowJobs.selectedManually}
|
||||
|
||||
@@ -53,6 +53,8 @@ export type GraphModuleState = {
|
||||
selectedForloopIndex?: number
|
||||
selectedForLoopSetManually?: boolean
|
||||
flow_jobs_success?: (boolean | undefined)[]
|
||||
/** A for-loop's per-iteration labels, positionally aligned with `flow_jobs`. */
|
||||
flow_jobs_labels?: (string | null)[]
|
||||
flow_jobs_duration?: {
|
||||
started_at?: (string | undefined)[]
|
||||
duration_ms?: (number | undefined)[]
|
||||
|
||||
@@ -93,7 +93,8 @@
|
||||
flowJobs: state?.flow_jobs,
|
||||
selected: state?.selectedForloopIndex ?? 0,
|
||||
selectedManually: state?.selectedForLoopSetManually,
|
||||
flowJobsSuccess: state?.flow_jobs_success
|
||||
flowJobsSuccess: state?.flow_jobs_success,
|
||||
flowJobsLabels: state?.flow_jobs_labels
|
||||
}
|
||||
: (undefined as any)
|
||||
)
|
||||
|
||||
@@ -912,6 +912,9 @@ components:
|
||||
$ref: '#/components/schemas/InputTransform'
|
||||
squash:
|
||||
type: boolean
|
||||
iteration_label:
|
||||
description: Optional expression evaluated once per iteration into the label the run viewers show for it, e.g. 'flow_input.iter.value.name'. It can reference 'flow_input.iter' and 'results'. Purely informational, a failing expression leaves the iteration unlabelled and never fails it. Labels are capped at 100 characters
|
||||
$ref: '#/components/schemas/InputTransform'
|
||||
required:
|
||||
- modules
|
||||
- iterator
|
||||
@@ -1454,6 +1457,12 @@ components:
|
||||
type: array
|
||||
items:
|
||||
type: boolean
|
||||
flow_jobs_labels:
|
||||
type: array
|
||||
description: For a for-loop step whose iteration_label is set, each iteration's label, aligned with flow_jobs. An entry is null when its label evaluated to nothing
|
||||
items:
|
||||
type: string
|
||||
nullable: true
|
||||
flow_jobs_duration:
|
||||
type: object
|
||||
properties:
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -320,6 +320,7 @@ can call one, not an agent used as a tool.
|
||||
## Loop Structure Rules
|
||||
|
||||
- A `forloopflow` runs its `modules` once per element of `iterator`, a javascript expression returning an array (e.g. `results.get_items`); `parallel: true` runs the iterations concurrently, and `skip_failures: true` carries on past a failed iteration
|
||||
- When iterations are items a user will look for in a run (customers, files, ids), set `iteration_label` on the `forloopflow` value to a javascript expression naming each one, e.g. `flow_input.iter.value.name`; it only labels iterations in the run viewer and never changes what runs
|
||||
- For `whileloopflow`, break the loop with a module-level `stop_after_if`: on the loop module itself, or on an inner step (required when that step carries state via its own `results` — see below)
|
||||
- `stop_after_if` is always a sibling of `id` and `value` on a flow module — never a direct key of the loop's `value` object
|
||||
- `stop_after_all_iters_if` is for checks after the whole loop finishes, not the normal per-iteration break condition
|
||||
|
||||
Reference in New Issue
Block a user