fix: retry js eval up to 3 times on timeout from slow DB (#7890)

When eval_timeout_quickjs hits the timeout (typically due to slow DB
result retrieval), retry up to 2 more times with a 5s interval between
attempts. Non-timeout errors are returned immediately without retry.

Also extract the eval timeout duration as EVAL_TIMEOUT_MS const (set to
20000ms, up from 10000ms) in windmill-jseval.

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: Alexander Petric <alpetric@users.noreply.github.com>

backend/windmill-jseval/src/lib.rs

@@ -19,6 +19,8 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
+pub const EVAL_TIMEOUT_MS: u64 = 20000;
+
 use lazy_static::lazy_static;
 use regex::Regex;
 #[cfg(feature = "quickjs")]
@@ -274,7 +276,7 @@ pub async fn eval_timeout_quickjs(
 
     // Run the QuickJS evaluation with a timeout
     tokio::time::timeout(
-        std::time::Duration::from_millis(10000),
+        std::time::Duration::from_millis(EVAL_TIMEOUT_MS),
         tokio::task::spawn_blocking(move || {
             // Create a new tokio runtime for async operations within the blocking context
             let rt = tokio::runtime::Builder::new_current_thread()
@@ -298,7 +300,7 @@ pub async fn eval_timeout_quickjs(
     )
     .await
     .map_err(|_| {
-        anyhow::anyhow!("The expression evaluation `{expr}` took too long to execute (>10000ms)")
+        anyhow::anyhow!("The expression evaluation `{expr}` took too long to execute (>{EVAL_TIMEOUT_MS}ms)")
     })??
 }
 
@@ -786,7 +788,7 @@ pub async fn eval_simple_js(
     globals: HashMap<String, serde_json::Value>,
 ) -> anyhow::Result<Box<RawValue>> {
     tokio::time::timeout(
-        std::time::Duration::from_millis(10000),
+        std::time::Duration::from_millis(EVAL_TIMEOUT_MS),
         tokio::task::spawn_blocking(move || {
             let rt = tokio::runtime::Builder::new_current_thread()
                 .enable_all()
@@ -819,7 +821,7 @@ pub async fn eval_simple_js(
     )
     .await
     .map_err(|_| {
-        anyhow::anyhow!("The expression evaluation took too long to execute (>10000ms)")
+        anyhow::anyhow!("The expression evaluation took too long to execute (>{EVAL_TIMEOUT_MS}ms)")
     })??
 }
 

backend/windmill-worker/src/js_eval.rs

@@ -78,16 +78,33 @@ pub async fn eval_timeout(
         }
     }
 
-    windmill_jseval::eval_timeout_quickjs(
-        expr,
-        transform_context,
-        flow_input,
-        flow_env,
-        authed_client,
-        by_id,
-        ctx,
-    )
-    .await
+    let mut attempts = 0;
+    loop {
+        let result = windmill_jseval::eval_timeout_quickjs(
+            expr.clone(),
+            transform_context.clone(),
+            flow_input.clone(),
+            flow_env,
+            authed_client,
+            by_id,
+            ctx.clone(),
+        )
+        .await;
+
+        match result {
+            Ok(v) => return Ok(v),
+            Err(e) if attempts < 2 && e.to_string().contains("took too long") => {
+                attempts += 1;
+                tracing::warn!(
+                    "js eval timed out (attempt {}/3), retrying in 5s: {}",
+                    attempts,
+                    expr
+                );
+                tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+            }
+            Err(e) => return Err(e),
+        }
+    }
 }
 
 #[cfg(feature = "deno_core")]