test: isolate WAC v2 python test from stack overflow (#8979)

* test: isolate WAC v2 python test from test-thread stack overflow

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* ci: bump RUST_MIN_STACK to 4MB for backend tests

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

.github/workflows/backend-test-windows.yml

@@ -145,6 +145,10 @@ jobs:
           RUST_LOG_STYLE: never
           CARGO_NET_GIT_FETCH_WITH_CLI: true
           CARGO_BUILD_JOBS: 12
+          # Tests' poll-time stack frames (deep nested async fn chains in
+          # debug builds) reach ~1.8MB. 4MB gives ~2x headroom against flaky
+          # overflows under parallel-test contention.
+          RUST_MIN_STACK: 4194304
           VCPKGRS_DYNAMIC: 1
           OPENSSL_DIR: ${{ env.VCPKG_INSTALLATION_ROOT }}\installed\x64-windows-static
           DENO_PATH: ${{ steps.runtime-paths.outputs.DENO_PATH }}

.github/workflows/backend-test.yml

@@ -244,6 +244,11 @@ jobs:
           RUST_LOG_STYLE: never
           CARGO_NET_GIT_FETCH_WITH_CLI: true
           CARGO_BUILD_JOBS: 12
+          # Tests' poll-time stack frames (deep nested async fn chains in
+          # debug builds) reach ~1.8MB, leaving very thin headroom on the
+          # default 2MB thread stack. 4MB gives ~2x buffer against flaky
+          # overflows under parallel-test contention.
+          RUST_MIN_STACK: 4194304
           WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1
           WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1
           WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1

backend/tests/python_jobs.rs

@@ -985,33 +985,42 @@ async def main(item: str, qty: int, email: str):
 "#
     .to_string();
 
-    // WAC requires at least 2 workers (parent + task sub-jobs)
-    let db = &db;
-    in_test_worker(
-        db,
-        async move {
-            let job = Box::pin(
-                RunJob::from(JobPayload::Code(RawCode {
-                    language: ScriptLang::Python3,
-                    content,
-                    ..RawCode::default()
-                }))
-                .arg("item", json!("widget"))
-                .arg("qty", json!(5))
-                .arg("email", json!("test@example.com"))
-                .run_until_complete(db, false, port),
-            )
-            .await;
+    // WAC requires at least 2 workers (parent + task sub-jobs).
+    //
+    // Run the heavy `in_test_worker` chain on an isolated OS thread with a
+    // larger stack: the deep nested async chain (test -> in_test_worker ->
+    // run_until_complete -> windmill_queue::push -> ...) composes into one
+    // synchronous poll-stack frame that exceeds the default 2 MB test-thread
+    // stack in debug builds. `Box::pin` at the call site only moves future
+    // *state* to the heap; it can't shrink poll-time stack frames.
+    let db = db.clone();
+    run_in_isolated_thread(move || async move {
+        in_test_worker(
+            &db,
+            async {
+                let job = Box::pin(
+                    RunJob::from(JobPayload::Code(RawCode {
+                        language: ScriptLang::Python3,
+                        content,
+                        ..RawCode::default()
+                    }))
+                    .arg("item", json!("widget"))
+                    .arg("qty", json!(5))
+                    .arg("email", json!("test@example.com"))
+                    .run_until_complete(&db, false, port),
+                )
+                .await;
 
-            let result = job.json_result().unwrap();
-            assert_eq!(result["item"], json!("widget"));
-            assert_eq!(result["qty"], json!(5));
-            assert_eq!(result["email"], json!("test@example.com"));
-            assert_eq!(result["greeting"], json!("hello widget x5"));
-        },
-        port,
-    )
-    .await;
+                let result = job.json_result().unwrap();
+                assert_eq!(result["item"], json!("widget"));
+                assert_eq!(result["qty"], json!(5));
+                assert_eq!(result["email"], json!("test@example.com"));
+                assert_eq!(result["greeting"], json!("hello widget x5"));
+            },
+            port,
+        )
+        .await;
+    });
     Ok(())
 }
 

backend/windmill-test-utils/src/lib.rs

@@ -395,6 +395,36 @@ pub async fn in_test_worker<Fut: std::future::Future>(
     res
 }
 
+/// Run an async test body on a freshly-spawned OS thread with a large stack
+/// and a fresh current-thread tokio runtime. Use this for tests whose async
+/// chain (e.g. nested `in_test_worker` -> `run_until_complete` ->
+/// `windmill_queue::push`) blows the default 2 MB test-thread stack in debug
+/// builds, where `async fn` state machines are unoptimized and `Box::pin` at
+/// the call site only moves *future state* to the heap, not the synchronous
+/// poll-time stack frames composed along the chain.
+///
+/// The chain contains `?Send` futures (trait objects), so `tokio::spawn`
+/// is not viable; `std::thread::spawn` sidesteps the shared-stack issue.
+pub fn run_in_isolated_thread<F, Fut, R>(f: F) -> R
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = R>,
+    R: Send + 'static,
+{
+    std::thread::Builder::new()
+        .stack_size(8 * 1024 * 1024)
+        .spawn(move || {
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("build current-thread runtime");
+            rt.block_on(f())
+        })
+        .expect("spawn isolated test thread")
+        .join()
+        .expect("isolated test thread panicked")
+}
+
 pub fn spawn_test_worker(
     conn: &Connection,
     port: u16,