From abcd920964e583bd0bda7ed28126ae4862828495 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 29 Apr 2026 19:50:17 +0000 Subject: [PATCH] test: isolate WAC v2 python test from stack overflow (#8979) * test: isolate WAC v2 python test from test-thread stack overflow Co-Authored-By: Claude Opus 4.5 * ci: bump RUST_MIN_STACK to 4MB for backend tests Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: Claude Opus 4.5 --- .github/workflows/backend-test-windows.yml | 4 ++ .github/workflows/backend-test.yml | 5 ++ backend/tests/python_jobs.rs | 61 +++++++++++++--------- backend/windmill-test-utils/src/lib.rs | 30 +++++++++++ 4 files changed, 74 insertions(+), 26 deletions(-) diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index 864693ebda..2b76eda11f 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -145,6 +145,10 @@ jobs: RUST_LOG_STYLE: never CARGO_NET_GIT_FETCH_WITH_CLI: true CARGO_BUILD_JOBS: 12 + # Tests' poll-time stack frames (deep nested async fn chains in + # debug builds) reach ~1.8MB. 4MB gives ~2x headroom against flaky + # overflows under parallel-test contention. + RUST_MIN_STACK: 4194304 VCPKGRS_DYNAMIC: 1 OPENSSL_DIR: ${{ env.VCPKG_INSTALLATION_ROOT }}\installed\x64-windows-static DENO_PATH: ${{ steps.runtime-paths.outputs.DENO_PATH }} diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index cd6651fc7e..9009b47e9d 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -244,6 +244,11 @@ jobs: RUST_LOG_STYLE: never CARGO_NET_GIT_FETCH_WITH_CLI: true CARGO_BUILD_JOBS: 12 + # Tests' poll-time stack frames (deep nested async fn chains in + # debug builds) reach ~1.8MB, leaving very thin headroom on the + # default 2MB thread stack. 4MB gives ~2x buffer against flaky + # overflows under parallel-test contention. + RUST_MIN_STACK: 4194304 WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1 WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1 WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1 diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index f105b052f9..e8fbb1670f 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -985,33 +985,42 @@ async def main(item: str, qty: int, email: str): "# .to_string(); - // WAC requires at least 2 workers (parent + task sub-jobs) - let db = &db; - in_test_worker( - db, - async move { - let job = Box::pin( - RunJob::from(JobPayload::Code(RawCode { - language: ScriptLang::Python3, - content, - ..RawCode::default() - })) - .arg("item", json!("widget")) - .arg("qty", json!(5)) - .arg("email", json!("test@example.com")) - .run_until_complete(db, false, port), - ) - .await; + // WAC requires at least 2 workers (parent + task sub-jobs). + // + // Run the heavy `in_test_worker` chain on an isolated OS thread with a + // larger stack: the deep nested async chain (test -> in_test_worker -> + // run_until_complete -> windmill_queue::push -> ...) composes into one + // synchronous poll-stack frame that exceeds the default 2 MB test-thread + // stack in debug builds. `Box::pin` at the call site only moves future + // *state* to the heap; it can't shrink poll-time stack frames. + let db = db.clone(); + run_in_isolated_thread(move || async move { + in_test_worker( + &db, + async { + let job = Box::pin( + RunJob::from(JobPayload::Code(RawCode { + language: ScriptLang::Python3, + content, + ..RawCode::default() + })) + .arg("item", json!("widget")) + .arg("qty", json!(5)) + .arg("email", json!("test@example.com")) + .run_until_complete(&db, false, port), + ) + .await; - let result = job.json_result().unwrap(); - assert_eq!(result["item"], json!("widget")); - assert_eq!(result["qty"], json!(5)); - assert_eq!(result["email"], json!("test@example.com")); - assert_eq!(result["greeting"], json!("hello widget x5")); - }, - port, - ) - .await; + let result = job.json_result().unwrap(); + assert_eq!(result["item"], json!("widget")); + assert_eq!(result["qty"], json!(5)); + assert_eq!(result["email"], json!("test@example.com")); + assert_eq!(result["greeting"], json!("hello widget x5")); + }, + port, + ) + .await; + }); Ok(()) } diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index a24f08bfda..b26eb622fa 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -395,6 +395,36 @@ pub async fn in_test_worker( res } +/// Run an async test body on a freshly-spawned OS thread with a large stack +/// and a fresh current-thread tokio runtime. Use this for tests whose async +/// chain (e.g. nested `in_test_worker` -> `run_until_complete` -> +/// `windmill_queue::push`) blows the default 2 MB test-thread stack in debug +/// builds, where `async fn` state machines are unoptimized and `Box::pin` at +/// the call site only moves *future state* to the heap, not the synchronous +/// poll-time stack frames composed along the chain. +/// +/// The chain contains `?Send` futures (trait objects), so `tokio::spawn` +/// is not viable; `std::thread::spawn` sidesteps the shared-stack issue. +pub fn run_in_isolated_thread(f: F) -> R +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future, + R: Send + 'static, +{ + std::thread::Builder::new() + .stack_size(8 * 1024 * 1024) + .spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build current-thread runtime"); + rt.block_on(f()) + }) + .expect("spawn isolated test thread") + .join() + .expect("isolated test thread panicked") +} + pub fn spawn_test_worker( conn: &Connection, port: u16,