diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index a513dad5dc..cdfb666d9f 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -7,6 +7,7 @@ use std::{ }; use anyhow::anyhow; +use futures::lock::Mutex; use itertools::Itertools; use regex::Regex; use serde_json::value::RawValue; @@ -36,6 +37,8 @@ use windmill_queue::{append_logs, CanceledBy}; lazy_static::lazy_static! { + static ref BUSY_WITH_UV_INSTALL: Mutex<()> = Mutex::new(()); + static ref PYTHON_PATH: String = std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string()); @@ -1354,6 +1357,7 @@ pub async fn handle_python_reqs( mut no_uv_install: bool, is_ansible: bool, ) -> error::Result> { + let lock = BUSY_WITH_UV_INSTALL.lock().await; let counter_arc = Arc::new(tokio::sync::Mutex::new(0)); // Append logs with line like this: // [9/21] + requests==2.32.3 << (S3) | in 57ms @@ -1517,6 +1521,12 @@ pub async fn handle_python_reqs( let pids = Arc::new(tokio::sync::Mutex::new(vec![None; total_to_install])); let mem_peak_thread_safe = Arc::new(tokio::sync::Mutex::new(0)); { + // when we cancel the job, it has up to 1 second window before actually getting cancelled + // Thus the directory with wheel in windmill's cache cleaned only after that. + // If we manage to start new job during that period windmill might see that wanted wheel is already there (because we have not cleaned it yet) + // and write it to installed wheels, meanwhile previous job will clean that wheel. + // To fix that we create lock, which will pipeline all uv installs on worker + let _lock = lock; let pids = pids.clone(); let mem_peak_thread_safe = mem_peak_thread_safe.clone(); tokio::spawn(async move {