From 567d6216d2631a90fbe59ec6142c38b3b352eea7 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 2 Nov 2024 13:57:30 +0100 Subject: [PATCH] fix: apply NO_PROXY and HTTP_PROXY, HTTPS_PROXY more consistently --- .../windmill-worker/src/ansible_executor.rs | 36 +++++++++---------- backend/windmill-worker/src/bash_executor.rs | 4 ++- backend/windmill-worker/src/bun_executor.rs | 3 +- .../windmill-worker/src/dedicated_worker.rs | 3 +- backend/windmill-worker/src/go_executor.rs | 2 ++ .../windmill-worker/src/python_executor.rs | 28 ++++----------- backend/windmill-worker/src/rust_executor.rs | 3 +- backend/windmill-worker/src/worker.rs | 25 ++++++++----- 8 files changed, 51 insertions(+), 53 deletions(-) diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index e897b86321..abd9b68415 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -35,7 +35,7 @@ use crate::{ handle_child::handle_child, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - TZ_ENV, + PROXY_ENVS, TZ_ENV, }; lazy_static::lazy_static! { @@ -147,6 +147,7 @@ async fn install_galaxy_collections( galaxy_command .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) // .env("BASE_INTERNAL_URL", base_internal_url) @@ -395,6 +396,7 @@ fi nsjail_cmd .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) // inject PYTHONPATH here - for some reason I had to do it in nsjail conf .envs(reserved_variables) .env("PATH", PATH_ENV.as_str()) @@ -556,19 +558,12 @@ async fn create_file_resources( } for file_res in &r.file_resources { - let r = get_resource_or_variable_content( - client, - &file_res.resource_path, - job_id.to_string(), - ) - .await?; + let r = + get_resource_or_variable_content(client, &file_res.resource_path, job_id.to_string()) + .await?; let path = file_res.target_path.clone(); - let validated_path = write_file_at_user_defined_location( - job_dir, - path.as_str(), - &r, - ) - .map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?; + let validated_path = write_file_at_user_defined_location(job_dir, path.as_str(), &r) + .map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?; nsjail_mounts.push( define_nsjail_mount(job_dir, &validated_path) @@ -596,13 +591,14 @@ async fn get_resource_or_variable_content( .get_resource_value_interpolated::(&p, Some(job_id)) .await?; - r.get("content").and_then(|v| v.as_str()).ok_or(anyhow!( - "Invalid text file resource {}, `content` field absent or invalid", - p - ))?.to_string() - } - ResourceOrVariablePath::Variable(p) => { - client.get_variable_value(&p).await? + r.get("content") + .and_then(|v| v.as_str()) + .ok_or(anyhow!( + "Invalid text file resource {}, `content` field absent or invalid", + p + ))? + .to_string() } + ResourceOrVariablePath::Variable(p) => client.get_variable_value(&p).await?, }) } diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index ed7a1dd0f6..2c4c46ab7d 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -29,7 +29,7 @@ use crate::{ }, handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - POWERSHELL_CACHE_DIR, POWERSHELL_PATH, TZ_ENV, + POWERSHELL_CACHE_DIR, POWERSHELL_PATH, PROXY_ENVS, TZ_ENV, }; #[cfg(windows)] @@ -143,6 +143,7 @@ wait .current_dir(job_dir) .env_clear() .envs(reserved_variables) + .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .args(cmd_args) @@ -435,6 +436,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", Command::new(NSJAIL_PATH.as_str()) .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) .envs(reserved_variables) .env("TZ", TZ_ENV.as_str()) .env("PATH", PATH_ENV.as_str()) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 4a8c0749bc..5b7c2a06c0 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -26,7 +26,7 @@ use crate::{ handle_child::handle_child, AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, - NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, TZ_ENV, + NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TZ_ENV, }; #[cfg(windows)] @@ -275,6 +275,7 @@ pub async fn install_bun_lockfile( child_cmd .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) .envs(common_bun_proc_envs) .args(vec!["install"]) .stdout(Stdio::piped()) diff --git a/backend/windmill-worker/src/dedicated_worker.rs b/backend/windmill-worker/src/dedicated_worker.rs index da3e25a699..1a332e76ef 100644 --- a/backend/windmill-worker/src/dedicated_worker.rs +++ b/backend/windmill-worker/src/dedicated_worker.rs @@ -77,7 +77,7 @@ pub async fn handle_dedicated_process( ) -> std::result::Result<(), error::Error> { //do not cache local dependencies - use crate::handle_child::process_status; + use crate::{handle_child::process_status, PROXY_ENVS}; let mut child = { let mut cmd = Command::new(command_path); @@ -85,6 +85,7 @@ pub async fn handle_dedicated_process( .env_clear() .envs(context_envs) .envs(envs) + .envs(PROXY_ENVS.clone()) .envs( reserved_variables .iter() diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index c171e52156..470e647383 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -1,3 +1,4 @@ +use crate::PROXY_ENVS; use std::{collections::HashMap, fs::DirBuilder, process::Stdio}; use itertools::Itertools; @@ -189,6 +190,7 @@ func Run(req Req) (interface{{}}, error){{ .env("BASE_INTERNAL_URL", base_internal_url) .env("GOPATH", GO_CACHE_DIR) .env("HOME", HOME_ENV.as_str()) + .envs(PROXY_ENVS.clone()) .args(vec!["build", "main.go"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 5ee846e00b..112b3c84ca 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -65,9 +65,9 @@ use crate::{ read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, - AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY, - LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, - PIP_INDEX_URL, TZ_ENV, UV_CACHE_DIR, + AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, LOCK_CACHE_DIR, + NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, TZ_ENV, + UV_CACHE_DIR, }; #[cfg(windows)] @@ -565,6 +565,7 @@ mount {{ .env_clear() // inject PYTHONPATH here - for some reason I had to do it in nsjail conf .envs(reserved_variables) + .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -1003,15 +1004,6 @@ pub async fn handle_python_reqs( if let Some(host) = PIP_TRUSTED_HOST.as_ref() { vars.push(("TRUSTED_HOST", host)); } - if let Some(http_proxy) = HTTP_PROXY.as_ref() { - vars.push(("HTTP_PROXY", http_proxy)); - } - if let Some(https_proxy) = HTTPS_PROXY.as_ref() { - vars.push(("HTTPS_PROXY", https_proxy)); - } - if let Some(no_proxy) = NO_PROXY.as_ref() { - vars.push(("NO_PROXY", no_proxy)); - } let _ = write_file( job_dir, @@ -1140,6 +1132,7 @@ pub async fn handle_python_reqs( .current_dir(job_dir) .env_clear() .envs(vars) + .envs(PROXY_ENVS.clone()) .args(vec!["--config", "download.config.proto"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -1193,15 +1186,6 @@ pub async fn handle_python_reqs( } let mut envs = vec![("PATH", PATH_ENV.as_str())]; - if let Some(http_proxy) = HTTP_PROXY.as_ref() { - envs.push(("HTTP_PROXY", http_proxy)); - } - if let Some(https_proxy) = HTTPS_PROXY.as_ref() { - envs.push(("HTTPS_PROXY", https_proxy)); - } - if let Some(no_proxy) = NO_PROXY.as_ref() { - envs.push(("NO_PROXY", no_proxy)); - } envs.push(("HOME", HOME_ENV.as_str())); @@ -1212,6 +1196,7 @@ pub async fn handle_python_reqs( let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); flock_cmd .env_clear() + .envs(PROXY_ENVS.clone()) .envs(envs) .args([ "-x", @@ -1230,6 +1215,7 @@ pub async fn handle_python_reqs( pip_cmd .env_clear() .envs(envs) + .envs(PROXY_ENVS.clone()) .env("SystemRoot", SYSTEM_ROOT.as_str()) .args(&command_args[1..]) .stdout(Stdio::piped()) diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 530be394ff..9391fd9a69 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -20,7 +20,7 @@ use crate::{ }, handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - RUST_CACHE_DIR, TZ_ENV, + PROXY_ENVS, RUST_CACHE_DIR, TZ_ENV, }; #[cfg(windows)] @@ -192,6 +192,7 @@ pub async fn build_rust_crate( build_rust_cmd .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 85cbaba79a..f4d78bf0ef 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -324,6 +324,20 @@ lazy_static::lazy_static! { pub static ref NO_PROXY: Option = std::env::var("no_proxy").ok().or(std::env::var("NO_PROXY").ok()); pub static ref HTTP_PROXY: Option = std::env::var("http_proxy").ok().or(std::env::var("HTTP_PROXY").ok()); pub static ref HTTPS_PROXY: Option = std::env::var("https_proxy").ok().or(std::env::var("HTTPS_PROXY").ok()); + + pub static ref PROXY_ENVS: Vec<(&'static str, String)> = { + let mut proxy_env = Vec::new(); + if let Some(no_proxy) = NO_PROXY.as_ref() { + proxy_env.push(("NO_PROXY", no_proxy.to_string())); + } + if let Some(http_proxy) = HTTP_PROXY.as_ref() { + proxy_env.push(("HTTP_PROXY", http_proxy.to_string())); + } + if let Some(https_proxy) = HTTPS_PROXY.as_ref() { + proxy_env.push(("HTTPS_PROXY", https_proxy.to_string())); + } + proxy_env + }; pub static ref DENO_PATH: String = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string()); pub static ref BUN_PATH: String = std::env::var("BUN_PATH").unwrap_or_else(|_| "/usr/bin/bun".to_string()); pub static ref NPM_PATH: String = std::env::var("NPM_PATH").unwrap_or_else(|_| "/usr/bin/npm".to_string()); @@ -2067,15 +2081,10 @@ pub fn build_envs( hm }; - if let Some(ref env) = *HTTPS_PROXY { - envs.insert("HTTPS_PROXY".to_string(), env.to_string()); - } - if let Some(ref env) = *HTTP_PROXY { - envs.insert("HTTP_PROXY".to_string(), env.to_string()); - } - if let Some(ref env) = *NO_PROXY { - envs.insert("NO_PROXY".to_string(), env.to_string()); + for (k, v) in PROXY_ENVS.iter() { + envs.insert(k.to_string(), v.to_string()); } + Ok(envs) }