starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-05 00:03:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: apply NO_PROXY and HTTP_PROXY, HTTPS_PROXY more consistently

Browse Source
This commit is contained in:
Ruben Fiszel
2024-11-02 13:57:30 +01:00
parent 69ac3cea4b
commit 567d6216d2
8 changed files with 51 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/windmill-worker/src/ansible_executor.rs
+16 -20
View File
@@ -35,7 +35,7 @@ use crate::{
handle_child::handle_child,
python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile},
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
TZ_ENV,
PROXY_ENVS, TZ_ENV,
};
lazy_static::lazy_static! {
@@ -147,6 +147,7 @@ async fn install_galaxy_collections(
galaxy_command
.current_dir(job_dir)
.env_clear()
.envs(PROXY_ENVS.clone())
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
// .env("BASE_INTERNAL_URL", base_internal_url)
@@ -395,6 +396,7 @@ fi
nsjail_cmd
.current_dir(job_dir)
.env_clear()
.envs(PROXY_ENVS.clone())
// inject PYTHONPATH here - for some reason I had to do it in nsjail conf
.envs(reserved_variables)
.env("PATH", PATH_ENV.as_str())
@@ -556,19 +558,12 @@ async fn create_file_resources(
}
for file_res in &r.file_resources {
let r = get_resource_or_variable_content(
client,
&file_res.resource_path,
job_id.to_string(),
)
.await?;
let r =
get_resource_or_variable_content(client, &file_res.resource_path, job_id.to_string())
.await?;
let path = file_res.target_path.clone();
let validated_path = write_file_at_user_defined_location(
job_dir,
path.as_str(),
&r,
)
.map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?;
let validated_path = write_file_at_user_defined_location(job_dir, path.as_str(), &r)
.map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?;
nsjail_mounts.push(
define_nsjail_mount(job_dir, &validated_path)
@@ -596,13 +591,14 @@ async fn get_resource_or_variable_content(
.get_resource_value_interpolated::<serde_json::Value>(&p, Some(job_id))
.await?;
r.get("content").and_then(|v| v.as_str()).ok_or(anyhow!(
"Invalid text file resource {}, `content` field absent or invalid",
p
))?.to_string()
}
ResourceOrVariablePath::Variable(p) => {
client.get_variable_value(&p).await?
r.get("content")
.and_then(|v| v.as_str())
.ok_or(anyhow!(
"Invalid text file resource {}, `content` field absent or invalid",
p
))?
.to_string()
}
ResourceOrVariablePath::Variable(p) => client.get_variable_value(&p).await?,
})
}
backend/windmill-worker/src/bash_executor.rs
+3 -1
View File
@@ -29,7 +29,7 @@ use crate::{
},
handle_child::handle_child,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
POWERSHELL_CACHE_DIR, POWERSHELL_PATH, TZ_ENV,
POWERSHELL_CACHE_DIR, POWERSHELL_PATH, PROXY_ENVS, TZ_ENV,
};
#[cfg(windows)]
@@ -143,6 +143,7 @@ wait
.current_dir(job_dir)
.env_clear()
.envs(reserved_variables)
.envs(PROXY_ENVS.clone())
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.args(cmd_args)
@@ -435,6 +436,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
Command::new(NSJAIL_PATH.as_str())
.current_dir(job_dir)
.env_clear()
.envs(PROXY_ENVS.clone())
.envs(reserved_variables)
.env("TZ", TZ_ENV.as_str())
.env("PATH", PATH_ENV.as_str())
backend/windmill-worker/src/bun_executor.rs
+2 -1
View File
@@ -26,7 +26,7 @@ use crate::{
handle_child::handle_child,
AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH,
NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, TZ_ENV,
NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TZ_ENV,
};
#[cfg(windows)]
@@ -275,6 +275,7 @@ pub async fn install_bun_lockfile(
child_cmd
.current_dir(job_dir)
.env_clear()
.envs(PROXY_ENVS.clone())
.envs(common_bun_proc_envs)
.args(vec!["install"])
.stdout(Stdio::piped())
backend/windmill-worker/src/dedicated_worker.rs
+2 -1
View File
@@ -77,7 +77,7 @@ pub async fn handle_dedicated_process(
) -> std::result::Result<(), error::Error> {
//do not cache local dependencies
use crate::handle_child::process_status;
use crate::{handle_child::process_status, PROXY_ENVS};
let mut child = {
let mut cmd = Command::new(command_path);
@@ -85,6 +85,7 @@ pub async fn handle_dedicated_process(
.env_clear()
.envs(context_envs)
.envs(envs)
.envs(PROXY_ENVS.clone())
.envs(
reserved_variables
.iter()
backend/windmill-worker/src/go_executor.rs
+2
View File
@@ -1,3 +1,4 @@
use crate::PROXY_ENVS;
use std::{collections::HashMap, fs::DirBuilder, process::Stdio};
use itertools::Itertools;
@@ -189,6 +190,7 @@ func Run(req Req) (interface{{}}, error){{
.env("BASE_INTERNAL_URL", base_internal_url)
.env("GOPATH", GO_CACHE_DIR)
.env("HOME", HOME_ENV.as_str())
.envs(PROXY_ENVS.clone())
.args(vec!["build", "main.go"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
backend/windmill-worker/src/python_executor.rs
+7 -21
View File
@@ -65,9 +65,9 @@ use crate::{
read_result, start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY,
LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL,
PIP_INDEX_URL, TZ_ENV, UV_CACHE_DIR,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, LOCK_CACHE_DIR,
NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, TZ_ENV,
UV_CACHE_DIR,
};
#[cfg(windows)]
@@ -565,6 +565,7 @@ mount {{
.env_clear()
// inject PYTHONPATH here - for some reason I had to do it in nsjail conf
.envs(reserved_variables)
.envs(PROXY_ENVS.clone())
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
@@ -1003,15 +1004,6 @@ pub async fn handle_python_reqs(
if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
vars.push(("TRUSTED_HOST", host));
}
if let Some(http_proxy) = HTTP_PROXY.as_ref() {
vars.push(("HTTP_PROXY", http_proxy));
}
if let Some(https_proxy) = HTTPS_PROXY.as_ref() {
vars.push(("HTTPS_PROXY", https_proxy));
}
if let Some(no_proxy) = NO_PROXY.as_ref() {
vars.push(("NO_PROXY", no_proxy));
}
let _ = write_file(
job_dir,
@@ -1140,6 +1132,7 @@ pub async fn handle_python_reqs(
.current_dir(job_dir)
.env_clear()
.envs(vars)
.envs(PROXY_ENVS.clone())
.args(vec!["--config", "download.config.proto"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
@@ -1193,15 +1186,6 @@ pub async fn handle_python_reqs(
}
let mut envs = vec![("PATH", PATH_ENV.as_str())];
if let Some(http_proxy) = HTTP_PROXY.as_ref() {
envs.push(("HTTP_PROXY", http_proxy));
}
if let Some(https_proxy) = HTTPS_PROXY.as_ref() {
envs.push(("HTTPS_PROXY", https_proxy));
}
if let Some(no_proxy) = NO_PROXY.as_ref() {
envs.push(("NO_PROXY", no_proxy));
}
envs.push(("HOME", HOME_ENV.as_str()));
@@ -1212,6 +1196,7 @@ pub async fn handle_python_reqs(
let mut flock_cmd = Command::new(FLOCK_PATH.as_str());
flock_cmd
.env_clear()
.envs(PROXY_ENVS.clone())
.envs(envs)
.args([
"-x",
@@ -1230,6 +1215,7 @@ pub async fn handle_python_reqs(
pip_cmd
.env_clear()
.envs(envs)
.envs(PROXY_ENVS.clone())
.env("SystemRoot", SYSTEM_ROOT.as_str())
.args(&command_args[1..])
.stdout(Stdio::piped())
backend/windmill-worker/src/rust_executor.rs
+2 -1
View File
@@ -20,7 +20,7 @@ use crate::{
},
handle_child::handle_child,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
RUST_CACHE_DIR, TZ_ENV,
PROXY_ENVS, RUST_CACHE_DIR, TZ_ENV,
};
#[cfg(windows)]
@@ -192,6 +192,7 @@ pub async fn build_rust_crate(
build_rust_cmd
.current_dir(job_dir)
.env_clear()
.envs(PROXY_ENVS.clone())
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.env("HOME", HOME_ENV.as_str())
backend/windmill-worker/src/worker.rs
+17 -8
View File
@@ -324,6 +324,20 @@ lazy_static::lazy_static! {
pub static ref NO_PROXY: Option<String> = std::env::var("no_proxy").ok().or(std::env::var("NO_PROXY").ok());
pub static ref HTTP_PROXY: Option<String> = std::env::var("http_proxy").ok().or(std::env::var("HTTP_PROXY").ok());
pub static ref HTTPS_PROXY: Option<String> = std::env::var("https_proxy").ok().or(std::env::var("HTTPS_PROXY").ok());
pub static ref PROXY_ENVS: Vec<(&'static str, String)> = {
let mut proxy_env = Vec::new();
if let Some(no_proxy) = NO_PROXY.as_ref() {
proxy_env.push(("NO_PROXY", no_proxy.to_string()));
}
if let Some(http_proxy) = HTTP_PROXY.as_ref() {
proxy_env.push(("HTTP_PROXY", http_proxy.to_string()));
}
if let Some(https_proxy) = HTTPS_PROXY.as_ref() {
proxy_env.push(("HTTPS_PROXY", https_proxy.to_string()));
}
proxy_env
};
pub static ref DENO_PATH: String = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string());
pub static ref BUN_PATH: String = std::env::var("BUN_PATH").unwrap_or_else(|_| "/usr/bin/bun".to_string());
pub static ref NPM_PATH: String = std::env::var("NPM_PATH").unwrap_or_else(|_| "/usr/bin/npm".to_string());
@@ -2067,15 +2081,10 @@ pub fn build_envs(
hm
};
if let Some(ref env) = *HTTPS_PROXY {
envs.insert("HTTPS_PROXY".to_string(), env.to_string());
}
if let Some(ref env) = *HTTP_PROXY {
envs.insert("HTTP_PROXY".to_string(), env.to_string());
}
if let Some(ref env) = *NO_PROXY {
envs.insert("NO_PROXY".to_string(), env.to_string());
for (k, v) in PROXY_ENVS.iter() {
envs.insert(k.to_string(), v.to_string());
}
Ok(envs)
}