From 9c855652212dbac0e49f87dedd447d3d7d7b500a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 10 Apr 2026 12:32:19 -0400 Subject: [PATCH] fix: bypass OTEL MITM tracing proxy for git sync jobs (#8796) Git sync runs as a DeploymentCallback job. When the OTEL MITM tracing proxy is enabled, all HTTP/HTTPS traffic from the script is rerouted through a local intercepting proxy that chains to the corporate upstream proxy. Git's HTTPS to GitHub fails in this setup (TLS interception with chained CONNECT tunneling is fragile, and git's CA env handling diverges from what the proxy injects), so customers see "GitHub.com URL couldn't be reached" until they disable OTEL. Detect DeploymentCallback jobs in get_proxy_envs_for_lang and fall back to the stock PROXY_ENVS so git talks to the corporate proxy directly, unmodified. The git sync script is system code; we don't need HTTP spans for it. Co-authored-by: Claude Opus 4.6 (1M context) --- backend/windmill-worker/src/bash_executor.rs | 20 ++++++++++--- backend/windmill-worker/src/bun_executor.rs | 29 +++++++++++++++---- .../windmill-worker/src/csharp_executor.rs | 20 ++++++++++--- backend/windmill-worker/src/deno_executor.rs | 12 ++++++-- backend/windmill-worker/src/go_executor.rs | 22 ++++++++++++-- backend/windmill-worker/src/nu_executor.rs | 22 ++++++++++++-- .../windmill-worker/src/python_executor.rs | 20 ++++++++++--- backend/windmill-worker/src/r_executor.rs | 20 ++++++++++--- backend/windmill-worker/src/ruby_executor.rs | 24 +++++++++++---- backend/windmill-worker/src/rust_executor.rs | 20 ++++++++++--- backend/windmill-worker/src/worker.rs | 12 ++++++-- 11 files changed, 182 insertions(+), 39 deletions(-) diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 011a72fe15..31254cf0cf 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -231,8 +231,14 @@ exit $exit_status .env_clear() .envs(reserved_variables) .envs( - get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn) - .await?, + get_proxy_envs_for_lang( + &ScriptLang::Bash, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -261,8 +267,14 @@ exit $exit_status .envs(envs) .envs(reserved_variables) .envs( - get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn) - .await?, + get_proxy_envs_for_lang( + &ScriptLang::Bash, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 842d2c6659..0e44cc4750 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -2134,7 +2134,14 @@ try {{ .envs(envs) .envs(reserved_variables) .envs( - get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn).await?, + get_proxy_envs_for_lang( + &ScriptLang::Bun, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ) .envs(common_bun_proc_envs) .env("PATH", PATH_ENV.as_str()) @@ -2154,8 +2161,14 @@ try {{ .envs(envs) .envs(reserved_variables) .envs( - get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn) - .await?, + get_proxy_envs_for_lang( + &ScriptLang::Bun, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ) .envs(common_bun_proc_envs) .stdin(Stdio::null()) @@ -2188,8 +2201,14 @@ try {{ .envs(envs) .envs(reserved_variables) .envs( - get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn) - .await?, + get_proxy_envs_for_lang( + &ScriptLang::Bun, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ) .envs(common_bun_proc_envs) .stdin(Stdio::null()) diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index e6f484a8ac..e00b8ac5da 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -616,8 +616,14 @@ pub async fn handle_csharp_job( .envs(envs) .envs(reserved_variables) .envs( - get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn) - .await?, + get_proxy_envs_for_lang( + &ScriptLang::CSharp, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) @@ -652,8 +658,14 @@ pub async fn handle_csharp_job( .envs(envs) .envs(reserved_variables) .envs( - get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn) - .await?, + get_proxy_envs_for_lang( + &ScriptLang::CSharp, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 3cc9a5ba1d..207cf5aba4 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -20,7 +20,9 @@ use windmill_common::client::AuthedClient; use windmill_common::worker::TypeScriptAnnotations; use tokio::{fs::File, io::AsyncReadExt, process::Command}; -use windmill_common::{error::Result, scripts::ScriptLang, worker::write_file, BASE_URL}; +use windmill_common::{ + error::Result, jobs::JobKind, scripts::ScriptLang, worker::write_file, BASE_URL, +}; use windmill_common::{ error::{self}, worker::Connection, @@ -66,6 +68,7 @@ lazy_static::lazy_static! { async fn get_common_deno_proc_envs( token: &str, base_internal_url: &str, + job_kind: JobKind, job_id: &Uuid, w_id: &str, conn: Option<&Connection>, @@ -141,7 +144,7 @@ async fn get_common_deno_proc_envs( // Add proxy envs (including OTEL tracing proxy if enabled for deno) if let Some(conn) = conn { - for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno, job_id, w_id, conn) + for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno, job_kind, job_id, w_id, conn) .await .unwrap_or_default() { @@ -177,7 +180,8 @@ pub async fn generate_deno_lock( write_file(job_dir, "import_map.json", &import_map)?; write_file(job_dir, "empty.ts", "")?; - let deno_envs = get_common_deno_proc_envs("", base_internal_url, job_id, w_id, db).await; + let deno_envs = + get_common_deno_proc_envs("", base_internal_url, JobKind::Script, job_id, w_id, db).await; let mut child_cmd = Command::new(DENO_PATH.as_str()); child_cmd @@ -408,6 +412,7 @@ try {{ let mut common_deno_proc_envs = get_common_deno_proc_envs( &client.token, base_internal_url, + job.kind, &job.id, &job.workspace_id, Some(conn), @@ -732,6 +737,7 @@ pub async fn start_worker( let common_deno_proc_envs = get_common_deno_proc_envs( &token, base_internal_url, + JobKind::Script, &Uuid::nil(), w_id, Some(&db.into()), diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index d57b757e40..10639e5d5d 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -358,7 +358,16 @@ func Run(req Req) (interface{{}}, error){{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?) + .envs( + get_proxy_envs_for_lang( + &ScriptLang::Go, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -379,7 +388,16 @@ func Run(req Req) (interface{{}}, error){{ .env_clear() .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?) + .envs( + get_proxy_envs_for_lang( + &ScriptLang::Go, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, + ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index 8a6adf4a80..c79f8ba968 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -268,7 +268,16 @@ async fn run<'a>( .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?) + .envs( + get_proxy_envs_for_lang( + &ScriptLang::Nu, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, + ) .args(vec![ "--config", "run.config.proto", @@ -307,7 +316,16 @@ async fn run<'a>( .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) - .envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?) + .envs( + get_proxy_envs_for_lang( + &ScriptLang::Nu, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, + ) // TODO(v1): // "--plugins", // &format!( diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index de8a38effb..e00ae24905 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -957,8 +957,14 @@ mount {{ // inject PYTHONPATH here - for some reason I had to do it in nsjail conf .envs(reserved_variables) .envs( - get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn) - .await?, + get_proxy_envs_for_lang( + &ScriptLang::Python3, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) @@ -986,8 +992,14 @@ mount {{ .envs(envs) .envs(reserved_variables) .envs( - get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn) - .await?, + get_proxy_envs_for_lang( + &ScriptLang::Python3, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) diff --git a/backend/windmill-worker/src/r_executor.rs b/backend/windmill-worker/src/r_executor.rs index fa8a1809b1..70e188dc46 100644 --- a/backend/windmill-worker/src/r_executor.rs +++ b/backend/windmill-worker/src/r_executor.rs @@ -591,8 +591,14 @@ async fn run<'a>( .envs(reserved_variables) .envs(R_PROXY_ENVS.clone()) .envs( - get_proxy_envs_for_lang(&ScriptLang::Rlang, &job.id, &job.workspace_id, conn) - .await?, + get_proxy_envs_for_lang( + &ScriptLang::Rlang, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ); if let Some(lp) = lib_path { cmd.env("R_LIBS_USER", r_libs_user(lp)); @@ -632,8 +638,14 @@ async fn run<'a>( .envs(reserved_variables) .envs(R_PROXY_ENVS.clone()) .envs( - get_proxy_envs_for_lang(&ScriptLang::Rlang, &job.id, &job.workspace_id, conn) - .await?, + get_proxy_envs_for_lang( + &ScriptLang::Rlang, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ) .envs(envs); if let Some(lp) = lib_path { diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index d94864766b..aff842eb9f 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -29,7 +29,9 @@ use crate::{ get_proxy_envs_for_lang, handle_child::{self}, is_sandboxing_enabled, read_ee_registry_url_list_with_workspace_override, - universal_pkg_installer::{par_install_language_dependencies_seq, InstallDeps, RequiredDependency}, + universal_pkg_installer::{ + par_install_language_dependencies_seq, InstallDeps, RequiredDependency, + }, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUBY_CACHE_DIR, RUBY_REPOS, TRACING_PROXY_CA_CERT_PATH, }; @@ -820,8 +822,14 @@ mount {{ .envs(reserved_variables) .envs(RUBY_PROXY_ENVS.clone()) .envs( - get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn) - .await?, + get_proxy_envs_for_lang( + &ScriptLang::Ruby, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ) .args(vec![ "--config", @@ -862,8 +870,14 @@ mount {{ .envs(reserved_variables) .envs(RUBY_PROXY_ENVS.clone()) .envs( - get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn) - .await?, + get_proxy_envs_for_lang( + &ScriptLang::Ruby, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ) .envs(envs); diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index b65407bc6d..96470f20d9 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -708,8 +708,14 @@ pub async fn handle_rust_job( .envs(envs) .envs(reserved_variables) .envs( - get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn) - .await?, + get_proxy_envs_for_lang( + &ScriptLang::Rust, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) @@ -727,8 +733,14 @@ pub async fn handle_rust_job( .envs(envs) .envs(reserved_variables) .envs( - get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn) - .await?, + get_proxy_envs_for_lang( + &ScriptLang::Rust, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?, ) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 04f3cfdc59..21c4648569 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -916,8 +916,14 @@ pub fn get_otel_context_envs(job_id: &uuid::Uuid) -> Vec<(&'static str, String)> /// Get proxy environment variables for job execution for a specific language. /// When OTEL tracing proxy is enabled for this language, routes all traffic through the proxy. /// Otherwise, uses the standard HTTP_PROXY/HTTPS_PROXY from environment. +/// +/// Deployment callback jobs (git sync) always bypass the MITM tracing proxy and use the +/// stock corporate proxy. Routing git's HTTPS through the local MITM breaks TLS for +/// GitHub/GitLab in chained-upstream-proxy setups, and we don't need HTTP spans for the +/// system git sync script anyway. pub async fn get_proxy_envs_for_lang( lang: &ScriptLang, + job_kind: JobKind, job_id: &uuid::Uuid, w_id: &str, conn: &Connection, @@ -925,14 +931,16 @@ pub async fn get_proxy_envs_for_lang( #[allow(unused_mut)] let mut envs; #[cfg(all(feature = "private", feature = "enterprise"))] - if is_otel_tracing_proxy_enabled_for_lang(lang).await { + if !matches!(job_kind, JobKind::DeploymentCallback) + && is_otel_tracing_proxy_enabled_for_lang(lang).await + { envs = get_otel_tracing_proxy_envs(job_id, w_id, conn).await?; } else { envs = PROXY_ENVS.clone(); } #[cfg(not(all(feature = "private", feature = "enterprise")))] { - let _ = (lang, w_id, conn); + let _ = (lang, job_kind, w_id, conn); envs = PROXY_ENVS.clone(); } envs.extend(get_otel_context_envs(job_id));