starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-24 00:00:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: bypass OTEL MITM tracing proxy for git sync jobs (#8796)

Browse Source 
Git sync runs as a DeploymentCallback job. When the OTEL MITM tracing
proxy is enabled, all HTTP/HTTPS traffic from the script is rerouted
through a local intercepting proxy that chains to the corporate upstream
proxy. Git's HTTPS to GitHub fails in this setup (TLS interception with
chained CONNECT tunneling is fragile, and git's CA env handling diverges
from what the proxy injects), so customers see "GitHub.com URL couldn't
be reached" until they disable OTEL.

Detect DeploymentCallback jobs in get_proxy_envs_for_lang and fall back
to the stock PROXY_ENVS so git talks to the corporate proxy directly,
unmodified. The git sync script is system code; we don't need HTTP spans
for it.

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
2026-04-10 12:32:19 -04:00
committed by GitHub
parent e48c7cf448
commit 9c85565221
11 changed files with 182 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/windmill-worker/src/bash_executor.rs
+16 -4
View File
@@ -231,8 +231,14 @@ exit $exit_status
.env_clear()
.envs(reserved_variables)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn)
.await?,
get_proxy_envs_for_lang(
&ScriptLang::Bash,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
@@ -261,8 +267,14 @@ exit $exit_status
.envs(envs)
.envs(reserved_variables)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn)
.await?,
get_proxy_envs_for_lang(
&ScriptLang::Bash,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
backend/windmill-worker/src/bun_executor.rs
+24 -5
View File
@@ -2134,7 +2134,14 @@ try {{
.envs(envs)
.envs(reserved_variables)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn).await?,
get_proxy_envs_for_lang(
&ScriptLang::Bun,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.envs(common_bun_proc_envs)
.env("PATH", PATH_ENV.as_str())
@@ -2154,8 +2161,14 @@ try {{
.envs(envs)
.envs(reserved_variables)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn)
.await?,
get_proxy_envs_for_lang(
&ScriptLang::Bun,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.envs(common_bun_proc_envs)
.stdin(Stdio::null())
@@ -2188,8 +2201,14 @@ try {{
.envs(envs)
.envs(reserved_variables)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn)
.await?,
get_proxy_envs_for_lang(
&ScriptLang::Bun,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.envs(common_bun_proc_envs)
.stdin(Stdio::null())
backend/windmill-worker/src/csharp_executor.rs
+16 -4
View File
@@ -616,8 +616,14 @@ pub async fn handle_csharp_job(
.envs(envs)
.envs(reserved_variables)
.envs(
get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn)
.await?,
get_proxy_envs_for_lang(
&ScriptLang::CSharp,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
@@ -652,8 +658,14 @@ pub async fn handle_csharp_job(
.envs(envs)
.envs(reserved_variables)
.envs(
get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn)
.await?,
get_proxy_envs_for_lang(
&ScriptLang::CSharp,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
backend/windmill-worker/src/deno_executor.rs
+9 -3
View File
@@ -20,7 +20,9 @@ use windmill_common::client::AuthedClient;
use windmill_common::worker::TypeScriptAnnotations;
use tokio::{fs::File, io::AsyncReadExt, process::Command};
use windmill_common::{error::Result, scripts::ScriptLang, worker::write_file, BASE_URL};
use windmill_common::{
error::Result, jobs::JobKind, scripts::ScriptLang, worker::write_file, BASE_URL,
};
use windmill_common::{
error::{self},
worker::Connection,
@@ -66,6 +68,7 @@ lazy_static::lazy_static! {
async fn get_common_deno_proc_envs(
token: &str,
base_internal_url: &str,
job_kind: JobKind,
job_id: &Uuid,
w_id: &str,
conn: Option<&Connection>,
@@ -141,7 +144,7 @@ async fn get_common_deno_proc_envs(
// Add proxy envs (including OTEL tracing proxy if enabled for deno)
if let Some(conn) = conn {
for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno, job_id, w_id, conn)
for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno, job_kind, job_id, w_id, conn)
.await
.unwrap_or_default()
{
@@ -177,7 +180,8 @@ pub async fn generate_deno_lock(
write_file(job_dir, "import_map.json", &import_map)?;
write_file(job_dir, "empty.ts", "")?;
let deno_envs = get_common_deno_proc_envs("", base_internal_url, job_id, w_id, db).await;
let deno_envs =
get_common_deno_proc_envs("", base_internal_url, JobKind::Script, job_id, w_id, db).await;
let mut child_cmd = Command::new(DENO_PATH.as_str());
child_cmd
@@ -408,6 +412,7 @@ try {{
let mut common_deno_proc_envs = get_common_deno_proc_envs(
&client.token,
base_internal_url,
job.kind,
&job.id,
&job.workspace_id,
Some(conn),
@@ -732,6 +737,7 @@ pub async fn start_worker(
let common_deno_proc_envs = get_common_deno_proc_envs(
&token,
base_internal_url,
JobKind::Script,
&Uuid::nil(),
w_id,
Some(&db.into()),
backend/windmill-worker/src/go_executor.rs
+20 -2
View File
@@ -358,7 +358,16 @@ func Run(req Req) (interface{{}}, error){{
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?)
.envs(
get_proxy_envs_for_lang(
&ScriptLang::Go,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
@@ -379,7 +388,16 @@ func Run(req Req) (interface{{}}, error){{
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?)
.envs(
get_proxy_envs_for_lang(
&ScriptLang::Go,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
backend/windmill-worker/src/nu_executor.rs
+20 -2
View File
@@ -268,7 +268,16 @@ async fn run<'a>(
.env("BASE_INTERNAL_URL", base_internal_url)
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?)
.envs(
get_proxy_envs_for_lang(
&ScriptLang::Nu,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.args(vec![
"--config",
"run.config.proto",
@@ -307,7 +316,16 @@ async fn run<'a>(
.env("BASE_INTERNAL_URL", base_internal_url)
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?)
.envs(
get_proxy_envs_for_lang(
&ScriptLang::Nu,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
// TODO(v1):
// "--plugins",
// &format!(
backend/windmill-worker/src/python_executor.rs
+16 -4
View File
@@ -957,8 +957,14 @@ mount {{
// inject PYTHONPATH here - for some reason I had to do it in nsjail conf
.envs(reserved_variables)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn)
.await?,
get_proxy_envs_for_lang(
&ScriptLang::Python3,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
@@ -986,8 +992,14 @@ mount {{
.envs(envs)
.envs(reserved_variables)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn)
.await?,
get_proxy_envs_for_lang(
&ScriptLang::Python3,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
backend/windmill-worker/src/r_executor.rs
+16 -4
View File
@@ -591,8 +591,14 @@ async fn run<'a>(
.envs(reserved_variables)
.envs(R_PROXY_ENVS.clone())
.envs(
get_proxy_envs_for_lang(&ScriptLang::Rlang, &job.id, &job.workspace_id, conn)
.await?,
get_proxy_envs_for_lang(
&ScriptLang::Rlang,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
);
if let Some(lp) = lib_path {
cmd.env("R_LIBS_USER", r_libs_user(lp));
@@ -632,8 +638,14 @@ async fn run<'a>(
.envs(reserved_variables)
.envs(R_PROXY_ENVS.clone())
.envs(
get_proxy_envs_for_lang(&ScriptLang::Rlang, &job.id, &job.workspace_id, conn)
.await?,
get_proxy_envs_for_lang(
&ScriptLang::Rlang,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.envs(envs);
if let Some(lp) = lib_path {
backend/windmill-worker/src/ruby_executor.rs
+19 -5
View File
@@ -29,7 +29,9 @@ use crate::{
get_proxy_envs_for_lang,
handle_child::{self},
is_sandboxing_enabled, read_ee_registry_url_list_with_workspace_override,
universal_pkg_installer::{par_install_language_dependencies_seq, InstallDeps, RequiredDependency},
universal_pkg_installer::{
par_install_language_dependencies_seq, InstallDeps, RequiredDependency,
},
DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUBY_CACHE_DIR, RUBY_REPOS,
TRACING_PROXY_CA_CERT_PATH,
};
@@ -820,8 +822,14 @@ mount {{
.envs(reserved_variables)
.envs(RUBY_PROXY_ENVS.clone())
.envs(
get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn)
.await?,
get_proxy_envs_for_lang(
&ScriptLang::Ruby,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.args(vec![
"--config",
@@ -862,8 +870,14 @@ mount {{
.envs(reserved_variables)
.envs(RUBY_PROXY_ENVS.clone())
.envs(
get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn)
.await?,
get_proxy_envs_for_lang(
&ScriptLang::Ruby,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.envs(envs);
backend/windmill-worker/src/rust_executor.rs
+16 -4
View File
@@ -708,8 +708,14 @@ pub async fn handle_rust_job(
.envs(envs)
.envs(reserved_variables)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn)
.await?,
get_proxy_envs_for_lang(
&ScriptLang::Rust,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
@@ -727,8 +733,14 @@ pub async fn handle_rust_job(
.envs(envs)
.envs(reserved_variables)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn)
.await?,
get_proxy_envs_for_lang(
&ScriptLang::Rust,
job.kind,
&job.id,
&job.workspace_id,
conn,
)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
backend/windmill-worker/src/worker.rs
+10 -2
View File
@@ -916,8 +916,14 @@ pub fn get_otel_context_envs(job_id: &uuid::Uuid) -> Vec<(&'static str, String)>
/// Get proxy environment variables for job execution for a specific language.
/// When OTEL tracing proxy is enabled for this language, routes all traffic through the proxy.
/// Otherwise, uses the standard HTTP_PROXY/HTTPS_PROXY from environment.
///
/// Deployment callback jobs (git sync) always bypass the MITM tracing proxy and use the
/// stock corporate proxy. Routing git's HTTPS through the local MITM breaks TLS for
/// GitHub/GitLab in chained-upstream-proxy setups, and we don't need HTTP spans for the
/// system git sync script anyway.
pub async fn get_proxy_envs_for_lang(
lang: &ScriptLang,
job_kind: JobKind,
job_id: &uuid::Uuid,
w_id: &str,
conn: &Connection,
@@ -925,14 +931,16 @@ pub async fn get_proxy_envs_for_lang(
#[allow(unused_mut)]
let mut envs;
#[cfg(all(feature = "private", feature = "enterprise"))]
if is_otel_tracing_proxy_enabled_for_lang(lang).await {
if !matches!(job_kind, JobKind::DeploymentCallback)
&& is_otel_tracing_proxy_enabled_for_lang(lang).await
{
envs = get_otel_tracing_proxy_envs(job_id, w_id, conn).await?;
} else {
envs = PROXY_ENVS.clone();
}
#[cfg(not(all(feature = "private", feature = "enterprise")))]
{
let _ = (lang, w_id, conn);
let _ = (lang, job_kind, w_id, conn);
envs = PROXY_ENVS.clone();
}
envs.extend(get_otel_context_envs(job_id));