feat: worker group toggle to keep the worker's proxy out of jobs (#11958)

* feat: worker group setting to strip env vars from the nsjail sandbox

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: apply the nsjail env denylist to profile envar directives too

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* refactor: drop the nsjail env denylist spawn hook

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* feat: replace the nsjail env denylist with a worker group toggle for the job proxy

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: honor the job proxy toggle in duckdb and drop the env var fallback

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: pre-install duckdb extensions with the proxy when jobs run without it

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* chore: update ee-repo-ref to 5cfc0dc6760ea7f631fc98ea1edd39adfc55bf11

This commit updates the EE repository reference after PR #914 was merged in windmill-ee-private.

Previous ee-repo-ref: 8394f2d1b67765ba854205836185342e62df841e

New ee-repo-ref: 5cfc0dc6760ea7f631fc98ea1edd39adfc55bf11

Automated by sync-ee-ref workflow.

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>

backend/ee-repo-ref.txt

@@ -1 +1 @@
-06b2eb32cd7d40fd93078285d8b6150648384602
+5cfc0dc6760ea7f631fc98ea1edd39adfc55bf11

backend/src/monitor.rs

@@ -441,6 +441,7 @@ pub async fn initial_load(
                     cache_clear: None,
                     additional_python_paths: None,
                     pip_local_dependencies: None,
+                    disable_job_proxy: false,
                     native_mode,
                     // an agent worker never reads its group's config, only its token
                     object_store_cache_config: None,

backend/windmill-api-configs/src/lib.rs

@@ -224,13 +224,15 @@ async fn update_config(
 
     #[cfg(not(feature = "enterprise"))]
     let config = if name.starts_with("worker__") {
-        // In CE, only allow setting worker_tags, cache_clear, init_bash, native_mode and paused
+        // In CE, only allow setting worker_tags, cache_clear, init_bash, native_mode, paused and
+        // disable_job_proxy
         serde_json::json!({
             "worker_tags": config.get("worker_tags"),
             "cache_clear": config.get("cache_clear"),
             "init_bash": config.get("init_bash"),
             "native_mode": config.get("native_mode"),
-            "paused": config.get("paused")
+            "paused": config.get("paused"),
+            "disable_job_proxy": config.get("disable_job_proxy")
         })
     } else {
         config

backend/windmill-common/src/instance_config.rs

@@ -1006,6 +1006,8 @@ pub struct WorkerGroupConfig {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub env_vars_allowlist: Option<Vec<String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    pub disable_job_proxy: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub min_alive_workers_alert_threshold: Option<u32>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub autoscaling: Option<AutoscalingConfig>,

backend/windmill-common/src/worker.rs

@@ -484,6 +484,7 @@ lazy_static::lazy_static! {
         additional_python_paths: Default::default(),
         pip_local_dependencies: Default::default(),
         env_vars: Default::default(),
+        disable_job_proxy: false,
         native_mode: false,
         object_store_cache_config: Default::default(),
         paused: false,
@@ -2767,6 +2768,7 @@ pub async fn load_worker_config(
             .additional_python_paths
             .or_else(|| load_additional_python_paths_from_env()),
         env_vars: resolved_env_vars,
+        disable_job_proxy: config.disable_job_proxy.unwrap_or(false),
         native_mode,
         object_store_cache_config: config.object_store_cache_config,
         paused: config.paused.unwrap_or(false),
@@ -2945,6 +2947,7 @@ pub struct WorkerConfigOpt {
     pub pip_local_dependencies: Option<Vec<String>>,
     pub env_vars_static: Option<HashMap<String, String>>,
     pub env_vars_allowlist: Option<Vec<String>>,
+    pub disable_job_proxy: Option<bool>,
     pub native_mode: Option<bool>,
     pub object_store_cache_config: Option<serde_json::Value>,
     pub paused: Option<bool>,
@@ -2966,6 +2969,7 @@ impl Default for WorkerConfigOpt {
             pip_local_dependencies: Default::default(),
             env_vars_static: Default::default(),
             env_vars_allowlist: Default::default(),
+            disable_job_proxy: Default::default(),
             native_mode: Default::default(),
             object_store_cache_config: Default::default(),
             paused: Default::default(),
@@ -2987,6 +2991,8 @@ pub struct WorkerConfig {
     pub additional_python_paths: Option<Vec<String>>,
     pub pip_local_dependencies: Option<Vec<String>>,
     pub env_vars: HashMap<String, String>,
+    /// Job runs get none of the worker's `HTTP(S)_PROXY`/`NO_PROXY`; dependency installs keep them.
+    pub disable_job_proxy: bool,
     pub native_mode: bool,
     /// Object store this group's dependency cache uses instead of the instance one, as stored
     /// in the group config. Raw JSON: `windmill-common` cannot depend on the object store crate
@@ -2999,8 +3005,8 @@ pub struct WorkerConfig {
 
 impl std::fmt::Debug for WorkerConfig {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?}, paused: {:?}, object_store_cache_config: {} }}",
-        self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::<Vec<String>>().join(", "), self.native_mode, self.paused,
+        write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, disable_job_proxy: {:?}, native_mode: {:?}, paused: {:?}, object_store_cache_config: {} }}",
+        self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::<Vec<String>>().join(", "), self.disable_job_proxy, self.native_mode, self.paused,
         // holds bucket credentials
         self.object_store_cache_config.as_ref().map(|_| "***").unwrap_or("None"))
     }

backend/windmill-worker/src/ansible_executor.rs

@@ -2098,7 +2098,7 @@ fi
         nsjail_cmd
             .current_dir(job_dir)
             .env_clear()
-            .envs(PROXY_ENVS.clone())
+            .envs(crate::job_proxy_envs())
             // inject PYTHONPATH here - for some reason I had to do it in nsjail conf
             .envs(reserved_variables)
             .env("PATH", PATH_ENV.as_str())

backend/windmill-worker/src/dbt_executor.rs

@@ -2343,7 +2343,12 @@ pub(crate) fn dbt_command(p: &PreparedProject, args: &[&str]) -> Command {
     }
     cmd.current_dir(&p.project_dir)
         .env_clear()
-        .envs(PROXY_ENVS.clone())
+        // `deps` is a dependency install, which keeps the proxy whatever the group says.
+        .envs(if args == ["deps"] {
+            PROXY_ENVS.clone()
+        } else {
+            crate::job_proxy_envs()
+        })
         .env("PATH", PATH_ENV.as_str())
         .env("TZ", TZ_ENV.as_str())
         .env("GIT_PATH", GIT_PATH.as_str())

backend/windmill-worker/src/duckdb_runner.rs

@@ -167,28 +167,37 @@ fn needed_extensions(blocks: &[String]) -> Vec<String> {
 
 static INSTALLED: LazyLock<Mutex<HashSet<String>>> = LazyLock::new(Default::default);
 
-/// Installs, outside the jail, what the job will load inside it. The runner only runs
-/// `INSTALL` here, and only of statements built above, so no script SQL and no extension
-/// code runs unjailed. Once per extension per worker process.
-async fn install_missing_extensions(blocks: &[String], job_dir: &str) -> Result<()> {
+/// Installs, ahead of the job and with the worker's proxy, what the job will load: into the
+/// jail's cache for a jailed run, or into the runner's default one for a run that does not get
+/// the proxy and so could not download them itself. The runner only runs `INSTALL` here, and
+/// only of statements built above, so no script SQL and no extension code runs unjailed. Once
+/// per extension per cache per worker process.
+async fn install_missing_extensions(
+    blocks: &[String],
+    job_dir: &str,
+    jail_cache: bool,
+) -> Result<()> {
+    let key = |s: &String| format!("{jail_cache}:{s}");
     let missing: Vec<String> = {
         let installed = INSTALLED.lock().unwrap();
         needed_extensions(blocks)
             .into_iter()
-            .filter(|s| s.starts_with("FORCE ") || !installed.contains(s))
+            .filter(|s| s.starts_with("FORCE ") || !installed.contains(&key(s)))
             .collect()
     };
     if missing.is_empty() {
         return Ok(());
     }
-    let ext_dir = extension_dir();
-    tokio::fs::create_dir_all(&ext_dir).await?;
-    let blocks: Vec<String> = std::iter::once(format!(
-        "SET extension_directory='{}';",
-        ext_dir.replace('\'', "''")
-    ))
-    .chain(missing.iter().cloned())
-    .collect();
+    let mut blocks = Vec::with_capacity(missing.len() + 1);
+    if jail_cache {
+        let ext_dir = extension_dir();
+        tokio::fs::create_dir_all(&ext_dir).await?;
+        blocks.push(format!(
+            "SET extension_directory='{}';",
+            ext_dir.replace('\'', "''")
+        ));
+    }
+    blocks.extend(missing.iter().cloned());
     let out = spawn_runner(
         RunnerMode::Plain,
         &RunnerRequest {
@@ -210,10 +219,13 @@ async fn install_missing_extensions(blocks: &[String], job_dir: &str) -> Result<
     if let Some(raw) = out.result.strip_prefix("ERROR ") {
         let msg = serde_json::from_str::<String>(raw).unwrap_or_else(|_| raw.to_string());
         return Err(Error::ExecutionErr(format!(
-            "Installing DuckDB extensions for the jailed runner failed: {msg}"
+            "Installing DuckDB extensions ahead of the job failed: {msg}"
         )));
     }
-    INSTALLED.lock().unwrap().extend(missing.into_iter().filter(|s| !s.starts_with("FORCE ")));
+    INSTALLED
+        .lock()
+        .unwrap()
+        .extend(missing.iter().filter(|s| !s.starts_with("FORCE ")).map(key));
     Ok(())
 }
 
@@ -289,10 +301,10 @@ pub(crate) async fn run(
     let spill_dir = format!("{}/spill", work_dir(job_dir));
     tokio::fs::create_dir_all(&spill_dir).await?;
     let req = RunnerRequest { temp_directory: &spill_dir, ..*req };
-    if mode != RunnerMode::Jailed {
+    if mode != RunnerMode::Jailed && crate::passes_proxy_to_jobs() {
         return spawn_runner(mode, &req, job_dir).await;
     }
-    install_missing_extensions(req.blocks, job_dir).await?;
+    install_missing_extensions(req.blocks, job_dir, mode == RunnerMode::Jailed).await?;
     let blocks = unforce_installs(req.blocks);
     spawn_runner(mode, &RunnerRequest { blocks: &blocks, ..req }, job_dir).await
 }
@@ -438,19 +450,24 @@ async fn spawn_runner(
     if let Some(cap) = &spill_cap {
         cmd.env("DUCKDB_MAX_TEMP_DIRECTORY_SIZE", cap);
     }
-    for var in [
-        "GOOGLE_APPLICATION_CREDENTIALS",
-        "HTTP_PROXY",
-        "HTTPS_PROXY",
-        "NO_PROXY",
-        "ALL_PROXY",
-        "http_proxy",
-        "https_proxy",
-        "no_proxy",
-        "all_proxy",
-    ] {
-        if let Ok(v) = std::env::var(var) {
-            cmd.env(var, v);
+    if let Ok(v) = std::env::var("GOOGLE_APPLICATION_CREDENTIALS") {
+        cmd.env("GOOGLE_APPLICATION_CREDENTIALS", v);
+    }
+    // An `Install` request only fetches extensions, a dependency install that keeps the proxy.
+    if matches!(req.mode, RequestMode::Install) || crate::passes_proxy_to_jobs() {
+        for var in [
+            "HTTP_PROXY",
+            "HTTPS_PROXY",
+            "NO_PROXY",
+            "ALL_PROXY",
+            "http_proxy",
+            "https_proxy",
+            "no_proxy",
+            "all_proxy",
+        ] {
+            if let Ok(v) = std::env::var(var) {
+                cmd.env(var, v);
+            }
         }
     }
 

backend/windmill-worker/src/java_executor.rs

@@ -252,7 +252,7 @@ pub async fn resolve<'a>(
 
         // Configure proxies
         {
-            let jps = parse_proxy()?;
+            let jps = parse_proxy(PROXY_ENVS.clone())?;
             if let Some(val) = jps.https_host {
                 cmd.arg(&format!("-Dhttps.proxyHost={}", val));
             }
@@ -417,7 +417,7 @@ async fn install<'a>(
                 .envs(PROXY_ENVS.clone());
             // Configure proxies
             {
-                let jps = parse_proxy()?;
+                let jps = parse_proxy(PROXY_ENVS.clone())?;
                 if let Some(val) = jps.https_host {
                     cmd.arg(&format!("-Dhttps.proxyHost={}", val));
                 }
@@ -803,7 +803,7 @@ async fn run<'a>(
         }
         // Configure proxies
         {
-            let jps = parse_proxy()?;
+            let jps = parse_proxy(crate::job_proxy_envs())?;
             if let Some(val) = jps.https_host {
                 cmd.arg(&format!("-Dhttps.proxyHost={}", val));
             }
@@ -882,7 +882,7 @@ async fn run<'a>(
         }
         // Configure proxies
         {
-            let jps = parse_proxy()?;
+            let jps = parse_proxy(crate::job_proxy_envs())?;
             if let Some(val) = jps.https_host {
                 cmd.arg(&format!("-Dhttps.proxyHost={}", val));
             }
@@ -943,9 +943,9 @@ struct JavaProxySettings {
     https_port: Option<String>,
     no_proxy: Option<String>,
 }
-fn parse_proxy() -> anyhow::Result<JavaProxySettings> {
+fn parse_proxy(proxy_envs: Vec<(&'static str, String)>) -> anyhow::Result<JavaProxySettings> {
     let mut jps = JavaProxySettings::default();
-    for (ident, mut val) in PROXY_ENVS.clone() {
+    for (ident, mut val) in proxy_envs {
         match ident {
             "HTTPS_PROXY" => {
                 if !val.contains("://") {

backend/windmill-worker/src/pwsh_executor.rs

@@ -36,7 +36,7 @@ use crate::{
     handle_child::handle_child,
     is_sandboxing_enabled, read_ee_registry_with_workspace_override, DISABLE_NUSER, HOME_ENV,
     NSJAIL_PATH, PATH_ENV, POWERSHELL_CACHE_DIR, POWERSHELL_PATH, POWERSHELL_REPO_PAT,
-    POWERSHELL_REPO_URL, PROXY_ENVS, TZ_ENV,
+    POWERSHELL_REPO_URL, TZ_ENV,
 };
 
 fn val_to_pwsh_param(v: serde_json::Value) -> String {
@@ -728,7 +728,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
         let mut cmd = Command::new(NSJAIL_PATH.as_str());
         cmd.current_dir(job_dir)
             .env_clear()
-            .envs(PROXY_ENVS.clone())
+            .envs(crate::job_proxy_envs())
             .envs(reserved_variables)
             .env("TZ", TZ_ENV.as_str())
             .env("PATH", PATH_ENV.as_str())

backend/windmill-worker/src/r_executor.rs

@@ -627,7 +627,7 @@ async fn run<'a>(
             .env("PATH", PATH_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
             .envs(&reserved_variables)
-            .envs(if no_network {
+            .envs(if no_network || !crate::passes_proxy_to_jobs() {
                 vec![]
             } else {
                 R_PROXY_ENVS.clone()
@@ -703,7 +703,11 @@ async fn run<'a>(
             .env("PATH", PATH_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
             .envs(reserved_variables)
-            .envs(R_PROXY_ENVS.clone())
+            .envs(if crate::passes_proxy_to_jobs() {
+                R_PROXY_ENVS.clone()
+            } else {
+                vec![]
+            })
             .envs(
                 get_proxy_envs_for_lang(
                     &ScriptLang::Rlang,

backend/windmill-worker/src/ruby_executor.rs

@@ -825,7 +825,11 @@ mount {{
             .env("BASE_INTERNAL_URL", base_internal_url)
             .env("RUBYLIB", rubylib.as_str())
             .envs(&reserved_variables)
-            .envs(RUBY_PROXY_ENVS.clone())
+            .envs(if crate::passes_proxy_to_jobs() {
+                RUBY_PROXY_ENVS.clone()
+            } else {
+                vec![]
+            })
             .envs(
                 get_proxy_envs_for_lang(
                     &ScriptLang::Ruby,
@@ -900,7 +904,11 @@ mount {{
             .env("RUBYLIB", rubylib.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
             .envs(reserved_variables)
-            .envs(RUBY_PROXY_ENVS.clone())
+            .envs(if crate::passes_proxy_to_jobs() {
+                RUBY_PROXY_ENVS.clone()
+            } else {
+                vec![]
+            })
             .envs(
                 get_proxy_envs_for_lang(
                     &ScriptLang::Ruby,

backend/windmill-worker/src/worker.rs

@@ -1199,6 +1199,21 @@ pub fn nsjail_clone_newnet(no_network: bool) -> &'static str {
     }
 }
 
+/// Whether job runs get the worker's proxy settings. Dependency installs always do: they read
+/// [`PROXY_ENVS`] directly, so a group that turns the proxy off for jobs still installs through it.
+pub fn passes_proxy_to_jobs() -> bool {
+    !WORKER_CONFIG.load().disable_job_proxy
+}
+
+/// [`PROXY_ENVS`] as a job run sees them; see [`passes_proxy_to_jobs`].
+pub fn job_proxy_envs() -> Vec<(&'static str, String)> {
+    if passes_proxy_to_jobs() {
+        PROXY_ENVS.clone()
+    } else {
+        vec![]
+    }
+}
+
 /// [`get_proxy_envs_for_lang`] for a run step. A `no_network` one reaches no proxy, the
 /// OTel tracing proxy included, so it gets only the trace context.
 pub async fn get_run_proxy_envs(
@@ -1372,12 +1387,12 @@ pub async fn get_proxy_envs_for_lang(
     {
         envs = get_otel_tracing_proxy_envs(job_id, w_id, conn).await?;
     } else {
-        envs = PROXY_ENVS.clone();
+        envs = job_proxy_envs();
     }
     #[cfg(not(all(feature = "private", feature = "enterprise")))]
     {
         let _ = (lang, job_kind, w_id, conn);
-        envs = PROXY_ENVS.clone();
+        envs = job_proxy_envs();
     }
     envs.extend(get_otel_context_envs(job_id));
     Ok(envs)
@@ -1399,7 +1414,7 @@ async fn get_otel_tracing_proxy_envs(
                 This job's HTTP requests will not be traced.";
             tracing::warn!("{}", reason);
             append_logs(job_id, w_id, format!("\n[warning] {reason}\n"), conn).await;
-            return Ok(PROXY_ENVS.clone());
+            return Ok(job_proxy_envs());
         }
     };
     let proxy_url = format!("http://127.0.0.1:{}", port);
@@ -5440,8 +5455,8 @@ pub fn build_envs(
         hm
     };
 
-    for (k, v) in PROXY_ENVS.iter() {
-        envs.insert(k.to_string(), v.clone());
+    for (k, v) in job_proxy_envs() {
+        envs.insert(k.to_string(), v);
     }
 
     Ok(envs)

frontend/src/lib/components/WorkerGroup.svelte

@@ -105,6 +105,7 @@
 		periodic_script_interval_seconds?: number
 		env_vars_static?: Map<string, string>
 		env_vars_allowlist?: string[]
+		disable_job_proxy?: boolean
 		additional_python_paths?: string[]
 		pip_local_dependencies?: string[]
 		min_alive_workers_alert_threshold?: number
@@ -212,6 +213,7 @@
 					init_bash?: string
 					env_vars_static?: Map<string, string>
 					env_vars_allowlist?: string[]
+					disable_job_proxy?: boolean
 					additional_python_paths?: string[]
 					pip_local_dependencies?: string[]
 					min_alive_workers_alert_threshold?: number
@@ -327,6 +329,7 @@
 		'priority_tags',
 		'additional_python_paths',
 		'pip_local_dependencies',
+		'disable_job_proxy',
 		'object_store_cache_config',
 		'serves_dev_workspaces'
 	] as const
@@ -909,6 +912,30 @@
 					disabled={!canEditConfig}
 				/>
 			</Label>
+			<div class="mt-8"></div>
+			<Label label="Worker proxy">
+				{#snippet header()}
+					<Tooltip>
+						{#snippet text()}
+							Whether jobs get the worker's HTTP_PROXY, HTTPS_PROXY and NO_PROXY. Dependency
+							installs always go through the proxy, whatever this is set to. Removing the variables
+							does not stop a job from reaching the proxy if it knows its address. Workers pick the
+							change up without restarting.
+						{/snippet}
+					</Tooltip>
+				{/snippet}
+				<Toggle
+					size="sm"
+					options={{ right: "Pass the worker's proxy to jobs" }}
+					checked={nconfig?.disable_job_proxy !== true}
+					on:change={(ev) => {
+						if (nconfig !== undefined) {
+							nconfig.disable_job_proxy = ev.detail ? undefined : true
+						}
+					}}
+					disabled={!canEditConfig}
+				/>
+			</Label>
 		{/if}
 
 		<div class="mt-8"></div>