From 17872018cc037e699b1f6e1589d0ad05e0883cca Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 15 Jul 2026 17:52:39 +0200 Subject: [PATCH] feat(nsjail): make python/ansible rlimit_as configurable per worker (GIT-921) (#10138) nsjail caps a jailed job's virtual address space at rlimit_as (4096 MiB for python3 and ansible). JIT runtimes (Bun/JavaScriptCore, the JVM) reserve large virtual ranges up front, so a subprocess spawned from a jailed Python/Ansible job can crash against this cap even when its physical memory use is modest (e.g. the Bun-compiled claude CLI hitting JSC/pthread allocation failures). Most other language protos already run with disable_rl: true (unlimited); python3 and ansible are the outliers with an explicit rlimit_as. This exposes that cap via a per-language env var (NSJAIL_PY_RLIMIT_AS_MB, NSJAIL_ANSIBLE_RLIMIT_AS_MB) so operators can raise or lift it on a dedicated worker pool without a source patch/rebuild and without weakening the mount/PID/user-namespace isolation that provides the real security boundary. Only the address-space limit changes; cpu/fsize/nofile rlimits are untouched. Value is in MiB, or unlimited/none/inf/0 to uncap (rlimit_as_type: INF). Unset keeps the historical 4096 default. Co-authored-by: Claude Opus 4.8 (1M context) --- .../nsjail/run.ansible.config.proto | 2 +- .../nsjail/run.python3.config.proto | 2 +- .../windmill-worker/src/ansible_executor.rs | 11 ++- backend/windmill-worker/src/common.rs | 82 +++++++++++++++++++ .../windmill-worker/src/python_executor.rs | 15 ++-- backend/windmill-worker/src/worker.rs | 8 ++ 6 files changed, 110 insertions(+), 10 deletions(-) diff --git a/backend/windmill-worker/nsjail/run.ansible.config.proto b/backend/windmill-worker/nsjail/run.ansible.config.proto index 2c731e34f9..afaf066f17 100644 --- a/backend/windmill-worker/nsjail/run.ansible.config.proto +++ b/backend/windmill-worker/nsjail/run.ansible.config.proto @@ -5,7 +5,7 @@ hostname: "ansible" log_level: ERROR time_limit: {TIMEOUT} -rlimit_as: 4096 +{RLIMIT_AS} rlimit_cpu: 1000 rlimit_fsize: 1000 rlimit_nofile: 10000 diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto index 53d5a6c64d..269ae1f0f8 100644 --- a/backend/windmill-worker/nsjail/run.python3.config.proto +++ b/backend/windmill-worker/nsjail/run.python3.config.proto @@ -5,7 +5,7 @@ hostname: "python" log_level: ERROR time_limit: {TIMEOUT} -rlimit_as: 4096 +{RLIMIT_AS} rlimit_cpu: 1000 rlimit_fsize: 1000 rlimit_nofile: 10000 diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 292246c62c..92bffe0cf0 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -31,13 +31,14 @@ use crate::{ bash_executor::BIN_BASH, common::{ build_command_with_isolation, check_executor_binary_exists, get_reserved_variables, - read_and_check_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, - start_child_process, transform_json, OccupancyMetrics, + read_and_check_result, render_nsjail_rlimit_as, resolve_nsjail_timeout, + resolve_nsjail_tmp_mount_block, start_child_process, transform_json, OccupancyMetrics, }, handle_child::handle_child, is_sandboxing_enabled, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, - DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, + DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_ANSIBLE_RLIMIT_AS_MB, NSJAIL_PATH, PATH_ENV, + PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -1659,6 +1660,10 @@ mount {{ job_dir, "run.config.proto", &NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT + .replace( + "{RLIMIT_AS}", + &render_nsjail_rlimit_as(NSJAIL_ANSIBLE_RLIMIT_AS_MB.as_deref(), 4096), + ) .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index a88ba67bb7..6618ac7906 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1124,6 +1124,45 @@ pub async fn resolve_nsjail_timeout( (duration.as_secs() + 15).to_string() } +/// Render the `rlimit_as` line for an nsjail run config, honoring a per-language +/// env-var override. +/// +/// nsjail caps a jailed job's virtual address space at `rlimit_as` MiB. JIT +/// runtimes (Bun/JavaScriptCore, the JVM) reserve large virtual ranges up front, +/// so a subprocess spawned from a jailed Python/Ansible job can crash against this +/// cap even when its physical memory use is modest. Lifting it lets operators run +/// such workloads on a dedicated worker pool (set the env var only there) without +/// giving up the mount/PID/user-namespace isolation that provides the real +/// security boundary. Only the address-space limit is affected; the other rlimits +/// (cpu/fsize/nofile) in the proto are untouched. +/// +/// `env_override` is the raw value of the language's `NSJAIL_*_RLIMIT_AS_MB` env var: +/// - unset/empty -> historical default (`rlimit_as: {default_mb}`) +/// - `unlimited`/`none`/`inf`/`0` -> `rlimit_as_type: INF` (address space uncapped) +/// - a positive integer (MiB) -> `rlimit_as: {n}` +pub fn render_nsjail_rlimit_as(env_override: Option<&str>, default_mb: u32) -> String { + match env_override.map(str::trim) { + None | Some("") => format!("rlimit_as: {default_mb}"), + Some(v) + if v.eq_ignore_ascii_case("unlimited") + || v.eq_ignore_ascii_case("none") + || v.eq_ignore_ascii_case("inf") + || v == "0" => + { + "rlimit_as_type: INF".to_string() + } + Some(v) => match v.parse::() { + Ok(mb) => format!("rlimit_as: {mb}"), + Err(_) => { + tracing::warn!( + "Invalid nsjail rlimit_as override {v:?}, using default {default_mb}MiB" + ); + format!("rlimit_as: {default_mb}") + } + }, + } +} + /// Default size (in bytes) of the `/tmp` tmpfs mount inside nsjail sandboxes, /// used when the `nsjail_tmpfs_size_mb` instance setting is unset. pub const DEFAULT_NSJAIL_TMPFS_SIZE_BYTES: u64 = 800_000_000; @@ -1233,6 +1272,49 @@ pub(crate) async fn resolve_nsjail_tmp_mount_block(job_dir: &str) -> String { bind_mount_block(&jail_tmp) } +#[cfg(test)] +mod nsjail_rlimit_as_tests { + use super::render_nsjail_rlimit_as; + + #[test] + fn unset_uses_default() { + assert_eq!(render_nsjail_rlimit_as(None, 4096), "rlimit_as: 4096"); + assert_eq!(render_nsjail_rlimit_as(Some(" "), 4096), "rlimit_as: 4096"); + } + + #[test] + fn numeric_override_is_used() { + assert_eq!( + render_nsjail_rlimit_as(Some("16384"), 4096), + "rlimit_as: 16384" + ); + assert_eq!( + render_nsjail_rlimit_as(Some(" 8192 "), 4096), + "rlimit_as: 8192" + ); + } + + #[test] + fn unlimited_keywords_emit_inf() { + for v in ["unlimited", "UNLIMITED", "none", "inf", "0"] { + assert_eq!( + render_nsjail_rlimit_as(Some(v), 4096), + "rlimit_as_type: INF", + "value {v:?}" + ); + } + } + + #[test] + fn invalid_falls_back_to_default() { + assert_eq!( + render_nsjail_rlimit_as(Some("abc"), 4096), + "rlimit_as: 4096" + ); + assert_eq!(render_nsjail_rlimit_as(Some("-1"), 4096), "rlimit_as: 4096"); + } +} + #[cfg(test)] mod nsjail_tmp_mount_tests { use super::*; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 9ca79c6852..a3ae036894 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -155,16 +155,17 @@ use windmill_object_store::OBJECT_STORE_SETTINGS; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, - OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL, + read_result, render_nsjail_rlimit_as, resolve_nsjail_timeout, + resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, StreamNotifier, + DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::handle_child, is_sandboxing_enabled, read_ee_registry_with_workspace_override, worker_utils::ping_job_status, - PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, - PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR, - UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY, UV_PYTHON_INSTALL_MIRROR, + PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_AVAILABLE, NSJAIL_PATH, NSJAIL_PY_RLIMIT_AS_MB, PATH_ENV, + PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, + TZ_ENV, UV_CACHE_DIR, UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY, UV_PYTHON_INSTALL_MIRROR, }; use windmill_common::client::AuthedClient; @@ -1077,6 +1078,10 @@ mount {{ job_dir, "run.config.proto", &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT + .replace( + "{RLIMIT_AS}", + &render_nsjail_rlimit_as(NSJAIL_PY_RLIMIT_AS_MB.as_deref(), 4096), + ) .replace("{JOB_DIR}", job_dir) .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index a4672145bb..69642cd4ea 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -342,6 +342,14 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); + /// Per-language override for the nsjail `rlimit_as` (virtual address space) cap. + /// Value is in MiB, or `unlimited`/`none`/`inf`/`0` to uncap. Unset keeps the + /// historical default baked into the proto. See `render_nsjail_rlimit_as`. + pub static ref NSJAIL_PY_RLIMIT_AS_MB: Option = + std::env::var("NSJAIL_PY_RLIMIT_AS_MB").ok(); + pub static ref NSJAIL_ANSIBLE_RLIMIT_AS_MB: Option = + std::env::var("NSJAIL_ANSIBLE_RLIMIT_AS_MB").ok(); + // pub static ref DISABLE_NSJAIL: bool = false; pub static ref DISABLE_NSJAIL: bool = std::env::var("DISABLE_NSJAIL") .ok()