diff --git a/backend/src/main.rs b/backend/src/main.rs index a322b716b1..ecc49c22f6 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -51,14 +51,15 @@ use windmill_common::{ JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, - NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NUGET_CONFIG_SETTING, - OAUTH_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, - POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, - REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RESTART_COORDINATION_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, - SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, - TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, - UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_REGISTRIES_SETTING, + NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, + NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, + PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, + PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING, + RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, + SCIM_TOKEN_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, + TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, + UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_REGISTRIES_SETTING, }, scripts::ScriptLang, stats_oss::schedule_stats, @@ -127,10 +128,10 @@ use crate::monitor::{ reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_instance_events_webhook_setting, reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, - reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmpfs_size_setting, - reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, - reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config, - reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, + reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, + reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting, + reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting, + reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, reload_uv_index_strategy_setting, reload_uv_python_install_mirror_setting, reload_worker_config, MonitorIteration, }; @@ -1785,6 +1786,7 @@ async fn process_notify_event( JOB_DEFAULT_TIMEOUT_SECS_SETTING => reload_job_default_timeout_setting(conn).await, JOB_ISOLATION_SETTING => reload_job_isolation_setting(conn).await, NSJAIL_TMPFS_SIZE_MB_SETTING => reload_nsjail_tmpfs_size_setting(conn).await, + NSJAIL_TMP_BACKING_SETTING => reload_nsjail_tmp_backing_setting(conn).await, #[cfg(feature = "parquet")] OBJECT_STORE_CONFIG_SETTING => { if !disable_s3_store { diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 70575aeb21..6f33e6255f 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -62,13 +62,13 @@ use windmill_common::{ HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, - NSJAIL_TMPFS_SIZE_MB_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING, - OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, - POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, - SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, STORE_AUDIT_LOGS_S3_SETTING, - TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, - UV_PYTHON_INSTALL_MIRROR_SETTING, + NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, NUGET_CONFIG_SETTING, + OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, + POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, + REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, + STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, + UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, }, indexer::load_indexer_config, jwt::JWT_SECRET, @@ -108,9 +108,9 @@ use windmill_worker::{ BUN_INSTALL_MIN_RELEASE_AGE, CARGO_REGISTRIES, INSTANCE_PYTHON_VERSION, JAVA_HOME_DIR, JOB_DEFAULT_TIMEOUT, JOB_ISOLATION, KEEP_JOB_DIR, MAVEN_REPOS, MAVEN_SETTINGS_XML, NO_DEFAULT_MAVEN, NPMRC, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE, NSJAIL_TMPFS_SIZE_MB, - NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, - POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY, - UV_PYTHON_INSTALL_MIRROR, WORKSPACE_REGISTRIES, + NSJAIL_TMP_BACKING, NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, + PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER, + UV_INDEX_STRATEGY, UV_PYTHON_INSTALL_MIRROR, WORKSPACE_REGISTRIES, }; #[cfg(feature = "parquet")] @@ -387,6 +387,7 @@ pub async fn initial_load( reload_job_default_timeout_setting(&conn).await; reload_job_isolation_setting(&conn).await; reload_nsjail_tmpfs_size_setting(&conn).await; + reload_nsjail_tmp_backing_setting(&conn).await; reload_extra_pip_index_url_setting(&conn).await; reload_pip_index_url_setting(&conn).await; reload_uv_index_strategy_setting(&conn).await; @@ -1909,6 +1910,16 @@ pub async fn reload_nsjail_tmpfs_size_setting(conn: &Connection) { .await; } +pub async fn reload_nsjail_tmp_backing_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + NSJAIL_TMP_BACKING_SETTING, + "NSJAIL_TMP_BACKING", + NSJAIL_TMP_BACKING.clone(), + ) + .await; +} + pub async fn reload_job_isolation_setting(conn: &Connection) { let value = match load_value_from_global_settings_with_conn(conn, JOB_ISOLATION_SETTING, true).await { diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 063f7d9a43..5b14d809bd 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -55,6 +55,9 @@ pub const KEEP_JOB_DIR_SETTING: &str = "keep_job_dir"; pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexisting_user_for_oauth"; pub const JOB_ISOLATION_SETTING: &str = "job_isolation"; pub const NSJAIL_TMPFS_SIZE_MB_SETTING: &str = "nsjail_tmpfs_size_mb"; +pub const NSJAIL_TMP_BACKING_SETTING: &str = "nsjail_tmp_backing"; +pub const NSJAIL_TMP_BACKING_DISK: &str = "disk"; +pub const NSJAIL_TMP_BACKING_TMPFS: &str = "tmpfs"; pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index eca3acd456..1872b52140 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -223,6 +223,8 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub nsjail_tmpfs_size_mb: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub nsjail_tmp_backing: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub bun_install_min_release_age: Option, #[serde(skip_serializing_if = "Option::is_none")] pub uv_exclude_newer: Option, diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto index 8880a565c7..e56ef66de0 100644 --- a/backend/windmill-worker/nsjail/download.py.config.proto +++ b/backend/windmill-worker/nsjail/download.py.config.proto @@ -86,12 +86,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { diff --git a/backend/windmill-worker/nsjail/download.ruby.config.proto b/backend/windmill-worker/nsjail/download.ruby.config.proto index 98422abb87..4d6b398fda 100644 --- a/backend/windmill-worker/nsjail/download.ruby.config.proto +++ b/backend/windmill-worker/nsjail/download.ruby.config.proto @@ -86,12 +86,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{TARGET}" diff --git a/backend/windmill-worker/nsjail/download.rust.config.proto b/backend/windmill-worker/nsjail/download.rust.config.proto index 0884034076..57e3c2d7e8 100644 --- a/backend/windmill-worker/nsjail/download.rust.config.proto +++ b/backend/windmill-worker/nsjail/download.rust.config.proto @@ -49,12 +49,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "/etc" diff --git a/backend/windmill-worker/nsjail/run.ansible.config.proto b/backend/windmill-worker/nsjail/run.ansible.config.proto index 7df486c53a..11660a5372 100644 --- a/backend/windmill-worker/nsjail/run.ansible.config.proto +++ b/backend/windmill-worker/nsjail/run.ansible.config.proto @@ -66,12 +66,7 @@ mount { is_bind: false } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{JOB_DIR}/main.yml" diff --git a/backend/windmill-worker/nsjail/run.bash.config.proto b/backend/windmill-worker/nsjail/run.bash.config.proto index 1136c8d298..899f7caa1e 100644 --- a/backend/windmill-worker/nsjail/run.bash.config.proto +++ b/backend/windmill-worker/nsjail/run.bash.config.proto @@ -68,12 +68,7 @@ mount { -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{JOB_DIR}/main.sh" diff --git a/backend/windmill-worker/nsjail/run.bun.config.proto b/backend/windmill-worker/nsjail/run.bun.config.proto index 280ef8a7f9..43e4464beb 100644 --- a/backend/windmill-worker/nsjail/run.bun.config.proto +++ b/backend/windmill-worker/nsjail/run.bun.config.proto @@ -60,12 +60,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{JOB_DIR}/package.json" diff --git a/backend/windmill-worker/nsjail/run.csharp.config.proto b/backend/windmill-worker/nsjail/run.csharp.config.proto index b58d6c330f..c624e55c43 100644 --- a/backend/windmill-worker/nsjail/run.csharp.config.proto +++ b/backend/windmill-worker/nsjail/run.csharp.config.proto @@ -57,12 +57,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { diff --git a/backend/windmill-worker/nsjail/run.go.config.proto b/backend/windmill-worker/nsjail/run.go.config.proto index 4fec8f3b2d..1e6de06c9c 100644 --- a/backend/windmill-worker/nsjail/run.go.config.proto +++ b/backend/windmill-worker/nsjail/run.go.config.proto @@ -50,12 +50,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { diff --git a/backend/windmill-worker/nsjail/run.java.config.proto b/backend/windmill-worker/nsjail/run.java.config.proto index 3b0a635b0c..9ae43f5b52 100644 --- a/backend/windmill-worker/nsjail/run.java.config.proto +++ b/backend/windmill-worker/nsjail/run.java.config.proto @@ -51,12 +51,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { diff --git a/backend/windmill-worker/nsjail/run.nu.config.proto b/backend/windmill-worker/nsjail/run.nu.config.proto index 40a5de69a1..6c517a5b39 100644 --- a/backend/windmill-worker/nsjail/run.nu.config.proto +++ b/backend/windmill-worker/nsjail/run.nu.config.proto @@ -51,12 +51,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{NU_PATH}" diff --git a/backend/windmill-worker/nsjail/run.php.config.proto b/backend/windmill-worker/nsjail/run.php.config.proto index 965ac7cd36..6910cbe1a0 100644 --- a/backend/windmill-worker/nsjail/run.php.config.proto +++ b/backend/windmill-worker/nsjail/run.php.config.proto @@ -51,12 +51,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{JOB_DIR}/main.php" diff --git a/backend/windmill-worker/nsjail/run.powershell.config.proto b/backend/windmill-worker/nsjail/run.powershell.config.proto index bab869c9c8..5fd758a813 100644 --- a/backend/windmill-worker/nsjail/run.powershell.config.proto +++ b/backend/windmill-worker/nsjail/run.powershell.config.proto @@ -64,12 +64,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{JOB_DIR}/main.ps1" diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto index dec9d6bae7..53d5a6c64d 100644 --- a/backend/windmill-worker/nsjail/run.python3.config.proto +++ b/backend/windmill-worker/nsjail/run.python3.config.proto @@ -54,12 +54,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{JOB_DIR}/{MAIN}.py" diff --git a/backend/windmill-worker/nsjail/run.r.config.proto b/backend/windmill-worker/nsjail/run.r.config.proto index 0f5d71bf5a..bc4d74bcba 100644 --- a/backend/windmill-worker/nsjail/run.r.config.proto +++ b/backend/windmill-worker/nsjail/run.r.config.proto @@ -51,12 +51,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { diff --git a/backend/windmill-worker/nsjail/run.ruby.config.proto b/backend/windmill-worker/nsjail/run.ruby.config.proto index 3b9509cea6..2527e785b6 100644 --- a/backend/windmill-worker/nsjail/run.ruby.config.proto +++ b/backend/windmill-worker/nsjail/run.ruby.config.proto @@ -51,12 +51,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { diff --git a/backend/windmill-worker/nsjail/run.rust.config.proto b/backend/windmill-worker/nsjail/run.rust.config.proto index 06da7be662..070a76e997 100644 --- a/backend/windmill-worker/nsjail/run.rust.config.proto +++ b/backend/windmill-worker/nsjail/run.rust.config.proto @@ -50,12 +50,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index f745bc1661..7b0ae3b3ea 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -30,7 +30,7 @@ use crate::{ bash_executor::BIN_BASH, common::{ build_command_with_isolation, check_executor_binary_exists, get_reserved_variables, - read_and_check_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, + read_and_check_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, transform_json, OccupancyMetrics, }, handle_child::handle_child, @@ -1457,8 +1457,8 @@ mount {{ additional_python_paths_folders.as_str(), ) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 47f100bd2d..516c75fdba 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -41,7 +41,7 @@ use crate::handle_child::run_future_with_polling_update_job_poller; use crate::{ common::{ build_args_map, build_command_with_isolation, get_reserved_variables, read_file, - read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -216,8 +216,8 @@ exit $exit_status .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 2346cb413f..1e4f4aafd6 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -16,7 +16,7 @@ use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_file_content, read_result, resolve_nsjail_timeout, - resolve_nsjail_tmpfs_size_bytes, start_child_process, write_file_binary, MaybeLock, + resolve_nsjail_tmp_mount_block, start_child_process, write_file_binary, MaybeLock, OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -2186,8 +2186,8 @@ try {{ .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 95c7063328..aa3c6dbe2a 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -15,6 +15,7 @@ use tokio::process::Command; use tokio::{fs::File, io::AsyncReadExt}; use windmill_common::flows::Step; +use windmill_common::global_settings::NSJAIL_TMP_BACKING_DISK; use windmill_common::variables::{build_crypt_with_key_suffix, decrypt}; use windmill_common::worker::{ to_raw_value, update_ping_for_failed_init_script_query, write_file, Connection, Ping, PingType, @@ -48,7 +49,8 @@ use tokio::{io::AsyncWriteExt, time::Instant}; use crate::agent_workers::UPDATE_PING_URL; use crate::{ - JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, NSJAIL_TMPFS_SIZE_MB, PATH_ENV, + JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, NSJAIL_TMPFS_SIZE_MB, + NSJAIL_TMP_BACKING, PATH_ENV, }; use windmill_common::client::AuthedClient; @@ -1023,6 +1025,278 @@ pub async fn resolve_nsjail_tmpfs_size_bytes() -> String { } } +/// Sub-directory inside each job dir used as the disk-backed `/tmp` when +/// `nsjail_tmp_disk_backed` is enabled. Kept under `{JOB_DIR}` so existing +/// job-dir cleanup removes it for free. +const NSJAIL_TMP_BIND_SUBDIR: &str = "jail_tmp"; + +fn tmpfs_mount_block(size_bytes: &str) -> String { + format!( + "mount {{\n dst: \"/tmp\"\n fstype: \"tmpfs\"\n rw: true\n options: \"size={size_bytes}\"\n}}" + ) +} + +fn bind_mount_block(jail_tmp: &str) -> String { + format!( + "mount {{\n src: \"{jail_tmp}\"\n dst: \"/tmp\"\n is_bind: true\n rw: true\n}}" + ) +} + +/// Build the nsjail `mount { ... }` block that backs `/tmp` inside the +/// sandbox. +/// +/// **Caller contract**: `job_dir` must be a trusted, worker-allocated job +/// directory (typically `{worker_dir}/{job_id}`). In disk-backed mode this +/// function creates `{job_dir}/jail_tmp` and bind-mounts it as `/tmp` with +/// `rw: true`. Callers must not pass user-controlled paths. +/// +/// Some executors (e.g. the bun codebase path) extract user-supplied archives +/// into `job_dir` before this resolver runs, so the resolver actively refuses +/// any pre-existing entry at `{job_dir}/jail_tmp` (including symlinks) to +/// avoid bind-mounting an attacker-controlled host directory as `/tmp`. +/// +/// When the `nsjail_tmp_backing` instance setting is `"disk"`, returns a +/// disk-backed bind mount of `{job_dir}/jail_tmp` after creating the +/// directory. If creation or the pre-existence check fails, logs an error and +/// falls back to the historical tmpfs block so the job can still start. For +/// any other value (including unset, `"tmpfs"`, or unrecognized), returns the +/// historical RAM-backed tmpfs mount sized via `nsjail_tmpfs_size_mb`. +pub(crate) async fn resolve_nsjail_tmp_mount_block(job_dir: &str) -> String { + let disk_backed = NSJAIL_TMP_BACKING + .read() + .await + .as_deref() + .map(|v| v.eq_ignore_ascii_case(NSJAIL_TMP_BACKING_DISK)) + .unwrap_or(false); + let size_bytes = resolve_nsjail_tmpfs_size_bytes().await; + if !disk_backed { + return tmpfs_mount_block(&size_bytes); + } + let jail_tmp = format!("{job_dir}/{NSJAIL_TMP_BIND_SUBDIR}"); + + // SECURITY: never bind-mount a symlinked (or otherwise non-directory) + // entry at jail_tmp. `symlink_metadata` returns the link's own metadata + // without following it, so `is_dir()` is true only for a real directory. + // User-controlled archives extracted into job_dir could otherwise plant + // `jail_tmp` as a symlink to an arbitrary host directory, which nsjail + // would then expose as a writable /tmp. + // + // A pre-existing real directory at this path is legitimate: several + // executors (python_executor, ruby_executor, rust_executor) invoke nsjail + // more than once per job_dir (e.g. dep install, then run), and the first + // invocation will have created it via the `create_dir` below. + match tokio::fs::symlink_metadata(&jail_tmp).await { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + if let Err(e) = tokio::fs::create_dir(&jail_tmp).await { + tracing::error!( + "Failed to create nsjail disk-backed /tmp at {jail_tmp}: {e:?}; \ + falling back to tmpfs for this job." + ); + return tmpfs_mount_block(&size_bytes); + } + } + Ok(meta) if meta.is_dir() => { + // Real directory left over from an earlier nsjail invocation in + // this same job_dir — safe to reuse. + } + Ok(_) => { + tracing::error!( + "Refusing to bind-mount nsjail disk-backed /tmp: {jail_tmp} \ + exists but is not a regular directory (possibly a symlink \ + planted by a user-controlled archive). Falling back to \ + RAM-backed tmpfs for this job." + ); + return tmpfs_mount_block(&size_bytes); + } + Err(e) => { + tracing::error!( + "Failed to stat nsjail disk-backed /tmp at {jail_tmp}: {e:?}; \ + falling back to tmpfs for this job." + ); + return tmpfs_mount_block(&size_bytes); + } + } + bind_mount_block(&jail_tmp) +} + +#[cfg(test)] +mod nsjail_tmp_mount_tests { + use super::*; + + #[test] + fn tmpfs_block_renders_size() { + let block = tmpfs_mount_block("800000000"); + assert!(block.contains("dst: \"/tmp\"")); + assert!(block.contains("fstype: \"tmpfs\"")); + assert!(block.contains("options: \"size=800000000\"")); + assert!(!block.contains("is_bind")); + } + + #[test] + fn bind_block_renders_source_path() { + let block = bind_mount_block("/var/lib/windmill/jobs/abc/jail_tmp"); + assert!(block.contains("src: \"/var/lib/windmill/jobs/abc/jail_tmp\"")); + assert!(block.contains("dst: \"/tmp\"")); + assert!(block.contains("is_bind: true")); + assert!(block.contains("rw: true")); + assert!(!block.contains("fstype")); + } + + /// Serializes tests that mutate the process-global `NSJAIL_TMP_BACKING` + /// so they don't race when cargo runs them in parallel. + static SETTING_GUARD: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + + async fn with_tmp_backing(value: Option, f: F) -> T + where + F: FnOnce() -> Fut, + Fut: std::future::Future, + { + let _serial = SETTING_GUARD.lock().await; + let prev = NSJAIL_TMP_BACKING.read().await.clone(); + *NSJAIL_TMP_BACKING.write().await = value; + let res = f().await; + *NSJAIL_TMP_BACKING.write().await = prev; + res + } + + #[tokio::test] + async fn tmpfs_mode_returns_tmpfs_block_for_any_job_dir() { + let block = with_tmp_backing(Some("tmpfs".to_string()), || async { + resolve_nsjail_tmp_mount_block("/anything").await + }) + .await; + assert!(block.contains("fstype: \"tmpfs\"")); + assert!(block.contains("options: \"size=")); + assert!(!block.contains("is_bind")); + } + + #[tokio::test] + async fn unset_defaults_to_tmpfs() { + let block = with_tmp_backing(None, || async { + resolve_nsjail_tmp_mount_block("/anything").await + }) + .await; + assert!(block.contains("fstype: \"tmpfs\"")); + assert!(!block.contains("is_bind")); + } + + /// Disk-backed branch: the resolver must create `{job_dir}/jail_tmp` and + /// emit a bind block pointing at it. + #[tokio::test] + async fn disk_backed_creates_jail_tmp_and_returns_bind_block() { + let tmp = tempfile::tempdir().expect("tempdir"); + let job_dir = tmp.path().to_str().expect("utf8 path").to_string(); + + let block = with_tmp_backing(Some("disk".to_string()), || async { + resolve_nsjail_tmp_mount_block(&job_dir).await + }) + .await; + + let expected_dir = format!("{job_dir}/{NSJAIL_TMP_BIND_SUBDIR}"); + assert!( + std::path::Path::new(&expected_dir).is_dir(), + "jail_tmp dir should have been created at {expected_dir}" + ); + assert!(block.contains("is_bind: true")); + assert!(block.contains(&format!("src: \"{expected_dir}\""))); + } + + /// Disk-backed branch fallback: if `create_dir_all` fails, we must emit + /// the tmpfs block instead of returning an invalid bind config. + #[tokio::test] + async fn disk_backed_falls_back_to_tmpfs_on_mkdir_error() { + // /proc is a kernel filesystem that disallows directory creation, + // so create_dir_all on a subpath returns EPERM/EACCES. + let job_dir = "/proc/win1967_should_not_exist"; + + let block = with_tmp_backing(Some("disk".to_string()), || async { + resolve_nsjail_tmp_mount_block(job_dir).await + }) + .await; + + assert!(block.contains("fstype: \"tmpfs\"")); + assert!(!block.contains("is_bind")); + } + + /// Unknown values fall through to the tmpfs branch instead of crashing. + #[tokio::test] + async fn unknown_value_defaults_to_tmpfs() { + let block = with_tmp_backing(Some("bogus".to_string()), || async { + resolve_nsjail_tmp_mount_block("/anything").await + }) + .await; + assert!(block.contains("fstype: \"tmpfs\"")); + assert!(!block.contains("is_bind")); + } + + /// Security regression: if a pre-existing symlink sits at the jail_tmp + /// path (e.g. planted by a user-controlled tarball extracted into + /// `job_dir` before the resolver runs), the resolver must refuse the + /// bind mount and fall back to tmpfs — never bind-mount the symlink + /// target into the sandbox as /tmp. + #[tokio::test] + async fn disk_backed_refuses_preexisting_symlink_at_jail_tmp() { + let tmp = tempfile::tempdir().expect("tempdir"); + let job_dir = tmp.path().to_str().expect("utf8 path").to_string(); + + // Plant a symlink at {job_dir}/jail_tmp pointing at an arbitrary host + // path. Target doesn't have to exist — what matters is that the + // resolver doesn't follow it. + let jail_tmp_path = format!("{job_dir}/{NSJAIL_TMP_BIND_SUBDIR}"); + std::os::unix::fs::symlink("/etc", &jail_tmp_path).expect("plant symlink"); + assert!(std::path::Path::new(&jail_tmp_path).is_symlink()); + + let block = with_tmp_backing(Some("disk".to_string()), || async { + resolve_nsjail_tmp_mount_block(&job_dir).await + }) + .await; + + // Fell back to tmpfs — no bind-mount of the attacker-controlled path. + assert!( + block.contains("fstype: \"tmpfs\""), + "expected tmpfs fallback, got: {block}" + ); + assert!( + !block.contains("is_bind"), + "must not emit bind block, got: {block}" + ); + assert!( + !block.contains("/etc"), + "must not leak the symlink target into the proto, got: {block}" + ); + } + + /// Sequential resolver calls in the same `job_dir` (e.g. Python uv install + /// → Python run, Ruby install → run, Rust build → run) must keep using + /// the bind mount instead of silently falling back to tmpfs on the + /// second call. The first call creates `jail_tmp`; subsequent calls see + /// it as a pre-existing real directory and must accept it. + #[tokio::test] + async fn disk_backed_reuses_jail_tmp_across_sequential_calls() { + let tmp = tempfile::tempdir().expect("tempdir"); + let job_dir = tmp.path().to_str().expect("utf8 path").to_string(); + let expected_dir = format!("{job_dir}/{NSJAIL_TMP_BIND_SUBDIR}"); + + let (first, second) = with_tmp_backing(Some("disk".to_string()), || async { + let first = resolve_nsjail_tmp_mount_block(&job_dir).await; + // Simulate an executor that completes its first nsjail invocation + // (e.g. uv install) leaving jail_tmp on disk, then invokes nsjail + // again for the main run. + assert!(std::path::Path::new(&expected_dir).is_dir()); + let second = resolve_nsjail_tmp_mount_block(&job_dir).await; + (first, second) + }) + .await; + + assert!(first.contains("is_bind: true"), "first call: {first}"); + assert!( + second.contains("is_bind: true"), + "second call regressed to tmpfs: {second}" + ); + assert!(second.contains(&format!("src: \"{expected_dir}\""))); + } +} + async fn hash_args( #[allow(unused)] db: &DB, #[allow(unused)] client: &AuthedClient, diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index cfd9a544a1..70df9fb4e0 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -27,7 +27,7 @@ use windmill_queue::CanceledBy; use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, - get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, + get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -604,8 +604,8 @@ pub async fn handle_csharp_job( .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index ee08aa1b30..c10e41752c 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -22,7 +22,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, capitalize, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, handle_child::handle_child, @@ -352,8 +352,8 @@ func Run(req Req) (interface{{}}, error){{ .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index f927f921c8..35541c3ee3 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -23,7 +23,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, }, handle_child, is_sandboxing_enabled, read_ee_registry_bool_with_workspace_override, @@ -671,8 +671,8 @@ async fn run<'a>( // .replace("{CACHED_TARGET}", &shared_mount) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index d9a55c40c9..8aa0946666 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -14,7 +14,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, @@ -259,8 +259,8 @@ async fn run<'a>( .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index e0ba015951..08a92b3641 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -20,7 +20,7 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, - get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, + get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, MaybeLock, OccupancyMetrics, }, handle_child::handle_child, @@ -426,8 +426,8 @@ try {{ .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/pwsh_executor.rs b/backend/windmill-worker/src/pwsh_executor.rs index 5ec6ff6bcd..1ce2eb09b3 100644 --- a/backend/windmill-worker/src/pwsh_executor.rs +++ b/backend/windmill-worker/src/pwsh_executor.rs @@ -26,7 +26,7 @@ lazy_static::lazy_static! { use crate::{ common::{ build_args_map, build_command_with_isolation, get_reserved_variables, read_file, - read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, MaybeLock, OccupancyMetrics, }, handle_child::handle_child, @@ -683,8 +683,8 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", .replace("{SHARED_MOUNT}", shared_mount) .replace("{CACHE_DIR}", &*POWERSHELL_CACHE_DIR) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 2930c9d339..4bec8d44bf 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -146,7 +146,7 @@ use windmill_object_store::OBJECT_STORE_SETTINGS; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -1028,8 +1028,8 @@ mount {{ .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; @@ -2056,8 +2056,8 @@ async fn spawn_uv_install( .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .as_str(), )?; diff --git a/backend/windmill-worker/src/r_executor.rs b/backend/windmill-worker/src/r_executor.rs index 19959ee22c..9b2bbc6874 100644 --- a/backend/windmill-worker/src/r_executor.rs +++ b/backend/windmill-worker/src/r_executor.rs @@ -20,7 +20,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_tmpfs_size_bytes, start_child_process, OccupancyMetrics, + read_result, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -582,8 +582,8 @@ async fn run<'a>( .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), )?; diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index c82d5ad7df..7871c050c6 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -23,7 +23,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -619,7 +619,7 @@ async fn install<'a>( envs.clone(), get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?, ); - let nsjail_tmpfs_size = resolve_nsjail_tmpfs_size_bytes().await; + let nsjail_tmp_mount_block = resolve_nsjail_tmp_mount_block(&job_dir).await; par_install_language_dependencies_seq( InstallDeps::Flat(deps.clone()), "ruby", @@ -639,7 +639,7 @@ async fn install<'a>( .replace("{TARGET}", &dependency.path) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) - .replace("{NSJAIL_TMPFS_SIZE}", &nsjail_tmpfs_size) + .replace("{TMP_MOUNT_BLOCK}", &nsjail_tmp_mount_block) .replace("#{DEV}", DEV_CONF_NSJAIL), // .replace("{BUILD}", &build_dir), )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); @@ -812,7 +812,10 @@ mount {{ .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{NSJAIL_TMPFS_SIZE}", &resolve_nsjail_tmpfs_size_bytes().await) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) .replace("{TIMEOUT}", &nsjail_timeout), )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 0f404569da..5941e9db1c 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -23,7 +23,7 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, - get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, + get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -481,8 +481,8 @@ pub async fn build_rust_crate( .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{BUILD}", &build_dir), )?; @@ -706,8 +706,8 @@ pub async fn handle_rust_job( .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{SHARED_MOUNT}", shared_mount) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index b85c35d65b..6b1216ad9d 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -687,6 +687,12 @@ lazy_static::lazy_static! { /// `DEFAULT_NSJAIL_TMPFS_SIZE_BYTES` (800MB). pub static ref NSJAIL_TMPFS_SIZE_MB: Arc>> = Arc::new(RwLock::new(None)); + /// Selects how `/tmp` is backed inside nsjail sandboxes. `Some("disk")` + /// switches to a bind mount on `{JOB_DIR}/jail_tmp` (disk-backed); any + /// other value (including `None` or `Some("tmpfs")`) keeps the historical + /// RAM-backed tmpfs sized by `nsjail_tmpfs_size_mb`. + pub static ref NSJAIL_TMP_BACKING: Arc>> = Arc::new(RwLock::new(None)); + /// Optional mirror URL for `uv python install`. Wires to the `UV_PYTHON_INSTALL_MIRROR` /// env var when forwarded to uv. Can be set via the `UV_PYTHON_INSTALL_MIRROR` env var /// or the `uv_python_install_mirror` instance setting. diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index dff8591322..834ef0b96c 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -72,6 +72,20 @@ return false } } + // Hide the nsjail-only settings only when isolation is *explicitly* a + // non-nsjail mode. When `job_isolation` is unset, nsjail may still be + // enabled via the legacy env-driven path (`DISABLE_NSJAIL=false`), so + // keep the controls reachable. + if (setting == 'nsjail_tmp_backing' || setting == 'nsjail_tmpfs_size_mb') { + const isolation = values['job_isolation'] + if (isolation === 'none' || isolation === 'unshare') { + return false + } + } + // The tmpfs size knob is meaningless when /tmp is disk-backed. + if (setting == 'nsjail_tmpfs_size_mb' && values['nsjail_tmp_backing'] === 'disk') { + return false + } return true } diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index d8cf5901ec..a127027dac 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -245,11 +245,25 @@ export const settings: Record = { } ] }, + { + label: 'Nsjail /tmp backing', + key: 'nsjail_tmp_backing', + fieldType: 'select', + description: + 'How /tmp is backed inside the nsjail sandbox. RAM (tmpfs) is the default — fast, with a hard size cap from Nsjail tmpfs size, but consumes worker memory. Disk (bind mount) uses a per-job directory on the worker disk — no RAM cost, but the only remaining per-file ceiling is rlimit_fsize (~1GB for python/ansible, unbounded for most other languages because they set disable_rl: true); pair with host disk monitoring or quotas.', + storage: 'setting', + placeholder: 'tmpfs', + defaultValue: () => 'tmpfs', + select_items: [ + { label: 'RAM (tmpfs) — default', value: 'tmpfs' }, + { label: 'Disk (bind mount)', value: 'disk' } + ] + }, { label: 'Nsjail tmpfs size (MB)', key: 'nsjail_tmpfs_size_mb', description: - 'Override the size of the /tmp tmpfs mount inside the nsjail sandbox (in MB). When left empty, defaults to 800MB. Only applies when the job isolation mode is set to Nsjail.', + 'Override the size of the /tmp tmpfs mount inside the nsjail sandbox (in MB). When left empty, defaults to 800MB. Only applies when Nsjail /tmp backing is RAM (tmpfs).', fieldType: 'number', placeholder: '800', storage: 'setting'