From b21be716ff41670c86f76d7ef0bd787c65fc31ec Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 21 May 2026 07:57:45 +0000 Subject: [PATCH] feat(nsjail): optional disk-backed /tmp via instance setting --- backend/src/main.rs | 26 +++++++------ backend/src/monitor.rs | 30 ++++++++++----- .../windmill-common/src/global_settings.rs | 1 + .../windmill-common/src/instance_config.rs | 2 + .../nsjail/download.py.config.proto | 7 +--- .../nsjail/download.ruby.config.proto | 7 +--- .../nsjail/download.rust.config.proto | 7 +--- .../nsjail/run.ansible.config.proto | 7 +--- .../nsjail/run.bash.config.proto | 7 +--- .../nsjail/run.bun.config.proto | 7 +--- .../nsjail/run.csharp.config.proto | 7 +--- .../nsjail/run.go.config.proto | 7 +--- .../nsjail/run.java.config.proto | 7 +--- .../nsjail/run.nu.config.proto | 7 +--- .../nsjail/run.php.config.proto | 7 +--- .../nsjail/run.powershell.config.proto | 7 +--- .../nsjail/run.python3.config.proto | 7 +--- .../windmill-worker/nsjail/run.r.config.proto | 7 +--- .../nsjail/run.ruby.config.proto | 7 +--- .../nsjail/run.rust.config.proto | 7 +--- .../windmill-worker/src/ansible_executor.rs | 6 +-- backend/windmill-worker/src/bash_executor.rs | 6 +-- backend/windmill-worker/src/bun_executor.rs | 6 +-- backend/windmill-worker/src/common.rs | 38 ++++++++++++++++++- .../windmill-worker/src/csharp_executor.rs | 6 +-- backend/windmill-worker/src/go_executor.rs | 6 +-- backend/windmill-worker/src/java_executor.rs | 6 +-- backend/windmill-worker/src/nu_executor.rs | 6 +-- backend/windmill-worker/src/php_executor.rs | 6 +-- backend/windmill-worker/src/pwsh_executor.rs | 6 +-- .../windmill-worker/src/python_executor.rs | 10 ++--- backend/windmill-worker/src/r_executor.rs | 6 +-- backend/windmill-worker/src/ruby_executor.rs | 11 ++++-- backend/windmill-worker/src/rust_executor.rs | 10 ++--- backend/windmill-worker/src/worker.rs | 5 +++ .../src/lib/components/instanceSettings.ts | 10 ++++- 36 files changed, 152 insertions(+), 163 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 89293c379a..97e0375018 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -51,14 +51,15 @@ use windmill_common::{ JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, - NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NUGET_CONFIG_SETTING, - OAUTH_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, - POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, - REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RESTART_COORDINATION_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, - SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, - TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, - UV_INDEX_STRATEGY_SETTING, WORKSPACE_REGISTRIES_SETTING, + NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_DISK_BACKED_SETTING, + NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, + PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, + PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING, + RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, + SCIM_TOKEN_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, + TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, + WORKSPACE_REGISTRIES_SETTING, }, scripts::ScriptLang, stats_oss::schedule_stats, @@ -127,10 +128,10 @@ use crate::monitor::{ reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_instance_events_webhook_setting, reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, - reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmpfs_size_setting, - reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, - reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config, - reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, + reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_disk_backed_setting, + reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting, + reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting, + reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, reload_uv_index_strategy_setting, reload_worker_config, MonitorIteration, }; @@ -1784,6 +1785,7 @@ async fn process_notify_event( JOB_DEFAULT_TIMEOUT_SECS_SETTING => reload_job_default_timeout_setting(conn).await, JOB_ISOLATION_SETTING => reload_job_isolation_setting(conn).await, NSJAIL_TMPFS_SIZE_MB_SETTING => reload_nsjail_tmpfs_size_setting(conn).await, + NSJAIL_TMP_DISK_BACKED_SETTING => reload_nsjail_tmp_disk_backed_setting(conn).await, #[cfg(feature = "parquet")] OBJECT_STORE_CONFIG_SETTING => { if !disable_s3_store { diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 2d883776e0..ad971eb9d6 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -62,12 +62,13 @@ use windmill_common::{ HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, - NSJAIL_TMPFS_SIZE_MB_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING, - OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, - POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, - SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, STORE_AUDIT_LOGS_S3_SETTING, - TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, + NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_DISK_BACKED_SETTING, NUGET_CONFIG_SETTING, + OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, + POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, + REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, + STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, + UV_INDEX_STRATEGY_SETTING, }, indexer::load_indexer_config, jwt::JWT_SECRET, @@ -107,9 +108,9 @@ use windmill_worker::{ BUN_INSTALL_MIN_RELEASE_AGE, CARGO_REGISTRIES, INSTANCE_PYTHON_VERSION, JAVA_HOME_DIR, JOB_DEFAULT_TIMEOUT, JOB_ISOLATION, KEEP_JOB_DIR, MAVEN_REPOS, MAVEN_SETTINGS_XML, NO_DEFAULT_MAVEN, NPMRC, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE, NSJAIL_TMPFS_SIZE_MB, - NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, - POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY, - WORKSPACE_REGISTRIES, + NSJAIL_TMP_DISK_BACKED, NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, + PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER, + UV_INDEX_STRATEGY, WORKSPACE_REGISTRIES, }; #[cfg(feature = "parquet")] @@ -386,6 +387,7 @@ pub async fn initial_load( reload_job_default_timeout_setting(&conn).await; reload_job_isolation_setting(&conn).await; reload_nsjail_tmpfs_size_setting(&conn).await; + reload_nsjail_tmp_disk_backed_setting(&conn).await; reload_extra_pip_index_url_setting(&conn).await; reload_pip_index_url_setting(&conn).await; reload_uv_index_strategy_setting(&conn).await; @@ -1897,6 +1899,16 @@ pub async fn reload_nsjail_tmpfs_size_setting(conn: &Connection) { .await; } +pub async fn reload_nsjail_tmp_disk_backed_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + NSJAIL_TMP_DISK_BACKED_SETTING, + "NSJAIL_TMP_DISK_BACKED", + NSJAIL_TMP_DISK_BACKED.clone(), + ) + .await; +} + pub async fn reload_job_isolation_setting(conn: &Connection) { let value = match load_value_from_global_settings_with_conn(conn, JOB_ISOLATION_SETTING, true).await { diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 7306865982..9b2d54ed9e 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -54,6 +54,7 @@ pub const KEEP_JOB_DIR_SETTING: &str = "keep_job_dir"; pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexisting_user_for_oauth"; pub const JOB_ISOLATION_SETTING: &str = "job_isolation"; pub const NSJAIL_TMPFS_SIZE_MB_SETTING: &str = "nsjail_tmpfs_size_mb"; +pub const NSJAIL_TMP_DISK_BACKED_SETTING: &str = "nsjail_tmp_disk_backed"; pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index eca3acd456..b21e108cbd 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -223,6 +223,8 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub nsjail_tmpfs_size_mb: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub nsjail_tmp_disk_backed: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub bun_install_min_release_age: Option, #[serde(skip_serializing_if = "Option::is_none")] pub uv_exclude_newer: Option, diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto index 8880a565c7..e56ef66de0 100644 --- a/backend/windmill-worker/nsjail/download.py.config.proto +++ b/backend/windmill-worker/nsjail/download.py.config.proto @@ -86,12 +86,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { diff --git a/backend/windmill-worker/nsjail/download.ruby.config.proto b/backend/windmill-worker/nsjail/download.ruby.config.proto index 98422abb87..4d6b398fda 100644 --- a/backend/windmill-worker/nsjail/download.ruby.config.proto +++ b/backend/windmill-worker/nsjail/download.ruby.config.proto @@ -86,12 +86,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{TARGET}" diff --git a/backend/windmill-worker/nsjail/download.rust.config.proto b/backend/windmill-worker/nsjail/download.rust.config.proto index 0884034076..57e3c2d7e8 100644 --- a/backend/windmill-worker/nsjail/download.rust.config.proto +++ b/backend/windmill-worker/nsjail/download.rust.config.proto @@ -49,12 +49,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "/etc" diff --git a/backend/windmill-worker/nsjail/run.ansible.config.proto b/backend/windmill-worker/nsjail/run.ansible.config.proto index 7df486c53a..11660a5372 100644 --- a/backend/windmill-worker/nsjail/run.ansible.config.proto +++ b/backend/windmill-worker/nsjail/run.ansible.config.proto @@ -66,12 +66,7 @@ mount { is_bind: false } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{JOB_DIR}/main.yml" diff --git a/backend/windmill-worker/nsjail/run.bash.config.proto b/backend/windmill-worker/nsjail/run.bash.config.proto index 1136c8d298..899f7caa1e 100644 --- a/backend/windmill-worker/nsjail/run.bash.config.proto +++ b/backend/windmill-worker/nsjail/run.bash.config.proto @@ -68,12 +68,7 @@ mount { -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{JOB_DIR}/main.sh" diff --git a/backend/windmill-worker/nsjail/run.bun.config.proto b/backend/windmill-worker/nsjail/run.bun.config.proto index 280ef8a7f9..43e4464beb 100644 --- a/backend/windmill-worker/nsjail/run.bun.config.proto +++ b/backend/windmill-worker/nsjail/run.bun.config.proto @@ -60,12 +60,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{JOB_DIR}/package.json" diff --git a/backend/windmill-worker/nsjail/run.csharp.config.proto b/backend/windmill-worker/nsjail/run.csharp.config.proto index b58d6c330f..c624e55c43 100644 --- a/backend/windmill-worker/nsjail/run.csharp.config.proto +++ b/backend/windmill-worker/nsjail/run.csharp.config.proto @@ -57,12 +57,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { diff --git a/backend/windmill-worker/nsjail/run.go.config.proto b/backend/windmill-worker/nsjail/run.go.config.proto index 4fec8f3b2d..1e6de06c9c 100644 --- a/backend/windmill-worker/nsjail/run.go.config.proto +++ b/backend/windmill-worker/nsjail/run.go.config.proto @@ -50,12 +50,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { diff --git a/backend/windmill-worker/nsjail/run.java.config.proto b/backend/windmill-worker/nsjail/run.java.config.proto index 3b0a635b0c..9ae43f5b52 100644 --- a/backend/windmill-worker/nsjail/run.java.config.proto +++ b/backend/windmill-worker/nsjail/run.java.config.proto @@ -51,12 +51,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { diff --git a/backend/windmill-worker/nsjail/run.nu.config.proto b/backend/windmill-worker/nsjail/run.nu.config.proto index 40a5de69a1..6c517a5b39 100644 --- a/backend/windmill-worker/nsjail/run.nu.config.proto +++ b/backend/windmill-worker/nsjail/run.nu.config.proto @@ -51,12 +51,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{NU_PATH}" diff --git a/backend/windmill-worker/nsjail/run.php.config.proto b/backend/windmill-worker/nsjail/run.php.config.proto index 965ac7cd36..6910cbe1a0 100644 --- a/backend/windmill-worker/nsjail/run.php.config.proto +++ b/backend/windmill-worker/nsjail/run.php.config.proto @@ -51,12 +51,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{JOB_DIR}/main.php" diff --git a/backend/windmill-worker/nsjail/run.powershell.config.proto b/backend/windmill-worker/nsjail/run.powershell.config.proto index bab869c9c8..5fd758a813 100644 --- a/backend/windmill-worker/nsjail/run.powershell.config.proto +++ b/backend/windmill-worker/nsjail/run.powershell.config.proto @@ -64,12 +64,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{JOB_DIR}/main.ps1" diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto index dec9d6bae7..53d5a6c64d 100644 --- a/backend/windmill-worker/nsjail/run.python3.config.proto +++ b/backend/windmill-worker/nsjail/run.python3.config.proto @@ -54,12 +54,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { src: "{JOB_DIR}/{MAIN}.py" diff --git a/backend/windmill-worker/nsjail/run.r.config.proto b/backend/windmill-worker/nsjail/run.r.config.proto index 0f5d71bf5a..bc4d74bcba 100644 --- a/backend/windmill-worker/nsjail/run.r.config.proto +++ b/backend/windmill-worker/nsjail/run.r.config.proto @@ -51,12 +51,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { diff --git a/backend/windmill-worker/nsjail/run.ruby.config.proto b/backend/windmill-worker/nsjail/run.ruby.config.proto index 3b9509cea6..2527e785b6 100644 --- a/backend/windmill-worker/nsjail/run.ruby.config.proto +++ b/backend/windmill-worker/nsjail/run.ruby.config.proto @@ -51,12 +51,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { diff --git a/backend/windmill-worker/nsjail/run.rust.config.proto b/backend/windmill-worker/nsjail/run.rust.config.proto index 06da7be662..070a76e997 100644 --- a/backend/windmill-worker/nsjail/run.rust.config.proto +++ b/backend/windmill-worker/nsjail/run.rust.config.proto @@ -50,12 +50,7 @@ mount { rw: true } -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size={NSJAIL_TMPFS_SIZE}" -} +{TMP_MOUNT_BLOCK} mount { diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index f745bc1661..7b0ae3b3ea 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -30,7 +30,7 @@ use crate::{ bash_executor::BIN_BASH, common::{ build_command_with_isolation, check_executor_binary_exists, get_reserved_variables, - read_and_check_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, + read_and_check_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, transform_json, OccupancyMetrics, }, handle_child::handle_child, @@ -1457,8 +1457,8 @@ mount {{ additional_python_paths_folders.as_str(), ) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 47f100bd2d..516c75fdba 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -41,7 +41,7 @@ use crate::handle_child::run_future_with_polling_update_job_poller; use crate::{ common::{ build_args_map, build_command_with_isolation, get_reserved_variables, read_file, - read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -216,8 +216,8 @@ exit $exit_status .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 2346cb413f..1e4f4aafd6 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -16,7 +16,7 @@ use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_file_content, read_result, resolve_nsjail_timeout, - resolve_nsjail_tmpfs_size_bytes, start_child_process, write_file_binary, MaybeLock, + resolve_nsjail_tmp_mount_block, start_child_process, write_file_binary, MaybeLock, OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -2186,8 +2186,8 @@ try {{ .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 95c7063328..e45f7dfc64 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -48,7 +48,8 @@ use tokio::{io::AsyncWriteExt, time::Instant}; use crate::agent_workers::UPDATE_PING_URL; use crate::{ - JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, NSJAIL_TMPFS_SIZE_MB, PATH_ENV, + JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, NSJAIL_TMPFS_SIZE_MB, + NSJAIL_TMP_DISK_BACKED, PATH_ENV, }; use windmill_common::client::AuthedClient; @@ -1023,6 +1024,41 @@ pub async fn resolve_nsjail_tmpfs_size_bytes() -> String { } } +/// Sub-directory inside each job dir used as the disk-backed `/tmp` when +/// `nsjail_tmp_disk_backed` is enabled. Kept under `{JOB_DIR}` so existing +/// job-dir cleanup removes it for free. +pub const NSJAIL_TMP_BIND_SUBDIR: &str = "jail_tmp"; + +/// Build the nsjail `mount { ... }` block that backs `/tmp` inside the +/// sandbox. When `nsjail_tmp_disk_backed` is `Some(true)`, returns a +/// disk-backed bind mount of `{job_dir}/jail_tmp` and ensures the directory +/// exists (cleanup is handled by the worker's job_dir removal). Otherwise +/// returns the historical RAM-backed tmpfs mount sized via +/// `nsjail_tmpfs_size_mb`. +pub async fn resolve_nsjail_tmp_mount_block(job_dir: &str) -> String { + if NSJAIL_TMP_DISK_BACKED.read().await.unwrap_or(false) { + let jail_tmp = format!("{job_dir}/{NSJAIL_TMP_BIND_SUBDIR}"); + if let Err(e) = tokio::fs::create_dir_all(&jail_tmp).await { + tracing::error!( + "Failed to create nsjail disk-backed /tmp at {jail_tmp}: {e:?}; \ + nsjail will fail to start. Falling back to tmpfs for this job." + ); + let size_bytes = resolve_nsjail_tmpfs_size_bytes().await; + return format!( + "mount {{\n dst: \"/tmp\"\n fstype: \"tmpfs\"\n rw: true\n options: \"size={size_bytes}\"\n}}" + ); + } + format!( + "mount {{\n src: \"{jail_tmp}\"\n dst: \"/tmp\"\n is_bind: true\n rw: true\n}}" + ) + } else { + let size_bytes = resolve_nsjail_tmpfs_size_bytes().await; + format!( + "mount {{\n dst: \"/tmp\"\n fstype: \"tmpfs\"\n rw: true\n options: \"size={size_bytes}\"\n}}" + ) + } +} + async fn hash_args( #[allow(unused)] db: &DB, #[allow(unused)] client: &AuthedClient, diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index cfd9a544a1..70df9fb4e0 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -27,7 +27,7 @@ use windmill_queue::CanceledBy; use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, - get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, + get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -604,8 +604,8 @@ pub async fn handle_csharp_job( .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index ee08aa1b30..c10e41752c 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -22,7 +22,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, capitalize, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, handle_child::handle_child, @@ -352,8 +352,8 @@ func Run(req Req) (interface{{}}, error){{ .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index f927f921c8..35541c3ee3 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -23,7 +23,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, }, handle_child, is_sandboxing_enabled, read_ee_registry_bool_with_workspace_override, @@ -671,8 +671,8 @@ async fn run<'a>( // .replace("{CACHED_TARGET}", &shared_mount) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index d9a55c40c9..8aa0946666 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -14,7 +14,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, @@ -259,8 +259,8 @@ async fn run<'a>( .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index e0ba015951..08a92b3641 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -20,7 +20,7 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, - get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, + get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, MaybeLock, OccupancyMetrics, }, handle_child::handle_child, @@ -426,8 +426,8 @@ try {{ .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/pwsh_executor.rs b/backend/windmill-worker/src/pwsh_executor.rs index 5ec6ff6bcd..1ce2eb09b3 100644 --- a/backend/windmill-worker/src/pwsh_executor.rs +++ b/backend/windmill-worker/src/pwsh_executor.rs @@ -26,7 +26,7 @@ lazy_static::lazy_static! { use crate::{ common::{ build_args_map, build_command_with_isolation, get_reserved_variables, read_file, - read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, MaybeLock, OccupancyMetrics, }, handle_child::handle_child, @@ -683,8 +683,8 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", .replace("{SHARED_MOUNT}", shared_mount) .replace("{CACHE_DIR}", &*POWERSHELL_CACHE_DIR) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 2023192075..5b593a278e 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -146,7 +146,7 @@ use windmill_object_store::OBJECT_STORE_SETTINGS; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -1024,8 +1024,8 @@ mount {{ .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; @@ -2048,8 +2048,8 @@ async fn spawn_uv_install( .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .as_str(), )?; diff --git a/backend/windmill-worker/src/r_executor.rs b/backend/windmill-worker/src/r_executor.rs index 19959ee22c..9b2bbc6874 100644 --- a/backend/windmill-worker/src/r_executor.rs +++ b/backend/windmill-worker/src/r_executor.rs @@ -20,7 +20,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_tmpfs_size_bytes, start_child_process, OccupancyMetrics, + read_result, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -582,8 +582,8 @@ async fn run<'a>( .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), )?; diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index c82d5ad7df..7871c050c6 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -23,7 +23,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -619,7 +619,7 @@ async fn install<'a>( envs.clone(), get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?, ); - let nsjail_tmpfs_size = resolve_nsjail_tmpfs_size_bytes().await; + let nsjail_tmp_mount_block = resolve_nsjail_tmp_mount_block(&job_dir).await; par_install_language_dependencies_seq( InstallDeps::Flat(deps.clone()), "ruby", @@ -639,7 +639,7 @@ async fn install<'a>( .replace("{TARGET}", &dependency.path) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) - .replace("{NSJAIL_TMPFS_SIZE}", &nsjail_tmpfs_size) + .replace("{TMP_MOUNT_BLOCK}", &nsjail_tmp_mount_block) .replace("#{DEV}", DEV_CONF_NSJAIL), // .replace("{BUILD}", &build_dir), )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); @@ -812,7 +812,10 @@ mount {{ .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{NSJAIL_TMPFS_SIZE}", &resolve_nsjail_tmpfs_size_bytes().await) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) .replace("{TIMEOUT}", &nsjail_timeout), )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 0f404569da..5941e9db1c 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -23,7 +23,7 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, - get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, + get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, @@ -481,8 +481,8 @@ pub async fn build_rust_crate( .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{BUILD}", &build_dir), )?; @@ -706,8 +706,8 @@ pub async fn handle_rust_job( .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{SHARED_MOUNT}", shared_mount) .replace( - "{NSJAIL_TMPFS_SIZE}", - &resolve_nsjail_tmpfs_size_bytes().await, + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index ff5a06a022..defdd67b89 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -686,6 +686,11 @@ lazy_static::lazy_static! { /// When `None` (or non-positive), executors fall back to the unified /// `DEFAULT_NSJAIL_TMPFS_SIZE_BYTES` (800MB). pub static ref NSJAIL_TMPFS_SIZE_MB: Arc>> = Arc::new(RwLock::new(None)); + + /// When `Some(true)`, swap the tmpfs `/tmp` mount inside nsjail sandboxes for a + /// disk-backed bind mount on `{JOB_DIR}/jail_tmp`. Otherwise the historical + /// tmpfs behavior is preserved (RAM-backed, sized by `nsjail_tmpfs_size_mb`). + pub static ref NSJAIL_TMP_DISK_BACKED: Arc>> = Arc::new(RwLock::new(None)); } pub fn sleep_queue() -> u64 { diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 1779f09bf8..7c7d87c485 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -249,11 +249,19 @@ export const settings: Record = { label: 'Nsjail tmpfs size (MB)', key: 'nsjail_tmpfs_size_mb', description: - 'Override the size of the /tmp tmpfs mount inside the nsjail sandbox (in MB). When left empty, defaults to 800MB. Only applies when the job isolation mode is set to Nsjail.', + 'Override the size of the /tmp tmpfs mount inside the nsjail sandbox (in MB). When left empty, defaults to 800MB. Only applies when the job isolation mode is set to Nsjail and the Disk-back /tmp setting below is off.', fieldType: 'number', placeholder: '800', storage: 'setting' }, + { + label: 'Disk-back nsjail /tmp', + key: 'nsjail_tmp_disk_backed', + description: + 'When enabled, the /tmp mount inside the nsjail sandbox is a disk-backed bind mount (under each job dir) instead of a RAM-backed tmpfs. Disk backing avoids consuming worker RAM but loses the hard size cap the tmpfs provides — only rlimit_fsize remains as a per-file ceiling. Only applies when the job isolation mode is set to Nsjail.', + fieldType: 'boolean', + storage: 'setting' + }, { label: 'Default timeout', key: 'job_default_timeout',