feat(nsjail): optional disk-backed /tmp via instance setting (#9272)

* feat(nsjail): optional disk-backed /tmp via instance setting

* test(nsjail): unit-test tmp mount resolver and narrow visibility

* refactor(nsjail): switch tmp backing to select + conditional UI

* ui(nsjail): make tmpfs the visible default in /tmp backing select

* fix(nsjail): refuse preexisting jail_tmp to block symlink escape

* fix(nsjail): allow jail_tmp reuse on sequential nsjail calls

Codex flagged that python/ruby/rust executors invoke nsjail twice per
job_dir (install then run). The previous resolver treated any preexisting
jail_tmp as hostile and silently fell back to tmpfs on the second call,
so disk-backed mode never reached the main script run for those langs.

Use symlink_metadata().is_dir() to distinguish a real directory left by
an earlier call in the same job_dir (safe to reuse) from a symlink or
other entity (still refused, as the codebase-tar escape requires).

Also loosen the frontend visibility predicate: only hide nsjail settings
when job_isolation is explicitly 'none' or 'unshare', so deployments
that enable nsjail via DISABLE_NSJAIL=false with no DB setting can
still see the controls.
This commit is contained in:
Ruben Fiszel
2026-05-21 15:34:49 +00:00
committed by GitHub
parent d0ee697e8b
commit b656dc6cdc
37 changed files with 413 additions and 164 deletions
+14 -12
View File
@@ -51,14 +51,15 @@ use windmill_common::{
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING,
KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING,
NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NUGET_CONFIG_SETTING,
OAUTH_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING,
POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING,
REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
RESTART_COORDINATION_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING,
TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING,
UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_REGISTRIES_SETTING,
NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING,
NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING,
PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING,
PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING,
RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING,
SCIM_TOKEN_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING,
TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING,
UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_REGISTRIES_SETTING,
},
scripts::ScriptLang,
stats_oss::schedule_stats,
@@ -127,10 +128,10 @@ use crate::monitor::{
reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting,
reload_hub_base_url_setting, reload_instance_events_webhook_setting,
reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting,
reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmpfs_size_setting,
reload_otel_tracing_proxy_setting, reload_pip_index_url_setting,
reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config,
reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting,
reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting,
reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting,
reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting,
reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting,
reload_uv_index_strategy_setting, reload_uv_python_install_mirror_setting,
reload_worker_config, MonitorIteration,
};
@@ -1785,6 +1786,7 @@ async fn process_notify_event(
JOB_DEFAULT_TIMEOUT_SECS_SETTING => reload_job_default_timeout_setting(conn).await,
JOB_ISOLATION_SETTING => reload_job_isolation_setting(conn).await,
NSJAIL_TMPFS_SIZE_MB_SETTING => reload_nsjail_tmpfs_size_setting(conn).await,
NSJAIL_TMP_BACKING_SETTING => reload_nsjail_tmp_backing_setting(conn).await,
#[cfg(feature = "parquet")]
OBJECT_STORE_CONFIG_SETTING => {
if !disable_s3_store {
+21 -10
View File
@@ -62,13 +62,13 @@ use windmill_common::{
HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING,
NSJAIL_TMPFS_SIZE_MB_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING,
OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING,
POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, STORE_AUDIT_LOGS_S3_SETTING,
TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING,
UV_PYTHON_INSTALL_MIRROR_SETTING,
NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, NUGET_CONFIG_SETTING,
OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING,
POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING,
REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING,
UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING,
},
indexer::load_indexer_config,
jwt::JWT_SECRET,
@@ -108,9 +108,9 @@ use windmill_worker::{
BUN_INSTALL_MIN_RELEASE_AGE, CARGO_REGISTRIES, INSTANCE_PYTHON_VERSION, JAVA_HOME_DIR,
JOB_DEFAULT_TIMEOUT, JOB_ISOLATION, KEEP_JOB_DIR, MAVEN_REPOS, MAVEN_SETTINGS_XML,
NO_DEFAULT_MAVEN, NPMRC, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE, NSJAIL_TMPFS_SIZE_MB,
NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL,
POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY,
UV_PYTHON_INSTALL_MIRROR, WORKSPACE_REGISTRIES,
NSJAIL_TMP_BACKING, NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL,
PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER,
UV_INDEX_STRATEGY, UV_PYTHON_INSTALL_MIRROR, WORKSPACE_REGISTRIES,
};
#[cfg(feature = "parquet")]
@@ -387,6 +387,7 @@ pub async fn initial_load(
reload_job_default_timeout_setting(&conn).await;
reload_job_isolation_setting(&conn).await;
reload_nsjail_tmpfs_size_setting(&conn).await;
reload_nsjail_tmp_backing_setting(&conn).await;
reload_extra_pip_index_url_setting(&conn).await;
reload_pip_index_url_setting(&conn).await;
reload_uv_index_strategy_setting(&conn).await;
@@ -1909,6 +1910,16 @@ pub async fn reload_nsjail_tmpfs_size_setting(conn: &Connection) {
.await;
}
pub async fn reload_nsjail_tmp_backing_setting(conn: &Connection) {
reload_option_setting_with_tracing(
conn,
NSJAIL_TMP_BACKING_SETTING,
"NSJAIL_TMP_BACKING",
NSJAIL_TMP_BACKING.clone(),
)
.await;
}
pub async fn reload_job_isolation_setting(conn: &Connection) {
let value =
match load_value_from_global_settings_with_conn(conn, JOB_ISOLATION_SETTING, true).await {
@@ -55,6 +55,9 @@ pub const KEEP_JOB_DIR_SETTING: &str = "keep_job_dir";
pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexisting_user_for_oauth";
pub const JOB_ISOLATION_SETTING: &str = "job_isolation";
pub const NSJAIL_TMPFS_SIZE_MB_SETTING: &str = "nsjail_tmpfs_size_mb";
pub const NSJAIL_TMP_BACKING_SETTING: &str = "nsjail_tmp_backing";
pub const NSJAIL_TMP_BACKING_DISK: &str = "disk";
pub const NSJAIL_TMP_BACKING_TMPFS: &str = "tmpfs";
pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config";
pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret";
@@ -223,6 +223,8 @@ pub struct GlobalSettings {
#[serde(skip_serializing_if = "Option::is_none")]
pub nsjail_tmpfs_size_mb: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub nsjail_tmp_backing: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub bun_install_min_release_age: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub uv_exclude_newer: Option<i64>,
@@ -86,12 +86,7 @@ mount {
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
@@ -86,12 +86,7 @@ mount {
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
src: "{TARGET}"
@@ -49,12 +49,7 @@ mount {
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
src: "/etc"
@@ -66,12 +66,7 @@ mount {
is_bind: false
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
src: "{JOB_DIR}/main.yml"
@@ -68,12 +68,7 @@ mount {
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
src: "{JOB_DIR}/main.sh"
@@ -60,12 +60,7 @@ mount {
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
src: "{JOB_DIR}/package.json"
@@ -57,12 +57,7 @@ mount {
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
@@ -50,12 +50,7 @@ mount {
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
@@ -51,12 +51,7 @@ mount {
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
@@ -51,12 +51,7 @@ mount {
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
src: "{NU_PATH}"
@@ -51,12 +51,7 @@ mount {
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
src: "{JOB_DIR}/main.php"
@@ -64,12 +64,7 @@ mount {
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
src: "{JOB_DIR}/main.ps1"
@@ -54,12 +54,7 @@ mount {
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
src: "{JOB_DIR}/{MAIN}.py"
@@ -51,12 +51,7 @@ mount {
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
@@ -51,12 +51,7 @@ mount {
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
@@ -50,12 +50,7 @@ mount {
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size={NSJAIL_TMPFS_SIZE}"
}
{TMP_MOUNT_BLOCK}
mount {
@@ -30,7 +30,7 @@ use crate::{
bash_executor::BIN_BASH,
common::{
build_command_with_isolation, check_executor_binary_exists, get_reserved_variables,
read_and_check_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes,
read_and_check_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block,
start_child_process, transform_json, OccupancyMetrics,
},
handle_child::handle_child,
@@ -1457,8 +1457,8 @@ mount {{
additional_python_paths_folders.as_str(),
)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
+3 -3
View File
@@ -41,7 +41,7 @@ use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::{
common::{
build_args_map, build_command_with_isolation, get_reserved_variables, read_file,
read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process,
read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process,
OccupancyMetrics, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
@@ -216,8 +216,8 @@ exit $exit_status
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
+3 -3
View File
@@ -16,7 +16,7 @@ use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
parse_npm_config, read_file, read_file_content, read_result, resolve_nsjail_timeout,
resolve_nsjail_tmpfs_size_bytes, start_child_process, write_file_binary, MaybeLock,
resolve_nsjail_tmp_mount_block, start_child_process, write_file_binary, MaybeLock,
OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
@@ -2186,8 +2186,8 @@ try {{
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
+275 -1
View File
@@ -15,6 +15,7 @@ use tokio::process::Command;
use tokio::{fs::File, io::AsyncReadExt};
use windmill_common::flows::Step;
use windmill_common::global_settings::NSJAIL_TMP_BACKING_DISK;
use windmill_common::variables::{build_crypt_with_key_suffix, decrypt};
use windmill_common::worker::{
to_raw_value, update_ping_for_failed_init_script_query, write_file, Connection, Ping, PingType,
@@ -48,7 +49,8 @@ use tokio::{io::AsyncWriteExt, time::Instant};
use crate::agent_workers::UPDATE_PING_URL;
use crate::{
JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, NSJAIL_TMPFS_SIZE_MB, PATH_ENV,
JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, NSJAIL_TMPFS_SIZE_MB,
NSJAIL_TMP_BACKING, PATH_ENV,
};
use windmill_common::client::AuthedClient;
@@ -1023,6 +1025,278 @@ pub async fn resolve_nsjail_tmpfs_size_bytes() -> String {
}
}
/// Sub-directory inside each job dir used as the disk-backed `/tmp` when
/// `nsjail_tmp_disk_backed` is enabled. Kept under `{JOB_DIR}` so existing
/// job-dir cleanup removes it for free.
const NSJAIL_TMP_BIND_SUBDIR: &str = "jail_tmp";
fn tmpfs_mount_block(size_bytes: &str) -> String {
format!(
"mount {{\n dst: \"/tmp\"\n fstype: \"tmpfs\"\n rw: true\n options: \"size={size_bytes}\"\n}}"
)
}
fn bind_mount_block(jail_tmp: &str) -> String {
format!(
"mount {{\n src: \"{jail_tmp}\"\n dst: \"/tmp\"\n is_bind: true\n rw: true\n}}"
)
}
/// Build the nsjail `mount { ... }` block that backs `/tmp` inside the
/// sandbox.
///
/// **Caller contract**: `job_dir` must be a trusted, worker-allocated job
/// directory (typically `{worker_dir}/{job_id}`). In disk-backed mode this
/// function creates `{job_dir}/jail_tmp` and bind-mounts it as `/tmp` with
/// `rw: true`. Callers must not pass user-controlled paths.
///
/// Some executors (e.g. the bun codebase path) extract user-supplied archives
/// into `job_dir` before this resolver runs, so the resolver actively refuses
/// any pre-existing entry at `{job_dir}/jail_tmp` (including symlinks) to
/// avoid bind-mounting an attacker-controlled host directory as `/tmp`.
///
/// When the `nsjail_tmp_backing` instance setting is `"disk"`, returns a
/// disk-backed bind mount of `{job_dir}/jail_tmp` after creating the
/// directory. If creation or the pre-existence check fails, logs an error and
/// falls back to the historical tmpfs block so the job can still start. For
/// any other value (including unset, `"tmpfs"`, or unrecognized), returns the
/// historical RAM-backed tmpfs mount sized via `nsjail_tmpfs_size_mb`.
pub(crate) async fn resolve_nsjail_tmp_mount_block(job_dir: &str) -> String {
let disk_backed = NSJAIL_TMP_BACKING
.read()
.await
.as_deref()
.map(|v| v.eq_ignore_ascii_case(NSJAIL_TMP_BACKING_DISK))
.unwrap_or(false);
let size_bytes = resolve_nsjail_tmpfs_size_bytes().await;
if !disk_backed {
return tmpfs_mount_block(&size_bytes);
}
let jail_tmp = format!("{job_dir}/{NSJAIL_TMP_BIND_SUBDIR}");
// SECURITY: never bind-mount a symlinked (or otherwise non-directory)
// entry at jail_tmp. `symlink_metadata` returns the link's own metadata
// without following it, so `is_dir()` is true only for a real directory.
// User-controlled archives extracted into job_dir could otherwise plant
// `jail_tmp` as a symlink to an arbitrary host directory, which nsjail
// would then expose as a writable /tmp.
//
// A pre-existing real directory at this path is legitimate: several
// executors (python_executor, ruby_executor, rust_executor) invoke nsjail
// more than once per job_dir (e.g. dep install, then run), and the first
// invocation will have created it via the `create_dir` below.
match tokio::fs::symlink_metadata(&jail_tmp).await {
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
if let Err(e) = tokio::fs::create_dir(&jail_tmp).await {
tracing::error!(
"Failed to create nsjail disk-backed /tmp at {jail_tmp}: {e:?}; \
falling back to tmpfs for this job."
);
return tmpfs_mount_block(&size_bytes);
}
}
Ok(meta) if meta.is_dir() => {
// Real directory left over from an earlier nsjail invocation in
// this same job_dir — safe to reuse.
}
Ok(_) => {
tracing::error!(
"Refusing to bind-mount nsjail disk-backed /tmp: {jail_tmp} \
exists but is not a regular directory (possibly a symlink \
planted by a user-controlled archive). Falling back to \
RAM-backed tmpfs for this job."
);
return tmpfs_mount_block(&size_bytes);
}
Err(e) => {
tracing::error!(
"Failed to stat nsjail disk-backed /tmp at {jail_tmp}: {e:?}; \
falling back to tmpfs for this job."
);
return tmpfs_mount_block(&size_bytes);
}
}
bind_mount_block(&jail_tmp)
}
#[cfg(test)]
mod nsjail_tmp_mount_tests {
use super::*;
#[test]
fn tmpfs_block_renders_size() {
let block = tmpfs_mount_block("800000000");
assert!(block.contains("dst: \"/tmp\""));
assert!(block.contains("fstype: \"tmpfs\""));
assert!(block.contains("options: \"size=800000000\""));
assert!(!block.contains("is_bind"));
}
#[test]
fn bind_block_renders_source_path() {
let block = bind_mount_block("/var/lib/windmill/jobs/abc/jail_tmp");
assert!(block.contains("src: \"/var/lib/windmill/jobs/abc/jail_tmp\""));
assert!(block.contains("dst: \"/tmp\""));
assert!(block.contains("is_bind: true"));
assert!(block.contains("rw: true"));
assert!(!block.contains("fstype"));
}
/// Serializes tests that mutate the process-global `NSJAIL_TMP_BACKING`
/// so they don't race when cargo runs them in parallel.
static SETTING_GUARD: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
async fn with_tmp_backing<F, Fut, T>(value: Option<String>, f: F) -> T
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = T>,
{
let _serial = SETTING_GUARD.lock().await;
let prev = NSJAIL_TMP_BACKING.read().await.clone();
*NSJAIL_TMP_BACKING.write().await = value;
let res = f().await;
*NSJAIL_TMP_BACKING.write().await = prev;
res
}
#[tokio::test]
async fn tmpfs_mode_returns_tmpfs_block_for_any_job_dir() {
let block = with_tmp_backing(Some("tmpfs".to_string()), || async {
resolve_nsjail_tmp_mount_block("/anything").await
})
.await;
assert!(block.contains("fstype: \"tmpfs\""));
assert!(block.contains("options: \"size="));
assert!(!block.contains("is_bind"));
}
#[tokio::test]
async fn unset_defaults_to_tmpfs() {
let block = with_tmp_backing(None, || async {
resolve_nsjail_tmp_mount_block("/anything").await
})
.await;
assert!(block.contains("fstype: \"tmpfs\""));
assert!(!block.contains("is_bind"));
}
/// Disk-backed branch: the resolver must create `{job_dir}/jail_tmp` and
/// emit a bind block pointing at it.
#[tokio::test]
async fn disk_backed_creates_jail_tmp_and_returns_bind_block() {
let tmp = tempfile::tempdir().expect("tempdir");
let job_dir = tmp.path().to_str().expect("utf8 path").to_string();
let block = with_tmp_backing(Some("disk".to_string()), || async {
resolve_nsjail_tmp_mount_block(&job_dir).await
})
.await;
let expected_dir = format!("{job_dir}/{NSJAIL_TMP_BIND_SUBDIR}");
assert!(
std::path::Path::new(&expected_dir).is_dir(),
"jail_tmp dir should have been created at {expected_dir}"
);
assert!(block.contains("is_bind: true"));
assert!(block.contains(&format!("src: \"{expected_dir}\"")));
}
/// Disk-backed branch fallback: if `create_dir_all` fails, we must emit
/// the tmpfs block instead of returning an invalid bind config.
#[tokio::test]
async fn disk_backed_falls_back_to_tmpfs_on_mkdir_error() {
// /proc is a kernel filesystem that disallows directory creation,
// so create_dir_all on a subpath returns EPERM/EACCES.
let job_dir = "/proc/win1967_should_not_exist";
let block = with_tmp_backing(Some("disk".to_string()), || async {
resolve_nsjail_tmp_mount_block(job_dir).await
})
.await;
assert!(block.contains("fstype: \"tmpfs\""));
assert!(!block.contains("is_bind"));
}
/// Unknown values fall through to the tmpfs branch instead of crashing.
#[tokio::test]
async fn unknown_value_defaults_to_tmpfs() {
let block = with_tmp_backing(Some("bogus".to_string()), || async {
resolve_nsjail_tmp_mount_block("/anything").await
})
.await;
assert!(block.contains("fstype: \"tmpfs\""));
assert!(!block.contains("is_bind"));
}
/// Security regression: if a pre-existing symlink sits at the jail_tmp
/// path (e.g. planted by a user-controlled tarball extracted into
/// `job_dir` before the resolver runs), the resolver must refuse the
/// bind mount and fall back to tmpfs — never bind-mount the symlink
/// target into the sandbox as /tmp.
#[tokio::test]
async fn disk_backed_refuses_preexisting_symlink_at_jail_tmp() {
let tmp = tempfile::tempdir().expect("tempdir");
let job_dir = tmp.path().to_str().expect("utf8 path").to_string();
// Plant a symlink at {job_dir}/jail_tmp pointing at an arbitrary host
// path. Target doesn't have to exist — what matters is that the
// resolver doesn't follow it.
let jail_tmp_path = format!("{job_dir}/{NSJAIL_TMP_BIND_SUBDIR}");
std::os::unix::fs::symlink("/etc", &jail_tmp_path).expect("plant symlink");
assert!(std::path::Path::new(&jail_tmp_path).is_symlink());
let block = with_tmp_backing(Some("disk".to_string()), || async {
resolve_nsjail_tmp_mount_block(&job_dir).await
})
.await;
// Fell back to tmpfs — no bind-mount of the attacker-controlled path.
assert!(
block.contains("fstype: \"tmpfs\""),
"expected tmpfs fallback, got: {block}"
);
assert!(
!block.contains("is_bind"),
"must not emit bind block, got: {block}"
);
assert!(
!block.contains("/etc"),
"must not leak the symlink target into the proto, got: {block}"
);
}
/// Sequential resolver calls in the same `job_dir` (e.g. Python uv install
/// → Python run, Ruby install → run, Rust build → run) must keep using
/// the bind mount instead of silently falling back to tmpfs on the
/// second call. The first call creates `jail_tmp`; subsequent calls see
/// it as a pre-existing real directory and must accept it.
#[tokio::test]
async fn disk_backed_reuses_jail_tmp_across_sequential_calls() {
let tmp = tempfile::tempdir().expect("tempdir");
let job_dir = tmp.path().to_str().expect("utf8 path").to_string();
let expected_dir = format!("{job_dir}/{NSJAIL_TMP_BIND_SUBDIR}");
let (first, second) = with_tmp_backing(Some("disk".to_string()), || async {
let first = resolve_nsjail_tmp_mount_block(&job_dir).await;
// Simulate an executor that completes its first nsjail invocation
// (e.g. uv install) leaving jail_tmp on disk, then invokes nsjail
// again for the main run.
assert!(std::path::Path::new(&expected_dir).is_dir());
let second = resolve_nsjail_tmp_mount_block(&job_dir).await;
(first, second)
})
.await;
assert!(first.contains("is_bind: true"), "first call: {first}");
assert!(
second.contains("is_bind: true"),
"second call regressed to tmpfs: {second}"
);
assert!(second.contains(&format!("src: \"{expected_dir}\"")));
}
}
async fn hash_args(
#[allow(unused)] db: &DB,
#[allow(unused)] client: &AuthedClient,
@@ -27,7 +27,7 @@ use windmill_queue::CanceledBy;
use crate::{
common::{
build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file,
get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes,
get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block,
start_child_process, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
@@ -604,8 +604,8 @@ pub async fn handle_csharp_job(
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
+3 -3
View File
@@ -22,7 +22,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use crate::{
common::{
build_command_with_isolation, capitalize, create_args_and_out_file, get_reserved_variables,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process,
OccupancyMetrics, DEV_CONF_NSJAIL,
},
handle_child::handle_child,
@@ -352,8 +352,8 @@ func Run(req Req) (interface{{}}, error){{
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
+3 -3
View File
@@ -23,7 +23,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process,
OccupancyMetrics,
},
handle_child, is_sandboxing_enabled, read_ee_registry_bool_with_workspace_override,
@@ -671,8 +671,8 @@ async fn run<'a>(
// .replace("{CACHED_TARGET}", &shared_mount)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
+3 -3
View File
@@ -14,7 +14,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process,
OccupancyMetrics, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang, handle_child, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH,
@@ -259,8 +259,8 @@ async fn run<'a>(
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
+3 -3
View File
@@ -20,7 +20,7 @@ use windmill_queue::{append_logs, CanceledBy};
use crate::{
common::{
build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file,
get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes,
get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block,
start_child_process, MaybeLock, OccupancyMetrics,
},
handle_child::handle_child,
@@ -426,8 +426,8 @@ try {{
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace("{SHARED_MOUNT}", shared_mount)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
+3 -3
View File
@@ -26,7 +26,7 @@ lazy_static::lazy_static! {
use crate::{
common::{
build_args_map, build_command_with_isolation, get_reserved_variables, read_file,
read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process,
read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process,
MaybeLock, OccupancyMetrics,
},
handle_child::handle_child,
@@ -683,8 +683,8 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
.replace("{SHARED_MOUNT}", shared_mount)
.replace("{CACHE_DIR}", &*POWERSHELL_CACHE_DIR)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
@@ -146,7 +146,7 @@ use windmill_object_store::OBJECT_STORE_SETTINGS;
use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process,
OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
@@ -1028,8 +1028,8 @@ mount {{
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
@@ -2056,8 +2056,8 @@ async fn spawn_uv_install(
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.as_str(),
)?;
+3 -3
View File
@@ -20,7 +20,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
read_result, resolve_nsjail_tmpfs_size_bytes, start_child_process, OccupancyMetrics,
read_result, resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics,
DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
@@ -582,8 +582,8 @@ async fn run<'a>(
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
)?;
+7 -4
View File
@@ -23,7 +23,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process,
OccupancyMetrics, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
@@ -619,7 +619,7 @@ async fn install<'a>(
envs.clone(),
get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?,
);
let nsjail_tmpfs_size = resolve_nsjail_tmpfs_size_bytes().await;
let nsjail_tmp_mount_block = resolve_nsjail_tmp_mount_block(&job_dir).await;
par_install_language_dependencies_seq(
InstallDeps::Flat(deps.clone()),
"ruby",
@@ -639,7 +639,7 @@ async fn install<'a>(
.replace("{TARGET}", &dependency.path)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("{NSJAIL_TMPFS_SIZE}", &nsjail_tmpfs_size)
.replace("{TMP_MOUNT_BLOCK}", &nsjail_tmp_mount_block)
.replace("#{DEV}", DEV_CONF_NSJAIL), // .replace("{BUILD}", &build_dir),
)?;
let mut cmd = Command::new(NSJAIL_PATH.as_str());
@@ -812,7 +812,10 @@ mount {{
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace("{NSJAIL_TMPFS_SIZE}", &resolve_nsjail_tmpfs_size_bytes().await)
.replace(
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
let mut cmd = Command::new(NSJAIL_PATH.as_str());
+5 -5
View File
@@ -23,7 +23,7 @@ use windmill_queue::{append_logs, CanceledBy};
use crate::{
common::{
build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file,
get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes,
get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block,
start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
@@ -481,8 +481,8 @@ pub async fn build_rust_crate(
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.replace("{BUILD}", &build_dir),
)?;
@@ -706,8 +706,8 @@ pub async fn handle_rust_job(
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace("{SHARED_MOUNT}", shared_mount)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
"{TMP_MOUNT_BLOCK}",
&resolve_nsjail_tmp_mount_block(job_dir).await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
+6
View File
@@ -687,6 +687,12 @@ lazy_static::lazy_static! {
/// `DEFAULT_NSJAIL_TMPFS_SIZE_BYTES` (800MB).
pub static ref NSJAIL_TMPFS_SIZE_MB: Arc<RwLock<Option<i64>>> = Arc::new(RwLock::new(None));
/// Selects how `/tmp` is backed inside nsjail sandboxes. `Some("disk")`
/// switches to a bind mount on `{JOB_DIR}/jail_tmp` (disk-backed); any
/// other value (including `None` or `Some("tmpfs")`) keeps the historical
/// RAM-backed tmpfs sized by `nsjail_tmpfs_size_mb`.
pub static ref NSJAIL_TMP_BACKING: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
/// Optional mirror URL for `uv python install`. Wires to the `UV_PYTHON_INSTALL_MIRROR`
/// env var when forwarded to uv. Can be set via the `UV_PYTHON_INSTALL_MIRROR` env var
/// or the `uv_python_install_mirror` instance setting.
@@ -72,6 +72,20 @@
return false
}
}
// Hide the nsjail-only settings only when isolation is *explicitly* a
// non-nsjail mode. When `job_isolation` is unset, nsjail may still be
// enabled via the legacy env-driven path (`DISABLE_NSJAIL=false`), so
// keep the controls reachable.
if (setting == 'nsjail_tmp_backing' || setting == 'nsjail_tmpfs_size_mb') {
const isolation = values['job_isolation']
if (isolation === 'none' || isolation === 'unshare') {
return false
}
}
// The tmpfs size knob is meaningless when /tmp is disk-backed.
if (setting == 'nsjail_tmpfs_size_mb' && values['nsjail_tmp_backing'] === 'disk') {
return false
}
return true
}
@@ -245,11 +245,25 @@ export const settings: Record<string, Setting[]> = {
}
]
},
{
label: 'Nsjail /tmp backing',
key: 'nsjail_tmp_backing',
fieldType: 'select',
description:
'How <code>/tmp</code> is backed inside the nsjail sandbox. <strong>RAM (tmpfs)</strong> is the default — fast, with a hard size cap from <em>Nsjail tmpfs size</em>, but consumes worker memory. <strong>Disk (bind mount)</strong> uses a per-job directory on the worker disk — no RAM cost, but the only remaining per-file ceiling is <code>rlimit_fsize</code> (~1GB for python/ansible, unbounded for most other languages because they set <code>disable_rl: true</code>); pair with host disk monitoring or quotas.',
storage: 'setting',
placeholder: 'tmpfs',
defaultValue: () => 'tmpfs',
select_items: [
{ label: 'RAM (tmpfs) — default', value: 'tmpfs' },
{ label: 'Disk (bind mount)', value: 'disk' }
]
},
{
label: 'Nsjail tmpfs size (MB)',
key: 'nsjail_tmpfs_size_mb',
description:
'Override the size of the <code>/tmp</code> tmpfs mount inside the nsjail sandbox (in MB). When left empty, defaults to 800MB. Only applies when the job isolation mode is set to Nsjail.',
'Override the size of the <code>/tmp</code> tmpfs mount inside the nsjail sandbox (in MB). When left empty, defaults to 800MB. Only applies when <em>Nsjail /tmp backing</em> is RAM (tmpfs).',
fieldType: 'number',
placeholder: '800',
storage: 'setting'