diff --git a/.env.local b/.env.local new file mode 100644 index 0000000000..abe2102d93 --- /dev/null +++ b/.env.local @@ -0,0 +1,3 @@ +BACKEND_PORT=8010 +FRONTEND_PORT=3010 +REMOTE=http://localhost:8010 diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c8ed2f12a7..d19d2b00b3 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -17462,6 +17462,7 @@ dependencies = [ "sha2 0.10.9", "sqlx", "tar", + "tempfile", "tiberius", "tokio", "tokio-postgres 0.7.13", diff --git a/backend/windmill-sandbox/src/nsjail.rs b/backend/windmill-sandbox/src/nsjail.rs index b2e8c8cb2c..696039077c 100644 --- a/backend/windmill-sandbox/src/nsjail.rs +++ b/backend/windmill-sandbox/src/nsjail.rs @@ -33,8 +33,11 @@ pub fn build_sandbox_mounts(setup: &SandboxSetupState) -> String { /// 2. Move the overlay root mount (`dst: "/"`) to be the FIRST mount block. /// nsjail applies mounts in order, so the root bind must come before /// tmpfs/bind mounts that go on top of it. +/// 3. Re-add read-only bind mounts for `runtime_bins` paths that fall under +/// stripped system dirs (e.g. `/usr/bin/bun`). These are layered on top of +/// the overlay so the host runtime binary is available inside the sandbox. /// If no overlay marker is present, returns the config unchanged. -pub fn finalize_nsjail_config(config: &str) -> String { +pub fn finalize_nsjail_config(config: &str, runtime_bins: &[&str]) -> String { if !config.contains(OVERLAY_MARKER) { return config.to_string(); } @@ -87,8 +90,7 @@ pub fn finalize_nsjail_config(config: &str) -> String { if should_remove_block { result_lines.truncate(mount_block_start); } else if is_overlay_root { - overlay_root_lines - .extend_from_slice(&result_lines[mount_block_start..]); + overlay_root_lines.extend_from_slice(&result_lines[mount_block_start..]); overlay_root_lines.push(lines[i]); result_lines.truncate(mount_block_start); } else { @@ -129,7 +131,22 @@ pub fn finalize_nsjail_config(config: &str) -> String { } } - result_lines.join("\n") + // Re-add bind mounts for runtime binaries that fall under stripped system dirs. + // These are appended at the end so they layer on top of the overlay root. + let runtime_mounts: String = runtime_bins + .iter() + .filter(|bin| system_dirs.iter().any(|d| bin.starts_with(d))) + .map(|bin| { + format!("\nmount {{\n src: \"{bin}\"\n dst: \"{bin}\"\n is_bind: true\n}}") + }) + .collect(); + + let mut result = result_lines.join("\n"); + if !runtime_mounts.is_empty() { + result.push_str(&runtime_mounts); + result.push('\n'); + } + result } #[cfg(test)] @@ -155,7 +172,10 @@ mod tests { let mut setup = SandboxSetupState::default(); setup.volume_mounts.insert( "data".to_string(), - (PathBuf::from("/job/volumes/data"), "/workspace/data".to_string()), + ( + PathBuf::from("/job/volumes/data"), + "/workspace/data".to_string(), + ), ); let mounts = build_sandbox_mounts(&setup); assert!(mounts.contains("dst: \"/workspace/data\"")); @@ -170,11 +190,17 @@ mod tests { let mut setup = SandboxSetupState::default(); setup.volume_mounts.insert( "input".to_string(), - (PathBuf::from("/job/volumes/input"), "/mnt/input".to_string()), + ( + PathBuf::from("/job/volumes/input"), + "/mnt/input".to_string(), + ), ); setup.volume_mounts.insert( "output".to_string(), - (PathBuf::from("/job/volumes/output"), "/mnt/output".to_string()), + ( + PathBuf::from("/job/volumes/output"), + "/mnt/output".to_string(), + ), ); let mounts = build_sandbox_mounts(&setup); assert!(mounts.contains("/mnt/input")); @@ -212,7 +238,10 @@ mod tests { }; setup.volume_mounts.insert( "data".to_string(), - (PathBuf::from("/job/volumes/data"), "/workspace/data".to_string()), + ( + PathBuf::from("/job/volumes/data"), + "/workspace/data".to_string(), + ), ); let mounts = build_sandbox_mounts(&setup); assert!(mounts.contains(OVERLAY_MARKER)); @@ -230,7 +259,7 @@ mod tests { let config = "name: \"test\"\n\ mount {\n src: \"/bin\"\n dst: \"/bin\"\n is_bind: true\n}\n\ mount {\n dst: \"/tmp\"\n fstype: \"tmpfs\"\n rw: true\n}\n"; - let result = finalize_nsjail_config(config); + let result = finalize_nsjail_config(config, &[]); assert_eq!(result, config); } @@ -245,14 +274,17 @@ mod tests { # SANDBOX_OVERLAY_ACTIVE\n\ mount {\n src: \"/job/merged\"\n dst: \"/\"\n is_bind: true\n rw: true\n}\n\ mount {\n dst: \"/tmp\"\n fstype: \"tmpfs\"\n rw: true\n}\n"; - let result = finalize_nsjail_config(config); + let result = finalize_nsjail_config(config, &[]); for dir in &["/bin", "/lib", "/lib64", "/usr", "/etc"] { assert!( !result.contains(&format!("dst: \"{dir}\"")), "System dir {dir} should be stripped" ); } - assert!(result.contains("dst: \"/\""), "Overlay root mount preserved"); + assert!( + result.contains("dst: \"/\""), + "Overlay root mount preserved" + ); assert!(result.contains("dst: \"/tmp\""), "tmpfs mount preserved"); } @@ -265,10 +297,13 @@ mod tests { mount {\n dst: \"/tmp\"\n fstype: \"tmpfs\"\n rw: true\n}\n\ # SANDBOX_OVERLAY_ACTIVE\n\ mount {\n src: \"/job/merged\"\n dst: \"/\"\n is_bind: true\n rw: true\n}\n"; - let result = finalize_nsjail_config(config); + let result = finalize_nsjail_config(config, &[]); assert!(!result.contains("dst: \"/bin\""), "/bin stripped"); assert!(result.contains("dst: \"/dev/null\""), "/dev/null kept"); - assert!(result.contains("dst: \"/opt/microsoft\""), "/opt/microsoft kept"); + assert!( + result.contains("dst: \"/opt/microsoft\""), + "/opt/microsoft kept" + ); assert!(result.contains("dst: \"/tmp\""), "/tmp tmpfs kept"); } @@ -277,11 +312,33 @@ mod tests { let config = "name: \"test\"\n\ # SANDBOX_OVERLAY_ACTIVE\n\ mount {\n src: \"/job/merged\"\n dst: \"/\"\n is_bind: true\n}\n"; - let result = finalize_nsjail_config(config); + let result = finalize_nsjail_config(config, &[]); assert!(!result.contains(OVERLAY_MARKER)); assert!(result.contains("dst: \"/\"")); } + #[test] + fn test_finalize_readds_runtime_bins_under_stripped_dirs() { + let config = "name: \"test\"\n\ + mount {\n src: \"/usr\"\n dst: \"/usr\"\n is_bind: true\n}\n\ + # SANDBOX_OVERLAY_ACTIVE\n\ + mount {\n src: \"/job/merged\"\n dst: \"/\"\n is_bind: true\n rw: true\n}\n"; + let result = finalize_nsjail_config(config, &["/usr/bin/bun"]); + assert!(!result.contains("dst: \"/usr\""), "/usr stripped"); + assert!(result.contains("dst: \"/usr/bin/bun\""), "bun re-added"); + assert!(result.contains("src: \"/usr/bin/bun\""), "bun src set"); + } + + #[test] + fn test_finalize_skips_runtime_bins_not_under_system_dirs() { + let config = "name: \"test\"\n\ + # SANDBOX_OVERLAY_ACTIVE\n\ + mount {\n src: \"/job/merged\"\n dst: \"/\"\n is_bind: true\n}\n"; + let result = finalize_nsjail_config(config, &["/tmp/windmill/cache/py_runtime"]); + // Should NOT add an extra mount since /tmp is not a stripped system dir + assert_eq!(result.matches("mount {").count(), 1); + } + #[test] fn test_mount_block_syntax() { let setup = SandboxSetupState { diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 463679cbdd..da9a44da5d 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -143,5 +143,8 @@ hyper-tls = { workspace = true, optional = true } hyper-util = { workspace = true, optional = true } rcgen = { workspace = true, optional = true } +[dev-dependencies] +tempfile.workspace = true + [build-dependencies] libffi-sys = { workspace = true, optional = true } diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index ace33153d0..039a42064b 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -33,9 +33,9 @@ use crate::{ read_and_check_result, start_child_process, transform_json, OccupancyMetrics, }, handle_child::handle_child, + is_sandboxing_enabled, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, - is_sandboxing_enabled, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, - PY_INSTALL_DIR, TZ_ENV, + DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -1192,6 +1192,7 @@ mount {{ "{ADDITIONAL_PYTHON_PATHS}", additional_python_paths_folders.as_str(), ), + &[], ); let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?; } else { diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 8300a03e1c..96b46252fd 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -186,8 +186,8 @@ exit $exit_status // Use nsjail if globally enabled, script has #sandbox annotation, // or sandbox mounts are present (snapshot/volume annotations) - let nsjail = - (is_sandboxing_enabled() || annotation.sandbox || !shared_mount.is_empty()) && is_regular_job; + let nsjail = (is_sandboxing_enabled() || annotation.sandbox || !shared_mount.is_empty()) + && is_regular_job; let child = if nsjail { let nsjail_config = windmill_sandbox::finalize_nsjail_config( &NSJAIL_CONFIG_RUN_BASH_CONTENT @@ -196,6 +196,7 @@ exit $exit_status .replace("{SHARED_MOUNT}", shared_mount) .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), + &[], ); let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?; let mut cmd_args = vec![ diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 7e0b16bd0b..6d5d1422fc 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -47,9 +47,9 @@ use windmill_common::{ DB, }; +use crate::global_cache::{exists_in_cache, save_cache}; #[cfg(all(feature = "enterprise", feature = "parquet"))] use windmill_object_store::attempt_fetch_bytes; -use crate::global_cache::{exists_in_cache, save_cache}; use windmill_parser::Typ; @@ -977,8 +977,7 @@ pub async fn handle_bun_job( } }; - let (cache, logs) = - crate::global_cache::load_cache(&local_path, &remote_path, false).await; + let (cache, logs) = crate::global_cache::load_cache(&local_path, &remote_path, false).await; (cache, logs, local_path, remote_path) } else { (false, "".to_string(), "".to_string(), "".to_string()) @@ -1452,6 +1451,11 @@ try {{ //do not cache local dependencies let child = if is_sandboxing_enabled() || !shared_mount.is_empty() { + let runtime_bin = if annotation.nodejs { + &*NODE_BIN_PATH + } else { + &*BUN_PATH + }; let nsjail_config = windmill_sandbox::finalize_nsjail_config( &NSJAIL_CONFIG_RUN_BUN_CONTENT .replace("{LANG}", if annotation.nodejs { "nodejs" } else { "bun" }) @@ -1470,6 +1474,7 @@ try {{ ) .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), + &[runtime_bin], ); let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?; diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index 880e89ae5e..1280edb670 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -13,14 +13,11 @@ use itertools::Itertools; #[cfg(feature = "csharp")] use tokio::{fs::File, io::AsyncReadExt, process::Command}; #[cfg(feature = "csharp")] -use windmill_common::{ - utils::calculate_hash, - worker::write_file, -}; +use windmill_common::{utils::calculate_hash, worker::write_file}; -use windmill_common::error::{self, Error}; #[cfg(feature = "csharp")] use crate::global_cache::save_cache; +use windmill_common::error::{self, Error}; #[cfg(feature = "csharp")] use windmill_queue::append_logs; @@ -519,8 +516,7 @@ pub async fn handle_csharp_job( let bin_path = format!("{}/{hash}", CSHARP_CACHE_DIR); let remote_path = format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}"); - let (cache, cache_logs) = - crate::global_cache::load_cache(&bin_path, &remote_path, false).await; + let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let cache_logs = if cache { #[cfg(unix)] @@ -595,6 +591,7 @@ pub async fn handle_csharp_job( .replace("{SHARED_MOUNT}", shared_mount) .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), + &[], ); write_file(job_dir, "run.config.proto", &nsjail_config)?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index d8936d1dc4..c709c1c924 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -2,6 +2,7 @@ use crate::{common::MaybeLock, get_proxy_envs_for_lang}; use std::{collections::HashMap, fs::DirBuilder, process::Stdio}; use windmill_common::scripts::ScriptLang; +use crate::global_cache::save_cache; use itertools::Itertools; use serde_json::value::RawValue; use tokio::{ @@ -15,7 +16,6 @@ use windmill_common::{ utils::calculate_hash, worker::{write_file, Connection, GoAnnotations}, }; -use crate::global_cache::save_cache; use windmill_parser_go::{parse_go_imports, REQUIRE_PARSE}; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -110,8 +110,7 @@ pub async fn handle_go_job( let hash = calculate_hash(&format!("{}{:?}v2", inner_content, &maybe_lock)); let bin_path = format!("{}/{hash}", GO_BIN_CACHE_DIR); let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}"); - let (cache, cache_logs) = - crate::global_cache::load_cache(&bin_path, &remote_path, false).await; + let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let (skip_go_mod, skip_tidy) = if cache { (true, true) @@ -347,6 +346,7 @@ func Run(req Req) (interface{{}}, error){{ .replace("{SHARED_MOUNT}", shared_mount) .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), + &[], ); let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index b9bde76577..4b5e5fa679 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, path::PathBuf, process::Stdio}; +use crate::global_cache::save_cache; use anyhow::{anyhow, bail}; use async_recursion::async_recursion; use itertools::Itertools; @@ -15,7 +16,6 @@ use windmill_common::{ utils::calculate_hash, worker::{copy_dir_recursively, write_file, Connection}, }; -use crate::global_cache::save_cache; use windmill_parser::Arg; use windmill_parser_java::parse_java_sig_meta; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -605,6 +605,7 @@ async fn run<'a>( .replace("{CACHE_DIR}", JAVA_CACHE_DIR) .replace("{SHARED_MOUNT}", &shared_mount) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), + &[], ); write_file(job_dir, "run.config.proto", &nsjail_config)?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index 33dac4ba7e..7badfc0893 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -16,8 +16,8 @@ use crate::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, - get_proxy_envs_for_lang, handle_child, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, - TRACING_PROXY_CA_CERT_PATH, + get_proxy_envs_for_lang, handle_child, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, + PATH_ENV, TRACING_PROXY_CA_CERT_PATH, }; use windmill_common::client::AuthedClient; use windmill_common::scripts::ScriptLang; @@ -253,6 +253,7 @@ async fn run<'a>( .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), + &[], ); write_file(job_dir, "run.config.proto", &nsjail_config)?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index cf7ccd4d73..0325137c88 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -22,7 +22,7 @@ use crate::{ get_reserved_variables, read_result, start_child_process, MaybeLock, OccupancyMetrics, }, handle_child::handle_child, - COMPOSER_CACHE_DIR, COMPOSER_PATH, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, + is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, }; use windmill_common::client::AuthedClient; @@ -299,6 +299,7 @@ try {{ .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount), + &[], ); let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?; diff --git a/backend/windmill-worker/src/pwsh_executor.rs b/backend/windmill-worker/src/pwsh_executor.rs index 918c9b5e1e..3eddfd2b39 100644 --- a/backend/windmill-worker/src/pwsh_executor.rs +++ b/backend/windmill-worker/src/pwsh_executor.rs @@ -524,6 +524,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) .replace("{CACHE_DIR}", POWERSHELL_CACHE_DIR), + &[], ); let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?; let cmd_args = vec![ diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index d469721248..8f0949fbb4 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -784,7 +784,7 @@ except BaseException as e: #[cfg(windows)] let additional_python_paths_folders = additional_python_paths_folders.replace(":", ";"); - if is_sandboxing_enabled() { + if is_sandboxing_enabled() || !shared_mount.is_empty() { let shared_deps = additional_python_paths .into_iter() .map(|pp| { @@ -815,6 +815,7 @@ mount {{ ) .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), + &[], ); let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?; } else { diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index 116b71cef5..062ecf8d18 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -801,6 +801,7 @@ mount {{ .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), + &[], ); write_file(job_dir, "run.config.proto", &nsjail_config)?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 95eb71999b..15d222654f 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -5,6 +5,7 @@ use std::{collections::HashMap, process::Stdio}; use uuid::Uuid; use windmill_parser_rust::parse_rust_deps_into_manifest; +use crate::global_cache::save_cache; use itertools::Itertools; use tokio::{ fs::{create_dir_all, File}, @@ -16,7 +17,6 @@ use windmill_common::{ utils::calculate_hash, worker::{write_file, Connection}, }; -use crate::global_cache::save_cache; use windmill_queue::MiniPulledJob; use windmill_queue::{append_logs, CanceledBy}; @@ -611,8 +611,7 @@ pub async fn handle_rust_job( let reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; - let (cache, cache_logs) = - crate::global_cache::load_cache(&bin_path, &remote_path, false).await; + let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let cache_logs = if cache { let target = format!("{job_dir}/main"); @@ -673,6 +672,7 @@ pub async fn handle_rust_job( .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{SHARED_MOUNT}", shared_mount), + &[], ); let _ = write_file(job_dir, "run.config.proto", &nsjail_config)?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/sandbox_setup.rs b/backend/windmill-worker/src/sandbox_setup.rs index 187a8e7de4..8c9f352dc3 100644 --- a/backend/windmill-worker/src/sandbox_setup.rs +++ b/backend/windmill-worker/src/sandbox_setup.rs @@ -39,11 +39,17 @@ mod tests { volume_mounts: HashMap::from([ ( "data".to_string(), - (PathBuf::from("/tmp/job123/volumes/data"), "/workspace/data".to_string()), + ( + PathBuf::from("/tmp/job123/volumes/data"), + "/workspace/data".to_string(), + ), ), ( "models".to_string(), - (PathBuf::from("/tmp/job123/volumes/models"), "/workspace/models".to_string()), + ( + PathBuf::from("/tmp/job123/volumes/models"), + "/workspace/models".to_string(), + ), ), ]), }; @@ -56,7 +62,7 @@ mod tests { .replace("{SHARED_MOUNT}", &sandbox_mounts) .replace("{TRACING_PROXY_CA_CERT_PATH}", "/dev/null"); - let final_config = finalize_nsjail_config(&config); + let final_config = finalize_nsjail_config(&config, &[]); for dir in &["/bin", "/lib", "/lib64", "/usr", "/etc"] { assert!(!final_config.contains(&format!("dst: \"{dir}\""))); @@ -96,7 +102,7 @@ mod tests { .replace("{ADDITIONAL_PYTHON_PATHS}", "/tmp") .replace("{TRACING_PROXY_CA_CERT_PATH}", "/dev/null"); - let final_config = finalize_nsjail_config(&config); + let final_config = finalize_nsjail_config(&config, &[]); for dir in &["/bin", "/lib", "/lib64", "/usr", "/etc"] { assert!(!final_config.contains(&format!("dst: \"{dir}\""))); @@ -113,7 +119,10 @@ mod tests { let mut setup = SandboxSetupState::default(); setup.volume_mounts.insert( "cache".to_string(), - (PathBuf::from("/tmp/job456/volumes/cache"), "/tmp/pip-cache".to_string()), + ( + PathBuf::from("/tmp/job456/volumes/cache"), + "/tmp/pip-cache".to_string(), + ), ); let sandbox_mounts = build_sandbox_mounts(&setup); @@ -129,7 +138,7 @@ mod tests { .replace("{ADDITIONAL_PYTHON_PATHS}", "/tmp") .replace("{TRACING_PROXY_CA_CERT_PATH}", "/dev/null"); - let final_config = finalize_nsjail_config(&config); + let final_config = finalize_nsjail_config(&config, &[]); assert!(final_config.contains("dst: \"/bin\"")); assert!(final_config.contains("dst: \"/lib\"")); @@ -147,7 +156,7 @@ mod tests { .replace("{SHARED_MOUNT}", "") .replace("{TRACING_PROXY_CA_CERT_PATH}", "/dev/null"); - let final_config = finalize_nsjail_config(&config); + let final_config = finalize_nsjail_config(&config, &[]); assert!(final_config.contains("dst: \"/bin\"")); assert!(final_config.contains("dst: \"/lib\"")); @@ -194,11 +203,17 @@ mod tests { .replace("{TRACING_PROXY_CA_CERT_PATH}", "/dev/null") .replace("#{DEV}", ""); - let final_config = finalize_nsjail_config(&config); + let final_config = finalize_nsjail_config(&config, &[]); std::fs::write(job_dir.join("run.config.proto"), &final_config).unwrap(); let output = Command::new("nsjail") - .args(["--config", "run.config.proto", "--", "/bin/bash", "wrapper.sh"]) + .args([ + "--config", + "run.config.proto", + "--", + "/bin/bash", + "wrapper.sh", + ]) .current_dir(job_dir) .output() .expect("Failed to execute nsjail"); @@ -247,10 +262,9 @@ mod tests { std::fs::write(vol_dir.join("input.txt"), "volume data here").unwrap(); let mut setup = SandboxSetupState::default(); - setup.volume_mounts.insert( - "data".to_string(), - (vol_dir, "/workspace/data".to_string()), - ); + setup + .volume_mounts + .insert("data".to_string(), (vol_dir, "/workspace/data".to_string())); let sandbox_mounts = build_sandbox_mounts(&setup); let (stdout, stderr, exit_code) = run_nsjail_bash( @@ -409,6 +423,706 @@ mod tests { assert_eq!(parsed["status"], "ok"); assert_eq!(parsed["count"], 42); } + + // ===================================================================== + // Python nsjail integration tests + // ===================================================================== + + fn python3_path() -> Option { + Command::new("which") + .arg("python3") + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) + } + + fn run_nsjail_python( + job_dir: &std::path::Path, + wrapper_script: &str, + extra_shared_mount: &str, + runtime_bins: &[&str], + ) -> (String, String, i32) { + let python3 = python3_path().expect("python3 must be available"); + let py_prefix = String::from_utf8( + Command::new(&python3) + .args(["-c", "import sys; print(sys.prefix)"]) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + + std::fs::write(job_dir.join("wrapper.py"), wrapper_script).unwrap(); + std::fs::write(job_dir.join("main.py"), "# placeholder").unwrap(); + std::fs::write(job_dir.join("args.json"), "{}").unwrap(); + std::fs::write(job_dir.join("result.json"), "").unwrap(); + + let raw_config = include_str!("../nsjail/run.python3.config.proto"); + let config = raw_config + .replace("{JOB_DIR}", &job_dir.to_string_lossy()) + .replace("{MAIN}", "main") + .replace("{CLONE_NEWUSER}", "true") + .replace("{SHARED_MOUNT}", extra_shared_mount) + .replace("{SHARED_DEPENDENCIES}", "") + .replace("{PY_INSTALL_DIR}", &py_prefix) + .replace("{GLOBAL_SITE_PACKAGES}", "/nonexistent") + .replace("{ADDITIONAL_PYTHON_PATHS}", "/tmp") + .replace("{TRACING_PROXY_CA_CERT_PATH}", "/dev/null") + .replace("#{DEV}", ""); + + let final_config = finalize_nsjail_config(&config, runtime_bins); + std::fs::write(job_dir.join("run.config.proto"), &final_config).unwrap(); + + let output = Command::new("nsjail") + .args([ + "--config", + "run.config.proto", + "--", + &python3, + "-u", + "/tmp/wrapper.py", + ]) + .current_dir(job_dir) + .output() + .expect("Failed to execute nsjail"); + + ( + String::from_utf8_lossy(&output.stdout).to_string(), + String::from_utf8_lossy(&output.stderr).to_string(), + output.status.code().unwrap_or(-1), + ) + } + + #[test] + fn test_nsjail_basic_python_execution() { + if !nsjail_available() || python3_path().is_none() { + eprintln!("Skipping: nsjail or python3 not available"); + return; + } + + let job_dir = tempfile::tempdir().unwrap(); + let (stdout, stderr, exit_code) = run_nsjail_python( + job_dir.path(), + "import json, sys\njson.dump({'lang': 'python', 'v': sys.version_info[0]}, open('result.json', 'w'))\n", + "", + &[], + ); + + if exit_code != 0 { + eprintln!("nsjail stdout: {stdout}"); + eprintln!("nsjail stderr: {stderr}"); + } + assert_eq!(exit_code, 0, "python nsjail should exit successfully"); + + let result_json = std::fs::read_to_string(job_dir.path().join("result.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(result_json.trim()).unwrap(); + assert_eq!(parsed["lang"], "python"); + assert_eq!(parsed["v"], 3); + } + + #[test] + fn test_nsjail_python_with_overlay_root() { + if !nsjail_available() || python3_path().is_none() { + eprintln!("Skipping: nsjail or python3 not available"); + return; + } + + let job_dir = tempfile::tempdir().unwrap(); + let setup = SandboxSetupState { + overlay: Some(OverlayMount { + merged: PathBuf::from("/"), + upper: PathBuf::from("/unused"), + work: PathBuf::from("/unused"), + is_fuse: false, + }), + volume_mounts: HashMap::new(), + }; + let sandbox_mounts = build_sandbox_mounts(&setup); + + let (stdout, stderr, exit_code) = run_nsjail_python( + job_dir.path(), + "import json, platform\njson.dump({'os': platform.system()}, open('result.json', 'w'))\n", + &sandbox_mounts, + &[], + ); + + if exit_code != 0 { + eprintln!("nsjail stdout: {stdout}"); + eprintln!("nsjail stderr: {stderr}"); + } + assert_eq!(exit_code, 0); + + let result_json = std::fs::read_to_string(job_dir.path().join("result.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(result_json.trim()).unwrap(); + assert_eq!(parsed["os"], "Linux"); + } + + #[test] + fn test_nsjail_python_volume_readwrite() { + if !nsjail_available() || python3_path().is_none() { + eprintln!("Skipping: nsjail or python3 not available"); + return; + } + + let job_dir = tempfile::tempdir().unwrap(); + let vol_dir = job_dir.path().join("volumes/data"); + std::fs::create_dir_all(&vol_dir).unwrap(); + std::fs::write(vol_dir.join("input.txt"), "python volume data").unwrap(); + + let mut setup = SandboxSetupState::default(); + setup.volume_mounts.insert( + "data".to_string(), + (vol_dir.clone(), "/workspace/data".to_string()), + ); + let sandbox_mounts = build_sandbox_mounts(&setup); + + let (stdout, stderr, exit_code) = run_nsjail_python( + job_dir.path(), + "import json\n\ + data = open('/workspace/data/input.txt').read().strip()\n\ + open('/workspace/data/output.txt', 'w').write('written by python')\n\ + json.dump({'read': data}, open('result.json', 'w'))\n", + &sandbox_mounts, + &[], + ); + + if exit_code != 0 { + eprintln!("nsjail stdout: {stdout}"); + eprintln!("nsjail stderr: {stderr}"); + } + assert_eq!(exit_code, 0); + + let result_json = std::fs::read_to_string(job_dir.path().join("result.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(result_json.trim()).unwrap(); + assert_eq!(parsed["read"], "python volume data"); + + let written = std::fs::read_to_string(vol_dir.join("output.txt")).unwrap(); + assert_eq!(written, "written by python"); + } + + #[test] + fn test_nsjail_python_overlay_with_volume() { + if !nsjail_available() || python3_path().is_none() { + eprintln!("Skipping: nsjail or python3 not available"); + return; + } + + let job_dir = tempfile::tempdir().unwrap(); + let vol_dir = job_dir.path().join("volumes/shared"); + std::fs::create_dir_all(&vol_dir).unwrap(); + std::fs::write(vol_dir.join("config.yaml"), "key: value").unwrap(); + + let mut setup = SandboxSetupState { + overlay: Some(OverlayMount { + merged: PathBuf::from("/"), + upper: PathBuf::from("/unused"), + work: PathBuf::from("/unused"), + is_fuse: false, + }), + volume_mounts: HashMap::new(), + }; + setup.volume_mounts.insert( + "shared".to_string(), + (vol_dir.clone(), "/tmp/volumes/shared".to_string()), + ); + let sandbox_mounts = build_sandbox_mounts(&setup); + + let (stdout, stderr, exit_code) = run_nsjail_python( + job_dir.path(), + "import json\n\ + data = open('/tmp/volumes/shared/config.yaml').read().strip()\n\ + open('/tmp/volumes/shared/output.txt', 'w').write('py output')\n\ + json.dump({'read': data}, open('result.json', 'w'))\n", + &sandbox_mounts, + &[], + ); + + if exit_code != 0 { + eprintln!("nsjail stdout: {stdout}"); + eprintln!("nsjail stderr: {stderr}"); + } + assert_eq!(exit_code, 0); + + let result_json = std::fs::read_to_string(job_dir.path().join("result.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(result_json.trim()).unwrap(); + assert_eq!(parsed["read"], "key: value"); + + let output = std::fs::read_to_string(vol_dir.join("output.txt")).unwrap(); + assert_eq!(output, "py output"); + } + + // ===================================================================== + // Bun nsjail integration tests + // ===================================================================== + + fn bun_available() -> bool { + Command::new("bun") + .arg("--version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) + } + + fn bun_binary_path() -> String { + String::from_utf8(Command::new("which").arg("bun").output().unwrap().stdout) + .unwrap() + .trim() + .to_string() + } + + fn run_nsjail_bun( + job_dir: &std::path::Path, + wrapper_script: &str, + extra_shared_mount: &str, + runtime_bins: &[&str], + ) -> (String, String, i32) { + let bun = bun_binary_path(); + + std::fs::write(job_dir.join("wrapper.mjs"), wrapper_script).unwrap(); + std::fs::write(job_dir.join("args.json"), "{}").unwrap(); + std::fs::write(job_dir.join("result.json"), "").unwrap(); + + let raw_config = include_str!("../nsjail/run.bun.config.proto"); + let config = raw_config + .replace("{JOB_DIR}", &job_dir.to_string_lossy()) + .replace("{LANG}", "bun") + .replace("{CLONE_NEWUSER}", "true") + .replace("{SHARED_MOUNT}", extra_shared_mount) + .replace("{TRACING_PROXY_CA_CERT_PATH}", "/dev/null") + .replace("#{DEV}", ""); + + let final_config = finalize_nsjail_config(&config, runtime_bins); + std::fs::write(job_dir.join("run.config.proto"), &final_config).unwrap(); + + let output = Command::new("nsjail") + .args([ + "--config", + "run.config.proto", + "--", + &bun, + "run", + "wrapper.mjs", + ]) + .current_dir(job_dir) + .output() + .expect("Failed to execute nsjail"); + + ( + String::from_utf8_lossy(&output.stdout).to_string(), + String::from_utf8_lossy(&output.stderr).to_string(), + output.status.code().unwrap_or(-1), + ) + } + + #[test] + fn test_nsjail_basic_bun_execution() { + if !nsjail_available() || !bun_available() { + eprintln!("Skipping: nsjail or bun not available"); + return; + } + + let job_dir = tempfile::tempdir().unwrap(); + let (stdout, stderr, exit_code) = run_nsjail_bun( + job_dir.path(), + "import { writeFileSync } from 'fs';\n\ + writeFileSync('result.json', JSON.stringify({ lang: 'bun' }));\n", + "", + &[], + ); + + if exit_code != 0 { + eprintln!("nsjail stdout: {stdout}"); + eprintln!("nsjail stderr: {stderr}"); + } + assert_eq!(exit_code, 0, "bun nsjail should exit successfully"); + + let result_json = std::fs::read_to_string(job_dir.path().join("result.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(result_json.trim()).unwrap(); + assert_eq!(parsed["lang"], "bun"); + } + + #[test] + fn test_nsjail_bun_with_overlay_root() { + if !nsjail_available() || !bun_available() { + eprintln!("Skipping: nsjail or bun not available"); + return; + } + + let bun = bun_binary_path(); + let job_dir = tempfile::tempdir().unwrap(); + let setup = SandboxSetupState { + overlay: Some(OverlayMount { + merged: PathBuf::from("/"), + upper: PathBuf::from("/unused"), + work: PathBuf::from("/unused"), + is_fuse: false, + }), + volume_mounts: HashMap::new(), + }; + let sandbox_mounts = build_sandbox_mounts(&setup); + + // With overlay, /usr is stripped so bun binary needs runtime_bins + let (stdout, stderr, exit_code) = run_nsjail_bun( + job_dir.path(), + "import { writeFileSync } from 'fs';\n\ + writeFileSync('result.json', JSON.stringify({ lang: 'bun', overlay: true }));\n", + &sandbox_mounts, + &[&bun], + ); + + if exit_code != 0 { + eprintln!("nsjail stdout: {stdout}"); + eprintln!("nsjail stderr: {stderr}"); + } + assert_eq!(exit_code, 0); + + let result_json = std::fs::read_to_string(job_dir.path().join("result.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(result_json.trim()).unwrap(); + assert_eq!(parsed["lang"], "bun"); + assert_eq!(parsed["overlay"], true); + } + + #[test] + fn test_nsjail_bun_volume_readwrite() { + if !nsjail_available() || !bun_available() { + eprintln!("Skipping: nsjail or bun not available"); + return; + } + + let job_dir = tempfile::tempdir().unwrap(); + let vol_dir = job_dir.path().join("volumes/data"); + std::fs::create_dir_all(&vol_dir).unwrap(); + std::fs::write(vol_dir.join("input.txt"), "bun volume data").unwrap(); + + let mut setup = SandboxSetupState::default(); + setup.volume_mounts.insert( + "data".to_string(), + (vol_dir.clone(), "/workspace/data".to_string()), + ); + let sandbox_mounts = build_sandbox_mounts(&setup); + + let (stdout, stderr, exit_code) = run_nsjail_bun( + job_dir.path(), + "import { readFileSync, writeFileSync } from 'fs';\n\ + const data = readFileSync('/workspace/data/input.txt', 'utf8').trim();\n\ + writeFileSync('/workspace/data/output.txt', 'written by bun');\n\ + writeFileSync('result.json', JSON.stringify({ read: data }));\n", + &sandbox_mounts, + &[], + ); + + if exit_code != 0 { + eprintln!("nsjail stdout: {stdout}"); + eprintln!("nsjail stderr: {stderr}"); + } + assert_eq!(exit_code, 0); + + let result_json = std::fs::read_to_string(job_dir.path().join("result.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(result_json.trim()).unwrap(); + assert_eq!(parsed["read"], "bun volume data"); + + let written = std::fs::read_to_string(vol_dir.join("output.txt")).unwrap(); + assert_eq!(written, "written by bun"); + } + + #[test] + fn test_nsjail_bun_overlay_with_volume() { + if !nsjail_available() || !bun_available() { + eprintln!("Skipping: nsjail or bun not available"); + return; + } + + let bun = bun_binary_path(); + let job_dir = tempfile::tempdir().unwrap(); + let vol_dir = job_dir.path().join("volumes/shared"); + std::fs::create_dir_all(&vol_dir).unwrap(); + std::fs::write(vol_dir.join("config.yaml"), "key: value").unwrap(); + + let mut setup = SandboxSetupState { + overlay: Some(OverlayMount { + merged: PathBuf::from("/"), + upper: PathBuf::from("/unused"), + work: PathBuf::from("/unused"), + is_fuse: false, + }), + volume_mounts: HashMap::new(), + }; + setup.volume_mounts.insert( + "shared".to_string(), + (vol_dir.clone(), "/tmp/volumes/shared".to_string()), + ); + let sandbox_mounts = build_sandbox_mounts(&setup); + + let (stdout, stderr, exit_code) = run_nsjail_bun( + job_dir.path(), + "import { readFileSync, writeFileSync } from 'fs';\n\ + const data = readFileSync('/tmp/volumes/shared/config.yaml', 'utf8').trim();\n\ + writeFileSync('/tmp/volumes/shared/output.txt', 'bun output');\n\ + writeFileSync('result.json', JSON.stringify({ read: data }));\n", + &sandbox_mounts, + &[&bun], + ); + + if exit_code != 0 { + eprintln!("nsjail stdout: {stdout}"); + eprintln!("nsjail stderr: {stderr}"); + } + assert_eq!(exit_code, 0); + + let result_json = std::fs::read_to_string(job_dir.path().join("result.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(result_json.trim()).unwrap(); + assert_eq!(parsed["read"], "key: value"); + + let output = std::fs::read_to_string(vol_dir.join("output.txt")).unwrap(); + assert_eq!(output, "bun output"); + } + + // ===================================================================== + // Go nsjail integration tests + // ===================================================================== + + fn go_available() -> bool { + Command::new("go") + .arg("version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) + } + + fn compile_go_binary(job_dir: &std::path::Path, go_source: &str) -> bool { + let src_dir = tempfile::tempdir().unwrap(); + std::fs::write(src_dir.path().join("main.go"), go_source).unwrap(); + Command::new("go") + .args([ + "build", + "-o", + &job_dir.join("main").to_string_lossy(), + "main.go", + ]) + .current_dir(src_dir.path()) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) + } + + fn run_nsjail_go( + job_dir: &std::path::Path, + go_source: &str, + extra_shared_mount: &str, + ) -> (String, String, i32) { + if !compile_go_binary(job_dir, go_source) { + return (String::new(), "Go compilation failed".to_string(), -1); + } + + std::fs::write(job_dir.join("args.json"), "{}").unwrap(); + std::fs::write(job_dir.join("result.json"), "").unwrap(); + + let raw_config = include_str!("../nsjail/run.go.config.proto"); + let config = raw_config + .replace("{JOB_DIR}", &job_dir.to_string_lossy()) + .replace("{CLONE_NEWUSER}", "true") + .replace("{SHARED_MOUNT}", extra_shared_mount) + .replace("{TRACING_PROXY_CA_CERT_PATH}", "/dev/null") + .replace("#{DEV}", ""); + + let final_config = finalize_nsjail_config(&config, &[]); + std::fs::write(job_dir.join("run.config.proto"), &final_config).unwrap(); + + let output = Command::new("nsjail") + .args(["--config", "run.config.proto", "--", "/tmp/go/main"]) + .current_dir(job_dir) + .output() + .expect("Failed to execute nsjail"); + + ( + String::from_utf8_lossy(&output.stdout).to_string(), + String::from_utf8_lossy(&output.stderr).to_string(), + output.status.code().unwrap_or(-1), + ) + } + + #[test] + fn test_nsjail_basic_go_execution() { + if !nsjail_available() || !go_available() { + eprintln!("Skipping: nsjail or go not available"); + return; + } + + let job_dir = tempfile::tempdir().unwrap(); + let (stdout, stderr, exit_code) = run_nsjail_go( + job_dir.path(), + r#"package main +import "os" +func main() { + os.WriteFile("result.json", []byte(`{"lang":"go"}`), 0644) +}"#, + "", + ); + + if exit_code != 0 { + eprintln!("nsjail stdout: {stdout}"); + eprintln!("nsjail stderr: {stderr}"); + } + assert_eq!(exit_code, 0, "go nsjail should exit successfully"); + + let result_json = std::fs::read_to_string(job_dir.path().join("result.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(result_json.trim()).unwrap(); + assert_eq!(parsed["lang"], "go"); + } + + #[test] + fn test_nsjail_go_with_overlay_root() { + if !nsjail_available() || !go_available() { + eprintln!("Skipping: nsjail or go not available"); + return; + } + + let job_dir = tempfile::tempdir().unwrap(); + let setup = SandboxSetupState { + overlay: Some(OverlayMount { + merged: PathBuf::from("/"), + upper: PathBuf::from("/unused"), + work: PathBuf::from("/unused"), + is_fuse: false, + }), + volume_mounts: HashMap::new(), + }; + let sandbox_mounts = build_sandbox_mounts(&setup); + + let (stdout, stderr, exit_code) = run_nsjail_go( + job_dir.path(), + r#"package main +import ( + "os" + "runtime" +) +func main() { + os.WriteFile("result.json", []byte(`{"os":"`+runtime.GOOS+`"}`), 0644) +}"#, + &sandbox_mounts, + ); + + if exit_code != 0 { + eprintln!("nsjail stdout: {stdout}"); + eprintln!("nsjail stderr: {stderr}"); + } + assert_eq!(exit_code, 0); + + let result_json = std::fs::read_to_string(job_dir.path().join("result.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(result_json.trim()).unwrap(); + assert_eq!(parsed["os"], "linux"); + } + + #[test] + fn test_nsjail_go_volume_readwrite() { + if !nsjail_available() || !go_available() { + eprintln!("Skipping: nsjail or go not available"); + return; + } + + let job_dir = tempfile::tempdir().unwrap(); + let vol_dir = job_dir.path().join("volumes/data"); + std::fs::create_dir_all(&vol_dir).unwrap(); + std::fs::write(vol_dir.join("input.txt"), "go volume data").unwrap(); + + let mut setup = SandboxSetupState::default(); + setup.volume_mounts.insert( + "data".to_string(), + (vol_dir.clone(), "/workspace/data".to_string()), + ); + let sandbox_mounts = build_sandbox_mounts(&setup); + + let (stdout, stderr, exit_code) = run_nsjail_go( + job_dir.path(), + r#"package main +import ( + "encoding/json" + "os" + "strings" +) +func main() { + data, _ := os.ReadFile("/workspace/data/input.txt") + os.WriteFile("/workspace/data/output.txt", []byte("written by go"), 0644) + result, _ := json.Marshal(map[string]string{"read": strings.TrimSpace(string(data))}) + os.WriteFile("result.json", result, 0644) +}"#, + &sandbox_mounts, + ); + + if exit_code != 0 { + eprintln!("nsjail stdout: {stdout}"); + eprintln!("nsjail stderr: {stderr}"); + } + assert_eq!(exit_code, 0); + + let result_json = std::fs::read_to_string(job_dir.path().join("result.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(result_json.trim()).unwrap(); + assert_eq!(parsed["read"], "go volume data"); + + let written = std::fs::read_to_string(vol_dir.join("output.txt")).unwrap(); + assert_eq!(written, "written by go"); + } + + #[test] + fn test_nsjail_go_overlay_with_volume() { + if !nsjail_available() || !go_available() { + eprintln!("Skipping: nsjail or go not available"); + return; + } + + let job_dir = tempfile::tempdir().unwrap(); + let vol_dir = job_dir.path().join("volumes/shared"); + std::fs::create_dir_all(&vol_dir).unwrap(); + std::fs::write(vol_dir.join("config.yaml"), "key: value").unwrap(); + + let mut setup = SandboxSetupState { + overlay: Some(OverlayMount { + merged: PathBuf::from("/"), + upper: PathBuf::from("/unused"), + work: PathBuf::from("/unused"), + is_fuse: false, + }), + volume_mounts: HashMap::new(), + }; + setup.volume_mounts.insert( + "shared".to_string(), + (vol_dir.clone(), "/tmp/volumes/shared".to_string()), + ); + let sandbox_mounts = build_sandbox_mounts(&setup); + + let (stdout, stderr, exit_code) = run_nsjail_go( + job_dir.path(), + r#"package main +import ( + "encoding/json" + "os" + "strings" +) +func main() { + data, _ := os.ReadFile("/tmp/volumes/shared/config.yaml") + os.WriteFile("/tmp/volumes/shared/output.txt", []byte("go output"), 0644) + result, _ := json.Marshal(map[string]string{"read": strings.TrimSpace(string(data))}) + os.WriteFile("result.json", result, 0644) +}"#, + &sandbox_mounts, + ); + + if exit_code != 0 { + eprintln!("nsjail stdout: {stdout}"); + eprintln!("nsjail stderr: {stderr}"); + } + assert_eq!(exit_code, 0); + + let result_json = std::fs::read_to_string(job_dir.path().join("result.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(result_json.trim()).unwrap(); + assert_eq!(parsed["read"], "key: value"); + + let output = std::fs::read_to_string(vol_dir.join("output.txt")).unwrap(); + assert_eq!(output, "go output"); + } } // ========================================================================= @@ -416,8 +1130,6 @@ mod tests { // ========================================================================= mod overlay_integration { - use super::*; - #[tokio::test] async fn test_mount_overlay_read_write_semantics() { let snapshot_dir = tempfile::tempdir().unwrap(); @@ -464,7 +1176,9 @@ mod tests { "from snapshot" ); - windmill_sandbox::unmount_overlay(&overlay).await.expect("unmount should succeed"); + windmill_sandbox::unmount_overlay(&overlay) + .await + .expect("unmount should succeed"); assert!(!overlay.merged.exists()); } } @@ -478,7 +1192,10 @@ mod tests { use std::process::Command; fn crane_path() -> Option { - for path in &["crane", &format!("{}/go/bin/crane", std::env::var("HOME").unwrap_or_default())] { + for path in &[ + "crane", + &format!("{}/go/bin/crane", std::env::var("HOME").unwrap_or_default()), + ] { if Command::new(path) .arg("version") .output() @@ -517,7 +1234,9 @@ mod tests { use std::io::Cursor; use tar::Archive; let mut archive = Archive::new(Cursor::new(&crane_output.stdout)); - archive.unpack(rootfs_dir.path()).expect("Failed to unpack crane output"); + archive + .unpack(rootfs_dir.path()) + .expect("Failed to unpack crane output"); } assert!( @@ -608,20 +1327,23 @@ mod tests { .replace("{SHARED_MOUNT}", &sandbox_mounts) .replace("{TRACING_PROXY_CA_CERT_PATH}", "/dev/null") .replace("#{DEV}", ""); - let final_config = finalize_nsjail_config(&config); + let final_config = finalize_nsjail_config(&config, &[]); std::fs::write(job_dir.path().join("run.config.proto"), &final_config).unwrap(); let output = Command::new("nsjail") - .args(["--config", "run.config.proto", "--", "/bin/sh", "wrapper.sh"]) + .args([ + "--config", + "run.config.proto", + "--", + "/bin/sh", + "wrapper.sh", + ]) .current_dir(job_dir.path()) .output() .expect("Failed to run nsjail"); if !output.status.success() { - eprintln!( - "nsjail stderr: {}", - String::from_utf8_lossy(&output.stderr) - ); + eprintln!("nsjail stderr: {}", String::from_utf8_lossy(&output.stderr)); } assert_eq!(output.status.code().unwrap_or(-1), 0); @@ -630,4 +1352,274 @@ mod tests { assert!(result.trim().starts_with("3.")); } } + + // ========================================================================= + // Integration tests: volume round-trip via filesystem object store + // ========================================================================= + + #[cfg(feature = "parquet")] + mod volume_store_integration { + use super::*; + use std::process::Command; + + fn nsjail_available() -> bool { + Command::new("nsjail") + .arg("--help") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) + } + + fn run_nsjail_bash_simple( + job_dir: &std::path::Path, + main_script: &str, + extra_shared_mount: &str, + ) -> i32 { + std::fs::write(job_dir.join("main.sh"), main_script).unwrap(); + std::fs::write( + job_dir.join("wrapper.sh"), + "#!/bin/bash\n/bin/bash /tmp/main.sh\n", + ) + .unwrap(); + std::fs::write(job_dir.join("result.json"), "").unwrap(); + std::fs::write(job_dir.join("result.out"), "").unwrap(); + std::fs::write(job_dir.join("result2.out"), "").unwrap(); + + let raw_config = include_str!("../nsjail/run.bash.config.proto"); + let config = raw_config + .replace("{JOB_DIR}", &job_dir.to_string_lossy()) + .replace("{CLONE_NEWUSER}", "true") + .replace("{SHARED_MOUNT}", extra_shared_mount) + .replace("{TRACING_PROXY_CA_CERT_PATH}", "/dev/null") + .replace("#{DEV}", ""); + + let final_config = finalize_nsjail_config(&config, &[]); + std::fs::write(job_dir.join("run.config.proto"), &final_config).unwrap(); + + let output = Command::new("nsjail") + .args([ + "--config", + "run.config.proto", + "--", + "/bin/bash", + "wrapper.sh", + ]) + .current_dir(job_dir) + .output() + .expect("Failed to execute nsjail"); + + if !output.status.success() { + eprintln!("nsjail stderr: {}", String::from_utf8_lossy(&output.stderr)); + } + output.status.code().unwrap_or(-1) + } + + /// Test: store volume data in filesystem object store → retrieve → mount in nsjail → read + #[tokio::test] + async fn test_volume_read_via_filesystem_store() { + if !nsjail_available() { + eprintln!("Skipping: nsjail not available"); + return; + } + + // 1. Create filesystem-backed object store + let store_root = tempfile::tempdir().unwrap(); + let store = + windmill_object_store::build_filesystem_client(store_root.path().to_str().unwrap()) + .unwrap(); + + // 2. Create volume content and upload to store + let source_dir = tempfile::tempdir().unwrap(); + std::fs::write(source_dir.path().join("data.txt"), "hello from store\n").unwrap(); + std::fs::create_dir(source_dir.path().join("subdir")).unwrap(); + std::fs::write( + source_dir.path().join("subdir/nested.txt"), + "nested content\n", + ) + .unwrap(); + + let tar_bytes = windmill_sandbox::tar_gz(source_dir.path()).unwrap(); + windmill_object_store::put_bytes_to_store( + store.clone(), + "sandbox/volumes/test-ws/mydata.tar.gz", + tar_bytes.into(), + ) + .await + .unwrap(); + + // 3. Download from store and unpack to volume dir + let job_dir = tempfile::tempdir().unwrap(); + let vol_dir = job_dir.path().join("volumes/mydata"); + std::fs::create_dir_all(&vol_dir).unwrap(); + + let downloaded = windmill_object_store::fetch_bytes_from_store( + store.clone(), + "sandbox/volumes/test-ws/mydata.tar.gz", + ) + .await + .unwrap() + .expect("should find stored volume data"); + windmill_sandbox::untar_gz(&downloaded, &vol_dir).unwrap(); + + // 4. Mount in nsjail and verify from inside sandbox + let mut setup = SandboxSetupState::default(); + setup.volume_mounts.insert( + "mydata".to_string(), + (vol_dir, "/workspace/data".to_string()), + ); + let sandbox_mounts = build_sandbox_mounts(&setup); + + let exit_code = run_nsjail_bash_simple( + job_dir.path(), + "#!/bin/bash\n\ + cat /workspace/data/data.txt > /tmp/result.out\n\ + cat /workspace/data/subdir/nested.txt >> /tmp/result.out\n", + &sandbox_mounts, + ); + assert_eq!(exit_code, 0); + + let result = std::fs::read_to_string(job_dir.path().join("result.out")).unwrap(); + let lines: Vec<&str> = result.trim().lines().collect(); + assert_eq!(lines[0], "hello from store"); + assert_eq!(lines[1], "nested content"); + } + + /// Test: nsjail writes to volume → tar → store → retrieve → verify round-trip + #[tokio::test] + async fn test_volume_write_roundtrip_via_filesystem_store() { + if !nsjail_available() { + eprintln!("Skipping: nsjail not available"); + return; + } + + let store_root = tempfile::tempdir().unwrap(); + let store = + windmill_object_store::build_filesystem_client(store_root.path().to_str().unwrap()) + .unwrap(); + + // 1. Run nsjail with an empty volume mount, write files inside + let job_dir = tempfile::tempdir().unwrap(); + let vol_dir = job_dir.path().join("volumes/output"); + std::fs::create_dir_all(&vol_dir).unwrap(); + + let mut setup = SandboxSetupState::default(); + setup.volume_mounts.insert( + "output".to_string(), + (vol_dir.clone(), "/workspace/output".to_string()), + ); + let sandbox_mounts = build_sandbox_mounts(&setup); + + let exit_code = run_nsjail_bash_simple( + job_dir.path(), + "#!/bin/bash\n\ + echo 'written in sandbox' > /workspace/output/result.txt\n\ + mkdir -p /workspace/output/subdir\n\ + echo 'nested write' > /workspace/output/subdir/nested.txt\n\ + echo 'done' > /tmp/result.out\n", + &sandbox_mounts, + ); + assert_eq!(exit_code, 0); + + // 2. Upload the written volume to the filesystem store + let tar_bytes = windmill_sandbox::tar_gz(&vol_dir).unwrap(); + windmill_object_store::put_bytes_to_store( + store.clone(), + "sandbox/volumes/test-ws/output.tar.gz", + tar_bytes.into(), + ) + .await + .unwrap(); + + // 3. Download to a fresh directory and verify content survived the round-trip + let verify_dir = tempfile::tempdir().unwrap(); + let downloaded = windmill_object_store::fetch_bytes_from_store( + store.clone(), + "sandbox/volumes/test-ws/output.tar.gz", + ) + .await + .unwrap() + .expect("should find stored volume data"); + windmill_sandbox::untar_gz(&downloaded, verify_dir.path()).unwrap(); + + let content = std::fs::read_to_string(verify_dir.path().join("result.txt")).unwrap(); + assert_eq!(content.trim(), "written in sandbox"); + + let nested = + std::fs::read_to_string(verify_dir.path().join("subdir/nested.txt")).unwrap(); + assert_eq!(nested.trim(), "nested write"); + } + + /// Test: simulate two successive job runs sharing a volume via the object store. + /// Job 1 writes to the volume, volume gets persisted to the store. + /// Job 2 reads the persisted volume and verifies the data. + #[tokio::test] + async fn test_volume_persistence_across_jobs_via_filesystem_store() { + if !nsjail_available() { + eprintln!("Skipping: nsjail not available"); + return; + } + + let store_root = tempfile::tempdir().unwrap(); + let store = + windmill_object_store::build_filesystem_client(store_root.path().to_str().unwrap()) + .unwrap(); + let s3_key = "sandbox/volumes/test-ws/shared.tar.gz"; + + // --- Job 1: write to volume --- + let job1_dir = tempfile::tempdir().unwrap(); + let vol1_dir = job1_dir.path().join("volumes/shared"); + std::fs::create_dir_all(&vol1_dir).unwrap(); + + let mut setup1 = SandboxSetupState::default(); + setup1.volume_mounts.insert( + "shared".to_string(), + (vol1_dir.clone(), "/workspace/shared".to_string()), + ); + let mounts1 = build_sandbox_mounts(&setup1); + + let exit1 = run_nsjail_bash_simple( + job1_dir.path(), + "#!/bin/bash\n\ + echo 'counter=1' > /workspace/shared/state.txt\n\ + echo 'done' > /tmp/result.out\n", + &mounts1, + ); + assert_eq!(exit1, 0); + + // Persist volume to store + let tar_bytes = windmill_sandbox::tar_gz(&vol1_dir).unwrap(); + windmill_object_store::put_bytes_to_store(store.clone(), s3_key, tar_bytes.into()) + .await + .unwrap(); + + // --- Job 2: read volume from store --- + let job2_dir = tempfile::tempdir().unwrap(); + let vol2_dir = job2_dir.path().join("volumes/shared"); + std::fs::create_dir_all(&vol2_dir).unwrap(); + + let downloaded = windmill_object_store::fetch_bytes_from_store(store.clone(), s3_key) + .await + .unwrap() + .expect("volume should exist in store"); + windmill_sandbox::untar_gz(&downloaded, &vol2_dir).unwrap(); + + let mut setup2 = SandboxSetupState::default(); + setup2.volume_mounts.insert( + "shared".to_string(), + (vol2_dir.clone(), "/workspace/shared".to_string()), + ); + let mounts2 = build_sandbox_mounts(&setup2); + + let exit2 = run_nsjail_bash_simple( + job2_dir.path(), + "#!/bin/bash\n\ + cat /workspace/shared/state.txt > /tmp/result.out\n", + &mounts2, + ); + assert_eq!(exit2, 0); + + let result = std::fs::read_to_string(job2_dir.path().join("result.out")).unwrap(); + assert_eq!(result.trim(), "counter=1"); + } + } } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 735201d293..227d3b09d2 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -3415,8 +3415,7 @@ pub async fn handle_queued_job( }, JobKind::SnapshotBuild => match conn { Connection::Sql(db) => { - Box::pin(crate::snapshot_build::handle_snapshot_build(&job, db, conn)) - .await + Box::pin(crate::snapshot_build::handle_snapshot_build(&job, db, conn)).await } Connection::Http(_) => { return Err(Error::internal_err( @@ -4193,18 +4192,19 @@ mount {{ if let Some(ref snap) = sandbox_config.snapshot { let snapshot_path = windmill_sandbox::ensure_snapshot_cached( - &job.workspace_id, &snap.name, &snap.tag, db, + &job.workspace_id, + &snap.name, + &snap.tag, + db, ) .await?; - setup.overlay = - Some(windmill_sandbox::mount_overlay(&snapshot_path, job_dir).await?); + setup.overlay = Some(windmill_sandbox::mount_overlay(&snapshot_path, job_dir).await?); } for (vol_name, mount_path) in &sandbox_config.volumes { let local_dir = format!("{job_dir}/volumes/{vol_name}"); - std::fs::create_dir_all(&local_dir).map_err(|e| { - Error::ExecutionErr(format!("Failed to create volume dir: {e}")) - })?; + std::fs::create_dir_all(&local_dir) + .map_err(|e| Error::ExecutionErr(format!("Failed to create volume dir: {e}")))?; windmill_sandbox::download_volume( &job.workspace_id, vol_name, @@ -4224,6 +4224,17 @@ mount {{ None }; + if sandbox_config.snapshot.is_some() || !sandbox_config.volumes.is_empty() { + let mut sandbox_logs = "\n--- SANDBOX ---\n".to_string(); + if let Some(ref snap) = sandbox_config.snapshot { + sandbox_logs.push_str(&format!("Snapshot: {}:{}\n", snap.name, snap.tag)); + } + for (vol_name, mount_path) in &sandbox_config.volumes { + sandbox_logs.push_str(&format!("Volume: {} -> {}\n", vol_name, mount_path)); + } + append_logs(&job.id, &job.workspace_id, sandbox_logs, conn).await; + } + let envs = build_envs(envs.as_ref())?; let Some(language) = language else { @@ -4682,13 +4693,9 @@ mount {{ if let Some(ref setup) = sandbox_setup { if let Some(db) = conn.as_sql() { for (vol_name, (local_dir, _)) in &setup.volume_mounts { - if let Err(e) = windmill_sandbox::upload_volume( - &job.workspace_id, - vol_name, - local_dir, - db, - ) - .await + if let Err(e) = + windmill_sandbox::upload_volume(&job.workspace_id, vol_name, local_dir, db) + .await { tracing::error!("Failed to upload volume {vol_name}: {e}"); } diff --git a/scripts/worktree-env b/scripts/worktree-env new file mode 100755 index 0000000000..765d7331ba --- /dev/null +++ b/scripts/worktree-env @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +port_in_use() { + lsof -nP -iTCP:"$1" -sTCP:LISTEN &>/dev/null +} + +find_port() { + local port=$1 + while port_in_use "$port"; do + ((port++)) + done + echo "$port" +} + +if [[ -z "${WM_SLOT:-}" ]]; then + # Auto-assign: find the first slot (1-99) where both ports are free + # Slot 0 (8000/3000) is reserved for the main worktree + for slot in $(seq 1 99); do + bp=$((8000 + slot * 10)) + fp=$((3000 + slot * 10)) + if ! port_in_use "$bp" && ! port_in_use "$fp"; then + WM_SLOT=$slot + break + fi + done + if [[ -z "${WM_SLOT:-}" ]]; then + echo "ERROR: No available slot found (tried 1-99)" >&2 + exit 1 + fi + echo "Auto-assigned slot $WM_SLOT" +fi + +# Slot-based: predictable ports for SSH forwarding +# Slot 0 = 8000/3000, slot 1 = 8010/3010, slot 2 = 8020/3020, etc. +backend_port=$((8000 + WM_SLOT * 10)) +frontend_port=$((3000 + WM_SLOT * 10)) + +if port_in_use "$backend_port" || port_in_use "$frontend_port"; then + echo "ERROR: Slot $WM_SLOT ports ($backend_port/$frontend_port) already in use" >&2 + exit 1 +fi + +# Generate .env.local with port overrides +cat > .env.local <